diff --git a/docs/toolchains.md b/docs/toolchains.md index de495d5122..3ddce7b97e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -240,12 +240,22 @@ existing attributes: via {attr}`python.override.minor_mapping`. * Per-version control of the coverage tool used using {attr}`python.single_version_platform_override.coverage_tool`. +* Control of shared `libpython` files using the `libpython` attribute on + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override`, or + {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions via {bzl:obj}`python.single_version_override` or {bzl:obj}`python.single_version_platform_override`. * Adding additional Python versions dynamically from a manifest file or URL via {attr}`python.override.add_runtime_manifest_files` or {attr}`python.override.add_runtime_manifest_urls`. +The `libpython` attribute accepts `auto`, `include`, or `exclude`, and defaults +to `auto`. In automatic mode, shared `libpython` files are excluded only for +recognized Astral Python Standalone builds from `20250517` onward, which are +known to statically link `libpython` into the interpreter. Unknown, custom, and +older runtimes retain the shared libraries. Use `include` or `exclude` to +override this behavior when mirroring or customizing a runtime. + ### Registering custom runtimes Because the python-build-standalone project has _thousands_ of prebuilt runtimes diff --git a/news/py-zipapp-size.fixed.md b/news/py-zipapp-size.fixed.md new file mode 100644 index 0000000000..508882b9da --- /dev/null +++ b/news/py-zipapp-size.fixed.md @@ -0,0 +1,4 @@ +(zipapp) Reduced self-contained archive sizes by preserving Python executable +symlinks instead of storing each alias as another copy of the interpreter, and +by omitting shared `libpython` files from recognized statically linked Astral +runtime builds. diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index b4ff84f14a..b88fb724a4 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -780,6 +780,7 @@ bzl_library( srcs = ["python_repository.bzl"], deps = [ ":auth", + ":pbs_manifest", ":repo_utils", ":text_util", "//python:versions", diff --git a/python/private/hermetic_runtime_repo_setup.bzl b/python/private/hermetic_runtime_repo_setup.bzl index 20b0324894..d61a040fe6 100644 --- a/python/private/hermetic_runtime_repo_setup.bzl +++ b/python/private/hermetic_runtime_repo_setup.bzl @@ -29,6 +29,7 @@ def define_hermetic_runtime_toolchain_impl( name, extra_files_glob_include, extra_files_glob_exclude, + interpreter_has_static_libpython = False, python_version, python_bin, coverage_tool): @@ -45,6 +46,8 @@ def define_hermetic_runtime_toolchain_impl( binaries). extra_files_glob_exclude: {type}`list[str]` additional glob exclude patterns for the target runtime files. + interpreter_has_static_libpython: {type}`bool` whether the interpreter + includes libpython statically. python_version: {type}`str` The Python version, in `major.minor.micro` format. python_bin: {type}`str` The path to the Python binary within the @@ -78,6 +81,8 @@ def define_hermetic_runtime_toolchain_impl( # During pyc creation, temp files named *.pyc.NNN are created "**/__pycache__/*.pyc.*", ] + if interpreter_has_static_libpython: + files_exclude.append("lib/libpython*.so*") files_exclude += extra_files_glob_exclude native.filegroup( diff --git a/python/private/pbs_manifest.bzl b/python/private/pbs_manifest.bzl index 86434dc0ea..ad019df0a8 100644 --- a/python/private/pbs_manifest.bzl +++ b/python/private/pbs_manifest.bzl @@ -1,5 +1,11 @@ """Helper functions to parse python-build-standalone manifests.""" +_ASTRAL_STATIC_LIBPYTHON_RELEASE = 20250517 +_ASTRAL_RELEASE_URL_PREFIXES = [ + "https://github.com/astral-sh/python-build-standalone/releases/download/", + "https://releases.astral.sh/github/python-build-standalone/releases/download/", +] + def parse_filename(filename): """Parses a python-build-standalone filename (or URL) into its components. @@ -112,6 +118,28 @@ def parse_filename(filename): "vendor": vendor, } +# buildifier: disable=function-docstring-args +# buildifier: disable=function-docstring-return +# urls: list[str], release_filename: str -> bool +def is_astral_static_libpython_build(urls, release_filename): + """Whether an Astral build includes libpython statically in its interpreter.""" + parsed = parse_filename(release_filename) + if not parsed: + return False + + build_version = parsed["build_version"] + if ( + not build_version.isdigit() or + int(build_version) < _ASTRAL_STATIC_LIBPYTHON_RELEASE + ): + return False + + return any([ + url.startswith(prefix) + for url in urls + for prefix in _ASTRAL_RELEASE_URL_PREFIXES + ]) + def parse_runtime_manifest(content): """Parses the SHA256SUMS file content into a list of structs. diff --git a/python/private/python.bzl b/python/private/python.bzl index 9fed9393ae..c7ca018102 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -612,6 +612,8 @@ def _process_single_version_overrides(*, tag, _fail = fail, default): kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content if tag.distutils: kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + if tag.libpython and tag.libpython != "auto": + kwargs.setdefault(tag.python_version, {})["libpython"] = tag.libpython def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if not _validate_version(tag.python_version, _fail = _fail): @@ -641,6 +643,8 @@ def _process_single_version_platform_overrides(*, tag, _fail = fail, default): if tag.urls: available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + if tag.libpython and tag.libpython != "auto": + available_versions[tag.python_version].setdefault("libpython", {})[tag.platform] = tag.libpython # If platform is customized, or doesn't exist, (re)define one. if ((tag.target_compatible_with or tag.target_settings or tag.os_name or tag.arch) or @@ -719,6 +723,7 @@ def _process_global_overrides(*, tag, default, _fail = fail): forwarded_attrs = sorted(AUTH_ATTRS) + [ "base_urls", + "libpython", "register_all_versions", ] for key in forwarded_attrs: @@ -1348,6 +1353,19 @@ dependencies are introduced. doc = """Deprecated; do not use. This attribute has no effect.""", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", + values = ["auto", "include", "exclude"], + ), "minor_mapping": attr.string_dict( mandatory = False, doc = """\ @@ -1424,6 +1442,19 @@ class. "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", + values = ["auto", "include", "exclude"], + ), "patch_strip": attr.int( mandatory = False, doc = "Same as the --strip argument of Unix patch.", @@ -1493,6 +1524,19 @@ The coverage tool to be used for a particular Python interpreter. This can overr `rules_python` defaults. """, ), + "libpython": attr.string( + default = "auto", + doc = """Whether to include shared libpython files. + +Valid values are `auto`, `include`, and `exclude`. With `auto`, recognized +Astral Python Standalone builds from 20250517 onward exclude the shared +libraries; other runtimes retain them. + +:::{versionadded} VERSION_NEXT_PATCH +::: +""", + values = ["auto", "include", "exclude"], + ), "os_name": attr.string( doc = """ The host OS the runtime is compatible with. diff --git a/python/private/python_register_toolchains.bzl b/python/private/python_register_toolchains.bzl index c6f827be22..770c6a6f3b 100644 --- a/python/private/python_register_toolchains.bzl +++ b/python/private/python_register_toolchains.bzl @@ -40,6 +40,7 @@ def python_register_toolchains( register_toolchains = True, register_coverage_tool = False, set_python_version_constraint = False, + libpython = None, tool_versions = None, platforms = PLATFORMS, minor_mapping = None, @@ -68,6 +69,8 @@ def python_register_toolchains( set_python_version_constraint: {type}`bool` When set to `True`, `target_compatible_with` for the toolchains will include a version constraint. + libpython: {type}`str` controls shared libpython files: `auto`, + `include`, or `exclude`. tool_versions: {type}`dict` contains a mapping of version with SHASUM and platform info. If not supplied, the defaults in python/versions.bzl will be used. @@ -145,6 +148,7 @@ def python_register_toolchains( urls = urls, strip_prefix = strip_prefix, coverage_tool = coverage_tool, + libpython = libpython or tool_versions[python_version].get("libpython", {}).get(platform, "auto"), **kwargs ) if register_toolchains: diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9c44971117..3fd0928a01 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -17,6 +17,7 @@ load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY") load(":auth.bzl", "get_auth") +load(":pbs_manifest.bzl", "is_astral_static_libpython_build") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -162,6 +163,12 @@ def _python_repository_impl(rctx): *python_version_info ) urls = rctx.attr.urls or [rctx.attr.url] + if rctx.attr.libpython == "include": + interpreter_has_static_libpython = False + elif rctx.attr.libpython == "exclude": + interpreter_has_static_libpython = True + else: + interpreter_has_static_libpython = is_astral_static_libpython_build(urls, release_filename) auth = get_auth(rctx, urls) if INSTALL_ONLY in release_filename: @@ -277,16 +284,18 @@ load("@rules_python//python/private:hermetic_runtime_repo_setup.bzl", "define_he package(default_visibility = ["//visibility:public"]) define_hermetic_runtime_toolchain_impl( - name = "define_runtime", - extra_files_glob_include = {extra_files_glob_include}, - extra_files_glob_exclude = {extra_files_glob_exclude}, - python_version = {python_version}, - python_bin = {python_bin}, - coverage_tool = {coverage_tool}, + name = "define_runtime", + extra_files_glob_include = {extra_files_glob_include}, + extra_files_glob_exclude = {extra_files_glob_exclude}, + interpreter_has_static_libpython = {interpreter_has_static_libpython}, + python_version = {python_version}, + python_bin = {python_bin}, + coverage_tool = {coverage_tool}, ) """.format( extra_files_glob_exclude = render.list(glob_exclude), extra_files_glob_include = render.list(glob_include), + interpreter_has_static_libpython = str(interpreter_has_static_libpython), python_bin = render.str(python_bin), python_version = render.str(rctx.attr.python_version), coverage_tool = render.str(coverage_tool), @@ -361,6 +370,11 @@ For more information see {attr}`py_runtime.coverage_tool`. doc = "Noop, will be removed in the next major release", mandatory = False, ), + "libpython": attr.string( + default = "auto", + doc = "Whether to include shared libpython files: auto, include, or exclude.", + values = ["auto", "include", "exclude"], + ), "netrc": attr.string( doc = ".netrc file to use for authentication; mirrors the eponymous attribute from http_archive", ), diff --git a/tests/py_zipapp/venv_zipapp_test.py b/tests/py_zipapp/venv_zipapp_test.py index bd26d533a3..b906073fce 100644 --- a/tests/py_zipapp/venv_zipapp_test.py +++ b/tests/py_zipapp/venv_zipapp_test.py @@ -84,6 +84,12 @@ def test_zipapp_structure(self): # On Windows, pyvenv.cfg and bin/python3 are generated at runtime. if os.name != "nt": + self.assertFalse( + any( + "/lib/libpython" in name and ".so" in name for name in namelist + ), + "Statically linked Python should not bundle libpython", + ) self.assertHasPathMatchingSuffix(namelist, "/pyvenv.cfg") # The venv directory name depends on the target name, so find it diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index ef7ccf034f..22cd5c26b0 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -419,6 +419,28 @@ def _test_auth_overrides(env): _tests.append(_test_auth_overrides) +def _test_libpython_override(env): + py = parse_modules( + module_ctx = python_ext.mctx( + python_ext.module( + name = "my_module", + is_root = True, + override = [ + python_ext.override(libpython = "exclude"), + ], + toolchain = [python_ext.toolchain(python_version = "3.12")], + ), + _rules_python_module(), + ), + logger = repo_utils.logger(verbosity_level = 0, name = "python"), + ) + + env.expect.that_dict(py.config.default).contains_at_least({ + "libpython": "exclude", + }) + +_tests.append(_test_libpython_override) + def _test_add_target_settings(env): py = parse_modules( module_ctx = python_ext.mctx( diff --git a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl index 4493576147..8c9176aeb2 100644 --- a/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl +++ b/tests/python_bzlmod_ext/parse_runtime_manifest_tests.bzl @@ -4,7 +4,7 @@ load("@bazel_skylib//lib:structs.bzl", "structs") load("@rules_testing//lib:analysis_test.bzl", "analysis_test") load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") -load("//python/private:pbs_manifest.bzl", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility +load("//python/private:pbs_manifest.bzl", "is_astral_static_libpython_build", "parse_filename", "parse_runtime_manifest") # buildifier: disable=bzl-visibility _tests = [] @@ -97,6 +97,47 @@ def _test_parse_filename_baseline_impl(env, target): _tests.append(_test_parse_filename_baseline) +def _test_is_astral_static_libpython_build(name): + rt_util.helper_target( + native.filegroup, + name = name + "_subject", + ) + analysis_test( + name = name, + target = name + "_subject", + impl = _test_is_astral_static_libpython_build_impl, + ) + +def _test_is_astral_static_libpython_build_impl(env, target): + _ = target # @unused + + github_url = "https://github.com/astral-sh/python-build-standalone/releases/download/20250517/archive.tar.gz" + mirror_url = "https://releases.astral.sh/github/python-build-standalone/releases/download/20250517/archive.tar.gz" + cutoff_filename = "cpython-3.13.4+20250517-x86_64-unknown-linux-gnu-install_only.tar.gz" + + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [mirror_url], + cutoff_filename, + )).equals(True) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "cpython-3.13.3+20250516-x86_64-unknown-linux-gnu-install_only.tar.gz", + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + ["https://example.com/20250604/archive.tar.gz"], + cutoff_filename, + )).equals(False) + env.expect.that_bool(is_astral_static_libpython_build( + [github_url], + "custom-python.tar.gz", + )).equals(False) + +_tests.append(_test_is_astral_static_libpython_build) + def _test_parse_runtime_manifest(name): """Sets up the manifest file parsing test. diff --git a/tests/support/mocks/python_ext.bzl b/tests/support/mocks/python_ext.bzl index 4c68e87e50..419afe8188 100644 --- a/tests/support/mocks/python_ext.bzl +++ b/tests/support/mocks/python_ext.bzl @@ -32,6 +32,7 @@ def _override(**kwargs): "available_python_versions": [], "base_urls": ["https://github.com/astral-sh/python-build-standalone/releases/download"], "ignore_root_user_error": True, + "libpython": "auto", "minor_mapping": {}, "register_all_versions": False, "runtime_manifest_sha": "", @@ -53,6 +54,7 @@ def _single_version_override(**kwargs): attrs = { "distutils": None, "distutils_content": "", + "libpython": "auto", "patch_strip": 0, "patches": [], "python_version": "", @@ -68,6 +70,7 @@ def _single_version_platform_override(**kwargs): attrs = { "arch": "", "coverage_tool": None, + "libpython": "auto", "os_name": "", "patch_strip": 0, "patches": [], diff --git a/tests/tools/zipapp/zipper_test.py b/tests/tools/zipapp/zipper_test.py index e4f25c5e21..e0ff1557d5 100644 --- a/tests/tools/zipapp/zipper_test.py +++ b/tests/tools/zipapp/zipper_test.py @@ -115,6 +115,65 @@ def test_create_zip_with_direct_symlink(tmp_path): ) +def test_create_zip_with_source_symlink_marked_as_regular(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + target_path = tmp_path / "python3.14" + target_path.write_text("python") + symlink_path = tmp_path / "python" + symlink_path.symlink_to(target_path.name) + manifest_path.write_text(f"rf-file|0|bin/python|{symlink_path}") + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + + +def test_create_zip_with_sandboxed_source_symlink(tmp_path): + manifest_path = tmp_path / "manifest.txt" + output_zip = tmp_path / "output.zip" + + source_dir = tmp_path / "source" + source_dir.mkdir() + target_path = source_dir / "python3.14" + target_path.write_text("python") + source_symlink = source_dir / "python" + source_symlink.symlink_to(target_path.name) + + sandbox_dir = tmp_path / "sandbox" + sandbox_dir.mkdir() + sandbox_symlink = sandbox_dir / "python" + sandbox_symlink.symlink_to(source_symlink) + sandbox_regular = sandbox_dir / "python3.14" + sandbox_regular.symlink_to(target_path) + manifest_path.write_text( + "\n".join( + [ + f"rf-file|0|bin/python|{sandbox_symlink}", + f"rf-file|0|bin/python3.14|{sandbox_regular}", + ] + ) + ) + + create_zip(manifest_path, output_zip) + + with zipfile.ZipFile(output_zip, "r") as zf: + assert_zip_file_content( + zf, + "runfiles/my_ws/bin/python", + is_symlink_file=True, + target=target_path.name, + ) + assert_zip_file_content(zf, "runfiles/my_ws/bin/python3.14", content="python") + + def test_pathsep_normalization(tmp_path): manifest_path = tmp_path / "manifest.txt" output_zip = tmp_path / "output.zip" diff --git a/tools/zipapp/zipper.py b/tools/zipapp/zipper.py index 5a8eb8c6fd..c0e547284a 100644 --- a/tools/zipapp/zipper.py +++ b/tools/zipapp/zipper.py @@ -118,6 +118,23 @@ def normalize_zip_path(path): return path.replace("\\", "/") +def _source_symlink_target(content_path, is_symlink_str): + if is_symlink_str == "1": + return os.readlink(content_path) + if is_symlink_str != "0" or not os.path.islink(content_path): + return None + + target = os.readlink(content_path) + if not os.path.isabs(target): + return target + + # Bazel sandboxes expose regular inputs as absolute symlinks. Look through + # that indirection to detect whether the original source is also a symlink. + if os.path.islink(target): + return os.readlink(target) + return None + + def _write_entry(zf, entry, compress_type, seen, platform_pathsep): type_, is_symlink_str, zip_path, content_path = entry # Normalize slashes, otherwise the `seen` logic doesn't @@ -155,14 +172,13 @@ def _write_entry(zf, entry, compress_type, seen, platform_pathsep): else: is_symlink_str = "0" - is_symlink = is_symlink_str == "1" - - if is_symlink: + symlink_target = _source_symlink_target(content_path, is_symlink_str) + if symlink_target is not None: zi = zipfile.ZipInfo(zip_path) zi.date_time = (1980, 1, 1, 0, 0, 0) zi.create_system = 3 # Unix zi.compress_type = compress_type - target = convert_symlink_target(os.readlink(content_path), platform_pathsep) + target = convert_symlink_target(symlink_target, platform_pathsep) # Set permissions to 777 for symlink (standard) zi.external_attr = (S_IFLNK | 0o777) << 16 zf.writestr(zi, target)