diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index c8832342d9892da..74b25ad1784ac94 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -79,3 +79,5 @@ jobs: run: python3 Platforms/emscripten run --test - name: "Test Repl" run: Platforms/emscripten/browser_test/run_test.sh + - name: "Test embedding" + run: Platforms/emscripten/web_embed_test/run_test.sh diff --git a/Makefile.pre.in b/Makefile.pre.in index 78a486623181fa8..d53de5dbf61f328 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -970,9 +970,12 @@ clinic: check-clean-src clinic-tests: check-clean-src $(srcdir)/Lib/test/clinic.test.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $(srcdir)/Lib/test/clinic.test.c +# Objects linked into the interpreter only, not into libpython. +PLATFORM_MAIN_OBJS= @PLATFORM_MAIN_OBJS@ + # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -1119,6 +1122,20 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.mjs web_example/python.mjs cp python.wasm web_example/python.wasm +WEB_EMBED_TEST_DIR=$(EMSCRIPTEN_DIR)/web_embed_test + +# Linked like an embedder would: against libpython.a, without $(LINKFORSHARED) +# and so without -sMAIN_MODULE or the interpreter's -sEXPORTED_FUNCTIONS. +web_embed_test/main.js: $(WEB_EMBED_TEST_DIR)/main.c $(LIBRARY) $(ZIP_STDLIB) + @mkdir -p web_embed_test + $(LINKCC) $(PY_STDMODULE_CFLAGS) $(PY_CORE_EXE_LDFLAGS) -O2 -g0 -o $@ \ + $(WEB_EMBED_TEST_DIR)/main.c $(LIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \ + -sALLOW_MEMORY_GROWTH -sSTACK_SIZE=5MB \ + --preload-file $(ZIP_STDLIB)@/lib/stdlib.zip + +.PHONY: web_embed_test +web_embed_test: web_embed_test/main.js + .PHONY: web_example web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) @@ -1614,8 +1631,8 @@ regen-re: $(BUILDPYTHON) # using Tools/build/generate_re_casefix.py $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py -Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +Programs/_testembed: Programs/_testembed.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_DEPS) + $(LINKCC) $(PY_CORE_EXE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(PLATFORM_MAIN_OBJS) $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) ############################################################################ # "Bootstrap Python" used to run Programs/_freeze_module.py @@ -2235,7 +2252,7 @@ regen-slots: Python/slots.toml $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \ --generate-all -$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) +$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS) ###################################################################### @@ -2961,6 +2978,9 @@ libainstall: all scripts fi; \ fi; \ $(INSTALL_DATA) Programs/python.o $(DESTDIR)$(LIBPL)/python.o; \ + for i in $(PLATFORM_MAIN_OBJS); do \ + $(INSTALL_DATA) $$i $(DESTDIR)$(LIBPL); \ + done; \ fi $(INSTALL_DATA) Modules/config.c $(DESTDIR)$(LIBPL)/config.c $(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in @@ -3282,7 +3302,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -rf web_example python.mjs python.wasm python*.symbols python*.map + -rm -rf web_example web_embed_test python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst b/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst new file mode 100644 index 000000000000000..fe664df69097405 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-09-01-23-32-22.gh-issue-156780.Qw3nRt.rst @@ -0,0 +1 @@ +Emscripten ``libpython.a`` can now be linked into a program statically. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae70420985..2ebaccedb715e15 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2961,7 +2961,7 @@ EM_JS(int, emscripten_set_up_async_input_device_js, (void), { await sleep(5); return bufs[(idx ++) % 3]; }); - return !!WebAssembly.promising; + return !!Module.Py_EmscriptenStackSwitching; }); static PyObject * diff --git a/Platforms/emscripten/web_embed_test/main.c b/Platforms/emscripten/web_embed_test/main.c new file mode 100644 index 000000000000000..462024d86cb75df --- /dev/null +++ b/Platforms/emscripten/web_embed_test/main.c @@ -0,0 +1,50 @@ +/* Smoke test for libpython linked into a program whose main() is not Python's. */ + +#include +#include + +// Imports come from the preloaded zip, len() goes through the call trampoline +// and poll() through the syscall overrides. +static const char *SCRIPT = + "import json, select\n" + "select.poll().poll(0)\n" + "print(json.dumps({'embedded': len('ok')}))\n"; + +int main(void) +{ + PyStatus status; + PyConfig config; + + PyConfig_InitIsolatedConfig(&config); + config.write_bytecode = 0; + config.module_search_paths_set = 1; + status = PyWideStringList_Append(&config.module_search_paths, + L"/lib/stdlib.zip"); + if (PyStatus_Exception(status)) { + goto exception; + } + status = PyConfig_SetBytesString(&config, &config.executable, "/embed"); + if (PyStatus_Exception(status)) { + goto exception; + } + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + + if (PyRun_SimpleString(SCRIPT) != 0) { + puts("web_embed_test: script failed"); + return 1; + } + if (Py_FinalizeEx() < 0) { + puts("web_embed_test: Py_FinalizeEx failed"); + return 1; + } + puts("web_embed_test: ok"); + return 0; + +exception: + PyConfig_Clear(&config); + Py_ExitStatusException(status); +} diff --git a/Platforms/emscripten/web_embed_test/run_test.sh b/Platforms/emscripten/web_embed_test/run_test.sh new file mode 100755 index 000000000000000..ecbbe73efd8b4f5 --- /dev/null +++ b/Platforms/emscripten/web_embed_test/run_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Build and run the libpython embedding smoke test. +set -euo pipefail +cd "$(dirname "$0")/../../.." + +BUILD_DIR=${CROSS_BUILD_DIR:-cross-build}/wasm32-emscripten/build/python +make -C "$BUILD_DIR" web_embed_test + +# Node 24 needs JSPI enabled explicitly; it is on by default afterwards. +NODE=${NODE:-node} +NODE_FLAGS= +if [ "$("$NODE" -p 'process.versions.node.split(".")[0]')" = 24 ]; then + NODE_FLAGS=--experimental-wasm-jspi +fi + +# Run from the build directory: Emscripten resolves main.data relative to cwd. +cd "$BUILD_DIR/web_embed_test" +rc=0 +out=$("$NODE" $NODE_FLAGS main.js 2>&1) || rc=$? +echo "$out" +[ "$rc" -eq 0 ] && grep -q "web_embed_test: ok" <<<"$out" diff --git a/Programs/emscripten_beforemain.c b/Programs/emscripten_beforemain.c new file mode 100644 index 000000000000000..54b0b0ff4be535d --- /dev/null +++ b/Programs/emscripten_beforemain.c @@ -0,0 +1,41 @@ +/* Emscripten setup for when Python is the program. Linked into the + * interpreter only, never into libpython. + * + * Runs main() under WebAssembly.promising so that libpython can suspend the + * wasm stack in a syscall (see Python/emscripten_syscalls.c). + */ + +#include + +EM_JS(void, _PyEmscripten_BeforeMain_js, (void), { + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + if (ENVIRONMENT_IS_NODE && !Module.onExit) { + Module.onExit = (code) => process.exit(code); + } + // promising() needs the raw export; _main may be a JS wrapper around it. + const main = WebAssembly.promising(wasmExports.__main_argc_argv); + _main = (...args) => { + Module.Py_EmscriptenStackSwitching = true; + // Exit the way callMain() would have, once main() is actually done. + main(...args).then((ret) => exitJS(ret, true)).catch(handleException); + // Unwind to callMain() without letting it exit: main() is still + // running on the promising stack. + throw "unwind"; + }; + // callMain() takes the entry point from _main, or from wasmImports.main + // when linked with -sMAIN_MODULE. + if ("main" in wasmImports) { + wasmImports.main = _main; + } +}) + +EM_JS_DEPS(_PyEmscripten_BeforeMain, "$exitJS,$handleException"); + +__attribute__((constructor)) void +_PyEmscripten_BeforeMain(void) +{ + _PyEmscripten_BeforeMain_js(); +} diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index cdd5fc6e91e89bc..ae3096276d07fce 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -54,13 +54,13 @@ __attribute__((constructor)) void __syscall_init_umask(void) { #define EM_JS_MACROS(ret, func_name, args, body...) \ EM_JS(ret, func_name, args, body) -EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { - // Define FS.createAsyncInputDevice(), This is quite similar to - // FS.createDevice() defined here: - // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 - // but instead of returning one byte at a time, the input() function should - // return a Uint8Array. This makes the handler code simpler, the - // `createAsyncInputDevice` simpler, and everything faster. +// Define FS.createAsyncInputDevice(), This is quite similar to +// FS.createDevice() defined here: +// https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 +// but instead of returning one byte at a time, the input() function should +// return a Uint8Array. This makes the handler code simpler, the +// `createAsyncInputDevice` simpler, and everything faster. +EM_JS_MACROS(void, __fs_init_async_input_device_js, (void), { FS.createAsyncInputDevice = function(parent, name, input) { parent = typeof parent == 'string' ? parent : FS.getPath(parent); var path = PATH.join2(parent, name); @@ -103,44 +103,14 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { FS.registerDevice(dev, ops); return FS.mkdev(path, mode, dev); }; - if (!WebAssembly.promising) { - // No stack switching support =( - return; - } - const origResolveGlobalSymbol = resolveGlobalSymbol; - if (ENVIRONMENT_IS_NODE && !Module.onExit) { - Module.onExit = (code) => process.exit(code); - } - // * wrap the main symbol with WebAssembly.promising, - // * call exit_with_live_runtime() to prevent emscripten from shutting down - // the runtime before the promise resolves, - // * call onExit / process.exit ourselves, since exit_with_live_runtime() - // prevented Emscripten from calling it normally. - resolveGlobalSymbol = function (name, direct = false) { - const orig = origResolveGlobalSymbol(name, direct); - if (name === "main") { - const main = WebAssembly.promising(orig.sym); - orig.sym = (...args) => { - (async () => { - const ret = await main(...args); - Module.onExit?.(ret); - })(); - _emscripten_exit_with_live_runtime(); - }; - } - return orig; - }; }) -EM_JS_DEPS(_emscripten_promising_main, - "$FS,$PATH,$FS_getMode,$resolveGlobalSymbol," - "emscripten_exit_with_live_runtime"); +EM_JS_DEPS(__fs_init_async_input_device, "$FS,$PATH,$FS_getMode"); -__attribute__((constructor)) void _emscripten_promising_main(void) { - _emscripten_promising_main_js(); +__attribute__((constructor)) void __fs_init_async_input_device(void) { + __fs_init_async_input_device_js(); } - #define IOVEC_T_BUF_OFFSET 0 #define IOVEC_T_BUF_LEN_OFFSET 4 #define IOVEC_T_SIZE 8 @@ -154,13 +124,20 @@ _Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, // If the stream has a readAsync handler, read to buffer defined in iovs, write // number of bytes read to *nread, and return a promise that resolves to the // errno. Otherwise, return null. +// +// Reading from an async input device and poll() suspend the wasm stack +// instead of blocking when main() runs under WebAssembly.promising, which +// Programs/emscripten_beforemain.c arranges for the interpreter. An embedder +// running its own promising entry point opts in with +// Module.Py_EmscriptenStackSwitching = true; +// Otherwise these calls keep their synchronous behavior. EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( __wasi_fd_t fd, const __wasi_iovec_t *iovs, size_t iovcnt, __wasi_size_t *nread ), { - if (!WebAssembly.promising) { + if (!Module.Py_EmscriptenStackSwitching) { return null; } var stream; @@ -244,7 +221,7 @@ _Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct _Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { - if (!WebAssembly.promising) { + if (!Module.Py_EmscriptenStackSwitching) { return null; } return (async function() { diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index 75cfde6b76f2174..547761027a3fb3d 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -47,6 +47,19 @@ typedef PyObject* (*TrampolineFunc)(int* success, PyObject* args, PyObject* kw); +// Lets JS reach _PyRuntime without it being in -sEXPORTED_FUNCTIONS. +EMSCRIPTEN_KEEPALIVE _PyRuntimeState *const _PyEM_runtime = &_PyRuntime; + +// Its table slot is taken over by the wasm-gc trampoline, so the table +// never grows. +static PyObject* +trampoline_placeholder(int* success, PyCFunctionWithKeywords func, + PyObject* self, PyObject* args, PyObject* kw) +{ + Py_FatalError("Emscripten trampoline slot was not set up"); +} +EMSCRIPTEN_KEEPALIVE const TrampolineFunc _PyEM_trampoline_slot = trampoline_placeholder; + /** * Backwards compatible trampoline works with all JS runtimes */ @@ -79,7 +92,9 @@ function getPyEMTrampolinePtr() { const trampolineInstance = new WebAssembly.Instance(trampolineModule, { env: { __indirect_function_table: wasmTable, memory: wasmMemory }, }); - return addFunction(trampolineInstance.exports.trampoline_call); + const slot = HEAPU32[__PyEM_trampoline_slot / 4]; + wasmTable.set(slot, trampolineInstance.exports.trampoline_call); + return slot; } // We have to be careful to work correctly with memory snapshots -- the value of // _PyRuntimeState.emscripten_trampoline needs to reflect whether wasm-gc is @@ -90,12 +105,12 @@ function getPyEMTrampolinePtr() { addOnPreRun(function setEmscriptenTrampoline() { const ptr = getPyEMTrampolinePtr(); const offset = HEAP32[__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET / 4]; - HEAP32[(__PyRuntime + offset) / 4] = ptr; + HEAP32[(HEAPU32[__PyEM_runtime / 4] + offset) / 4] = ptr; }); ); EM_JS_DEPS(_PyEM_TrampolineCall, - "$wasmTable,$wasmMemory,$addFunction,$addOnPreRun"); + "$wasmTable,$wasmMemory,$addOnPreRun"); PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, diff --git a/configure b/configure index ae3f0450dc7af1b..219eff4bf021f24 100755 --- a/configure +++ b/configure @@ -879,6 +879,7 @@ TRUE MACHDEP_OBJS DYNLOADFILE DLINCLDIR +PLATFORM_MAIN_OBJS PLATFORM_OBJS PLATFORM_HEADERS DTRACE_OBJS @@ -9978,7 +9979,7 @@ fi as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" @@ -20275,11 +20276,13 @@ fi PLATFORM_HEADERS= PLATFORM_OBJS= +PLATFORM_MAIN_OBJS= case $ac_sys_system in #( Emscripten) : as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o' + as_fn_append PLATFORM_MAIN_OBJS ' Programs/emscripten_beforemain.o' if test "x$enable_emscripten_syscalls" = xyes then : @@ -20294,6 +20297,7 @@ esac + # -I${DLINCLDIR} is added to the compile rule for importdl.o DLINCLDIR=. diff --git a/configure.ac b/configure.ac index 1c42900cb975c35..d574f6d022c1658 100644 --- a/configure.ac +++ b/configure.ac @@ -2440,7 +2440,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) @@ -5427,10 +5427,12 @@ fi dnl Platform-specific C and header files. PLATFORM_HEADERS= PLATFORM_OBJS= +PLATFORM_MAIN_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o']) + AS_VAR_APPEND([PLATFORM_MAIN_OBJS], [' Programs/emscripten_beforemain.o']) AS_VAR_IF([enable_emscripten_syscalls], [yes], [ AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_syscalls.o']) ]) @@ -5439,6 +5441,7 @@ AS_CASE([$ac_sys_system], ) AC_SUBST([PLATFORM_HEADERS]) AC_SUBST([PLATFORM_OBJS]) +AC_SUBST([PLATFORM_MAIN_OBJS]) # -I${DLINCLDIR} is added to the compile rule for importdl.o AC_SUBST([DLINCLDIR])