Skip to content

Commit bfb0788

Browse files
authored
gh-127111: Emscripten Make web example work again (#127113)
Moves the Emscripten web example into a standalone folder, and updates Makefile targets to build the web example. Instructions for usage have also been added.
1 parent edefb86 commit bfb0788

File tree

9 files changed

+175
-92
lines changed

9 files changed

+175
-92
lines changed

Makefile.pre.in

+32-19
Original file line numberDiff line numberDiff line change
@@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@
269269
# Other subdirectories
270270
SUBDIRSTOO= Include Lib Misc
271271

272-
# assets for Emscripten browser builds
273-
WASM_ASSETS_DIR=.$(prefix)
274-
WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py
275-
276272
# Files and directories to be distributed
277273
CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
278274
DISTFILES= README.rst ChangeLog $(CONFIGFILES)
@@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh
737733
build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
738734
python-config checksharedmods
739735

736+
.PHONY: build_emscripten
737+
build_emscripten: build_wasm web_example
738+
740739
# Check that the source is clean when building out of source.
741740
.PHONY: check-clean-src
742741
check-clean-src:
@@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
10161015
else true; \
10171016
fi
10181017

1019-
# wasm32-emscripten browser build
1020-
# wasm assets directory is relative to current build dir, e.g. "./usr/local".
1021-
# --preload-file turns a relative asset path into an absolute path.
1018+
# wasm32-emscripten browser web example
1019+
1020+
WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/
1021+
web_example/python.html: $(WEBEX_DIR)/python.html
1022+
@mkdir -p web_example
1023+
@cp $< $@
1024+
1025+
web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs
1026+
@mkdir -p web_example
1027+
@cp $< $@
10221028

1023-
.PHONY: wasm_stdlib
1024-
wasm_stdlib: $(WASM_STDLIB)
1025-
$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1026-
$(srcdir)/Tools/wasm/wasm_assets.py \
1029+
web_example/server.py: $(WEBEX_DIR)/server.py
1030+
@mkdir -p web_example
1031+
@cp $< $@
1032+
1033+
WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip
1034+
$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
1035+
$(WEBEX_DIR)/wasm_assets.py \
10271036
Makefile pybuilddir.txt Modules/Setup.local
1028-
$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
1029-
--buildroot . --prefix $(prefix)
1037+
$(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \
1038+
--buildroot . --prefix $(prefix) -o $@
10301039

1031-
python.html: $(srcdir)/Tools/wasm/python.html python.worker.js
1032-
@cp $(srcdir)/Tools/wasm/python.html $@
1040+
web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON)
1041+
@if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \
1042+
echo "Can only build web_example when target is Emscripten" ;\
1043+
exit 1 ;\
1044+
fi
1045+
cp python.mjs web_example/python.mjs
1046+
cp python.wasm web_example/python.wasm
10331047

1034-
python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
1035-
@cp $(srcdir)/Tools/wasm/python.worker.js $@
1048+
.PHONY: web_example
1049+
web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB)
10361050

10371051
############################################################################
10381052
# Header files
@@ -3053,8 +3067,7 @@ clean-retain-profile: pycremoval
30533067
find build -name '*.py[co]' -exec rm -f {} ';' || true
30543068
-rm -f pybuilddir.txt
30553069
-rm -f _bootstrap_python
3056-
-rm -f python.html python*.js python.data python*.symbols python*.map
3057-
-rm -f $(WASM_STDLIB)
3070+
-rm -rf web_example python.mjs python.wasm python*.symbols python*.map
30583071
-rm -f Programs/_testembed Programs/_freeze_module
30593072
-rm -rf Python/deepfreeze
30603073
-rm -f Python/frozen_modules/*.h
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Updated the Emscripten web example to use ES6 modules and be built into a
2+
distinct ``web_example`` subfolder.

Tools/wasm/README.md

+86-36
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information.
2323

2424
To cross compile to the ``wasm32-emscripten`` platform you need
2525
[the Emscripten compiler toolchain](https://emscripten.org/),
26-
a Python interpreter, and an installation of Node version 18 or newer. Emscripten
27-
version 3.1.42 or newer is recommended. All commands below are relative to a checkout
28-
of the Python repository.
26+
a Python interpreter, and an installation of Node version 18 or newer.
27+
Emscripten version 3.1.73 or newer is recommended. All commands below are
28+
relative to a checkout of the Python repository.
2929

3030
#### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html)
3131

@@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable.
5050
export EM_COMPILER_WRAPPER=ccache
5151
```
5252

53-
### Compile and build Python interpreter
53+
#### Compile and build Python interpreter
5454

5555
You can use `python Tools/wasm/emscripten` to compile and build targetting
5656
Emscripten. You can do everything at once with:
@@ -70,6 +70,88 @@ instance, to do a debug build, you can use:
7070
python Tools/wasm/emscripten build --with-py-debug
7171
```
7272

73+
### Running from node
74+
75+
If you want to run the normal Python CLI, you can use `python.sh`. It takes the
76+
same options as the normal Python CLI entrypoint, though the REPL does not
77+
function and will crash.
78+
79+
`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the
80+
Python process and starts it up with the appropriate settings. If you wish to
81+
make a node application that "embeds" the interpreter instead of acting like the
82+
CLI you will need to write your own alternative to `node_entry.mjs`.
83+
84+
85+
### The Web Example
86+
87+
When building for Emscripten, the web example will be built automatically. It is
88+
in the ``web_example`` directory. To run the web example, ``cd`` into the
89+
``web_example`` directory, then run ``python server.py``. This will start a web
90+
server; you can then visit ``http://localhost:8000/python.html`` in a browser to
91+
see a simple REPL example.
92+
93+
The web example relies on a bug fix in Emscripten version 3.1.73 so if you build
94+
with earlier versions of Emscripten it may not work. The web example uses
95+
``SharedArrayBuffer``. For security reasons browsers only provide
96+
``SharedArrayBuffer`` in secure environments with cross-origin isolation. The
97+
webserver must send cross-origin headers and correct MIME types for the
98+
JavaScript and WebAssembly files. Otherwise the terminal will fail to load with
99+
an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See
100+
more information here:
101+
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements
102+
103+
Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the
104+
web example. If cross-origin isolation is not appropriate for your use case you
105+
may make your own application embedding `python.mjs` which does not use
106+
``SharedArrayBuffer`` and serve it without the cross-origin isolation headers.
107+
108+
### Embedding Python in a custom JavaScript application
109+
110+
You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a
111+
minimum you must import ``createEmscriptenModule`` and you need to call
112+
``createEmscriptenModule`` with an appropriate settings object. This settings
113+
object will need a prerun hook that installs the Python standard library into
114+
the Emscripten file system.
115+
116+
#### NodeJs
117+
118+
In Node, you can use the NodeFS to mount the standard library in your native
119+
file system into the Emscripten file system:
120+
```js
121+
import createEmscriptenModule from "./python.mjs";
122+
123+
await createEmscriptenModule({
124+
preRun(Module) {
125+
Module.FS.mount(
126+
Module.FS.filesystems.NODEFS,
127+
{ root: "/path/to/python/stdlib" },
128+
"/lib/",
129+
);
130+
},
131+
});
132+
```
133+
134+
#### Browser
135+
136+
In the browser, the simplest approach is to put the standard library in a zip
137+
file it and install it. With Python 3.14 this could look like:
138+
```js
139+
import createEmscriptenModule from "./python.mjs";
140+
141+
await createEmscriptenModule({
142+
async preRun(Module) {
143+
Module.FS.mkdirTree("/lib/python3.14/lib-dynload/");
144+
Module.addRunDependency("install-stdlib");
145+
const resp = await fetch("python3.14.zip");
146+
const stdlibBuffer = await resp.arrayBuffer();
147+
Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), {
148+
canOwn: true,
149+
});
150+
Module.removeRunDependency("install-stdlib");
151+
},
152+
});
153+
```
154+
73155
### Limitations and issues
74156

75157
#### Network stack
@@ -151,38 +233,6 @@ python Tools/wasm/emscripten build --with-py-debug
151233
- Test modules are disabled by default. Use ``--enable-test-modules`` build
152234
test modules like ``_testcapi``.
153235

154-
### wasm32-emscripten in node
155-
156-
Node builds use ``NODERAWFS``.
157-
158-
- Node RawFS allows direct access to the host file system without need to
159-
perform ``FS.mount()`` call.
160-
161-
### Hosting Python WASM builds
162-
163-
The simple REPL terminal uses SharedArrayBuffer. For security reasons
164-
browsers only provide the feature in secure environments with cross-origin
165-
isolation. The webserver must send cross-origin headers and correct MIME types
166-
for the JavaScript and WASM files. Otherwise the terminal will fail to load
167-
with an error message like ``Browsers disable shared array buffer``.
168-
169-
#### Apache HTTP .htaccess
170-
171-
Place a ``.htaccess`` file in the same directory as ``python.wasm``.
172-
173-
```
174-
# .htaccess
175-
Header set Cross-Origin-Opener-Policy same-origin
176-
Header set Cross-Origin-Embedder-Policy require-corp
177-
178-
AddType application/javascript js
179-
AddType application/wasm wasm
180-
181-
<IfModule mod_deflate.c>
182-
AddOutputFilterByType DEFLATE text/html application/javascript application/wasm
183-
</IfModule>
184-
```
185-
186236
## WASI (wasm32-wasi)
187237

188238
See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi).

Tools/wasm/python.html renamed to Tools/wasm/emscripten/web_example/python.html

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747

4848
async initialiseWorker() {
4949
if (!this.worker) {
50-
this.worker = new Worker(this.workerURL)
50+
this.worker = new Worker(this.workerURL, {type: "module"})
5151
this.worker.addEventListener('message', this.handleMessageFromWorker)
5252
}
5353
}
@@ -347,7 +347,7 @@
347347
programRunning(false)
348348
}
349349

350-
const pythonWorkerManager = new WorkerManager('./python.worker.js', stdio, readyCallback, finishedCallback)
350+
const pythonWorkerManager = new WorkerManager('./python.worker.mjs', stdio, readyCallback, finishedCallback)
351351
}
352352
</script>
353353
</head>

Tools/wasm/python.worker.js renamed to Tools/wasm/emscripten/web_example/python.worker.mjs

+21-4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import createEmscriptenModule from "./python.mjs";
2+
13
class StdinBuffer {
24
constructor() {
35
this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT)
@@ -59,29 +61,44 @@ const stderr = (charCode) => {
5961

6062
const stdinBuffer = new StdinBuffer()
6163

62-
var Module = {
64+
const emscriptenSettings = {
6365
noInitialRun: true,
6466
stdin: stdinBuffer.stdin,
6567
stdout: stdout,
6668
stderr: stderr,
6769
onRuntimeInitialized: () => {
6870
postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab})
71+
},
72+
async preRun(Module) {
73+
const versionHex = Module.HEAPU32[Module._Py_Version/4].toString(16);
74+
const versionTuple = versionHex.padStart(8, "0").match(/.{1,2}/g).map((x) => parseInt(x, 16));
75+
const [major, minor, ..._] = versionTuple;
76+
// Prevent complaints about not finding exec-prefix by making a lib-dynload directory
77+
Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`);
78+
Module.addRunDependency("install-stdlib");
79+
const resp = await fetch(`python${major}.${minor}.zip`);
80+
const stdlibBuffer = await resp.arrayBuffer();
81+
Module.FS.writeFile(`/lib/python${major}${minor}.zip`, new Uint8Array(stdlibBuffer), { canOwn: true });
82+
Module.removeRunDependency("install-stdlib");
6983
}
7084
}
7185

72-
onmessage = (event) => {
86+
const modulePromise = createEmscriptenModule(emscriptenSettings);
87+
88+
89+
onmessage = async (event) => {
7390
if (event.data.type === 'run') {
91+
const Module = await modulePromise;
7492
if (event.data.files) {
7593
for (const [filename, contents] of Object.entries(event.data.files)) {
7694
Module.FS.writeFile(filename, contents)
7795
}
7896
}
79-
const ret = callMain(event.data.args)
97+
const ret = Module.callMain(event.data.args);
8098
postMessage({
8199
type: 'finished',
82100
returnCode: ret
83101
})
84102
}
85103
}
86104

87-
importScripts('python.js')

Tools/wasm/wasm_webserver.py renamed to Tools/wasm/emscripten/web_example/server.py

+1-7
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,6 @@
1414

1515

1616
class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler):
17-
extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy()
18-
extensions_map.update(
19-
{
20-
".wasm": "application/wasm",
21-
}
22-
)
23-
2417
def end_headers(self) -> None:
2518
self.send_my_headers()
2619
super().end_headers()
@@ -42,5 +35,6 @@ def main() -> None:
4235
bind=args.bind,
4336
)
4437

38+
4539
if __name__ == "__main__":
4640
main()

0 commit comments

Comments
 (0)