Along the way, leave a deprecated Tools/wasm/wasi/__main__.py behind for backwards-compatibility.
Co-authored-by: Zachary Ware <zachary.ware@gmail.com>
Tools/wasm/emscripten @freakboy3742 @emmatyping
# WebAssembly (WASI)
+Platforms/WASI @brettcannon @emmatyping @savannahostrowski
Tools/wasm/wasi-env @brettcannon @emmatyping @savannahostrowski
Tools/wasm/wasi.py @brettcannon @emmatyping @savannahostrowski
Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski
- name: "Runner image version"
run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
- name: "Configure build Python"
- run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug
+ run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug
- name: "Make build Python"
- run: python3 Tools/wasm/wasi make-build-python
+ run: python3 Platforms/WASI make-build-python
- name: "Configure host"
# `--with-pydebug` inferred from configure-build-python
- run: python3 Tools/wasm/wasi configure-host -- --config-cache
+ run: python3 Platforms/WASI configure-host -- --config-cache
- name: "Make host"
- run: python3 Tools/wasm/wasi make-host
+ run: python3 Platforms/WASI make-host
- name: "Display build info"
run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
- name: "Test"
--- /dev/null
+Move WASI-related files to :file:`Platforms/WASI`. Along the way, leave a
+deprecated :file:`Tools/wasm/wasi/__main__.py` behind for
+backwards-compatibility.
--- /dev/null
+extend = "../../.ruff.toml" # Inherit the project-wide settings
+
+[format]
+preview = true
+docstring-code-format = true
+
+[lint]
+select = [
+ "C4", # flake8-comprehensions
+ "E", # pycodestyle
+ "F", # pyflakes
+ "I", # isort
+ "ISC", # flake8-implicit-str-concat
+ "LOG", # flake8-logging
+ "PGH", # pygrep-hooks
+ "PT", # flake8-pytest-style
+ "PYI", # flake8-pyi
+ "RUF100", # Ban unused `# noqa` comments
+ "UP", # pyupgrade
+ "W", # pycodestyle
+ "YTT", # flake8-2020
+]
+ignore = [
+ "E501", # Line too long
+]
--- /dev/null
+# Python WASI (wasm32-wasi) build
+
+**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).**
+
+This directory contains configuration and helpers to facilitate cross
+compilation of CPython to WebAssembly (WASM) using WASI. WASI builds
+use WASM runtimes such as [wasmtime](https://wasmtime.dev/).
+
+**NOTE**: If you are looking for general information about WebAssembly that is
+not directly related to CPython, please see https://github.com/psf/webassembly.
+
+## Build
+
+See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi).
+
+## Detecting WASI builds
+
+### Python code
+
+```python
+import os, sys
+
+if sys.platform == "wasi":
+ # Python on WASI
+ ...
+
+if os.name == "posix":
+ # WASM platforms identify as POSIX-like.
+ # Windows does not provide os.uname().
+ machine = os.uname().machine
+ if machine.startswith("wasm"):
+ # WebAssembly (wasm32, wasm64 potentially in the future)
+```
+
+```python
+>>> import os, sys
+>>> os.uname()
+posix.uname_result(
+ sysname='wasi',
+ nodename='(none)',
+ release='0.0.0',
+ version='0.0.0',
+ machine='wasm32'
+)
+>>> os.name
+'posix'
+>>> sys.platform
+'wasi'
+```
+
+### C code
+
+WASI SDK defines several built-in macros. You can dump a full list of built-ins
+with ``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``.
+
+* WebAssembly ``__wasm__`` (also ``__wasm``)
+* wasm32 ``__wasm32__`` (also ``__wasm32``)
+* wasm64 ``__wasm64__``
+* WASI ``__wasi__``
--- /dev/null
+#!/usr/bin/env python3
+
+import argparse
+import contextlib
+import functools
+import os
+
+import tomllib
+
+try:
+ from os import process_cpu_count as cpu_count
+except ImportError:
+ from os import cpu_count
+import pathlib
+import shutil
+import subprocess
+import sys
+import sysconfig
+import tempfile
+
+CHECKOUT = HERE = pathlib.Path(__file__).parent
+
+while CHECKOUT != CHECKOUT.parent:
+ if (CHECKOUT / "configure").is_file():
+ break
+ CHECKOUT = CHECKOUT.parent
+else:
+ raise FileNotFoundError(
+ "Unable to find the root of the CPython checkout by looking for 'configure'"
+ )
+
+CROSS_BUILD_DIR = CHECKOUT / "cross-build"
+# Build platform can also be found via `config.guess`.
+BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
+
+LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
+LOCAL_SETUP_MARKER = (
+ b"# Generated by Platforms/WASI .\n"
+ b"# Required to statically build extension modules."
+)
+
+WASI_SDK_VERSION = 29
+
+WASMTIME_VAR_NAME = "WASMTIME"
+WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
+
+
+def separator():
+ """Print a separator line across the terminal width."""
+ try:
+ tput_output = subprocess.check_output(
+ ["tput", "cols"], encoding="utf-8"
+ )
+ except subprocess.CalledProcessError:
+ terminal_width = 80
+ else:
+ terminal_width = int(tput_output.strip())
+ print("โฏ" * terminal_width)
+
+
+def log(emoji, message, *, spacing=None):
+ """Print a notification with an emoji.
+
+ If 'spacing' is None, calculate the spacing based on the number of code points
+ in the emoji as terminals "eat" a space when the emoji has multiple code points.
+ """
+ if spacing is None:
+ spacing = " " if len(emoji) == 1 else " "
+ print("".join([emoji, spacing, message]))
+
+
+def updated_env(updates={}):
+ """Create a new dict representing the environment to use.
+
+ The changes made to the execution environment are printed out.
+ """
+ env_defaults = {}
+ # https://reproducible-builds.org/docs/source-date-epoch/
+ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
+ try:
+ epoch = subprocess.check_output(
+ git_epoch_cmd, encoding="utf-8"
+ ).strip()
+ env_defaults["SOURCE_DATE_EPOCH"] = epoch
+ except subprocess.CalledProcessError:
+ pass # Might be building from a tarball.
+ # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
+ environment = env_defaults | os.environ | updates
+
+ env_diff = {}
+ for key, value in environment.items():
+ if os.environ.get(key) != value:
+ env_diff[key] = value
+
+ env_vars = (
+ f"\n {key}={item}" for key, item in sorted(env_diff.items())
+ )
+ log("๐", f"Environment changes:{''.join(env_vars)}")
+
+ return environment
+
+
+def subdir(working_dir, *, clean_ok=False):
+ """Decorator to change to a working directory."""
+
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(context):
+ nonlocal working_dir
+
+ if callable(working_dir):
+ working_dir = working_dir(context)
+ separator()
+ log("๐", os.fsdecode(working_dir))
+ if (
+ clean_ok
+ and getattr(context, "clean", False)
+ and working_dir.exists()
+ ):
+ log("๐ฎ", "Deleting directory (--clean)...")
+ shutil.rmtree(working_dir)
+
+ working_dir.mkdir(parents=True, exist_ok=True)
+
+ with contextlib.chdir(working_dir):
+ return func(context, working_dir)
+
+ return wrapper
+
+ return decorator
+
+
+def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
+ """Execute a command.
+
+ If 'quiet' is true, then redirect stdout and stderr to a temporary file.
+ """
+ if context is not None:
+ quiet = context.quiet
+ logdir = context.logdir
+ elif quiet and logdir is None:
+ raise ValueError("When quiet is True, logdir must be specified")
+
+ log("โฏ", " ".join(map(str, command)), spacing=" ")
+ if not quiet:
+ stdout = None
+ stderr = None
+ else:
+ stdout = tempfile.NamedTemporaryFile(
+ "w",
+ encoding="utf-8",
+ delete=False,
+ dir=logdir,
+ prefix="cpython-wasi-",
+ suffix=".log",
+ )
+ stderr = subprocess.STDOUT
+ log("๐", f"Logging output to {stdout.name} (--quiet)...")
+
+ subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
+
+
+def build_python_path():
+ """The path to the build Python binary."""
+ binary = BUILD_DIR / "python"
+ if not binary.is_file():
+ binary = binary.with_suffix(".exe")
+ if not binary.is_file():
+ raise FileNotFoundError(
+ f"Unable to find `python(.exe)` in {BUILD_DIR}"
+ )
+
+ return binary
+
+
+def build_python_is_pydebug():
+ """Find out if the build Python is a pydebug build."""
+ test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
+ result = subprocess.run(
+ [build_python_path(), "-c", test],
+ capture_output=True,
+ )
+ return bool(result.returncode)
+
+
+@subdir(BUILD_DIR, clean_ok=True)
+def configure_build_python(context, working_dir):
+ """Configure the build/host Python."""
+ if LOCAL_SETUP.exists():
+ if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
+ log("๐", f"{LOCAL_SETUP} exists ...")
+ else:
+ log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents")
+ else:
+ log("๐", f"Creating {LOCAL_SETUP} ...")
+ LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
+
+ configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
+ if context.args:
+ configure.extend(context.args)
+
+ call(configure, context=context)
+
+
+@subdir(BUILD_DIR)
+def make_build_python(context, working_dir):
+ """Make/build the build Python."""
+ call(["make", "--jobs", str(cpu_count()), "all"], context=context)
+
+ binary = build_python_path()
+ cmd = [
+ binary,
+ "-c",
+ "import sys; "
+ "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+ ]
+ version = subprocess.check_output(cmd, encoding="utf-8").strip()
+
+ log("๐", f"{binary} {version}")
+
+
+def find_wasi_sdk(config):
+ """Find the path to the WASI SDK."""
+ wasi_sdk_path = None
+ wasi_sdk_version = config["targets"]["wasi-sdk"]
+
+ if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
+ wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
+ else:
+ opt_path = pathlib.Path("/opt")
+ # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
+ # has said they don't plan to ever do a point release and all of their Git tags
+ # lack the ``.0`` suffix.
+ # Starting with WASI SDK 23, the tarballs went from containing a directory named
+ # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
+ # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
+ potential_sdks = [
+ path
+ for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*")
+ if path.is_dir()
+ ]
+ if len(potential_sdks) == 1:
+ wasi_sdk_path = potential_sdks[0]
+ elif (default_path := opt_path / "wasi-sdk").is_dir():
+ wasi_sdk_path = default_path
+
+ # Starting with WASI SDK 25, a VERSION file is included in the root
+ # of the SDK directory that we can read to warn folks when they are using
+ # an unsupported version.
+ if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
+ version_details = version_file.read_text(encoding="utf-8")
+ found_version = version_details.splitlines()[0]
+ # Make sure there's a trailing dot to avoid false positives if somehow the
+ # supported version is a prefix of the found version (e.g. `25` and `2567`).
+ if not found_version.startswith(f"{wasi_sdk_version}."):
+ major_version = found_version.partition(".")[0]
+ log(
+ "โ ๏ธ",
+ f" Found WASI SDK {major_version}, "
+ f"but WASI SDK {wasi_sdk_version} is the supported version",
+ )
+
+ return wasi_sdk_path
+
+
+def wasi_sdk_env(context):
+ """Calculate environment variables for building with wasi-sdk."""
+ wasi_sdk_path = context.wasi_sdk_path
+ sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
+ env = {
+ "CC": "clang",
+ "CPP": "clang-cpp",
+ "CXX": "clang++",
+ "AR": "llvm-ar",
+ "RANLIB": "ranlib",
+ }
+
+ for env_var, binary_name in list(env.items()):
+ env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
+
+ if not wasi_sdk_path.name.startswith("wasi-sdk"):
+ for compiler in ["CC", "CPP", "CXX"]:
+ env[compiler] += f" --sysroot={sysroot}"
+
+ env["PKG_CONFIG_PATH"] = ""
+ env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
+ map(
+ os.fsdecode,
+ [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
+ )
+ )
+ env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
+
+ env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
+ env["WASI_SYSROOT"] = os.fsdecode(sysroot)
+
+ env["PATH"] = os.pathsep.join([
+ os.fsdecode(wasi_sdk_path / "bin"),
+ os.environ["PATH"],
+ ])
+
+ return env
+
+
+@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
+def configure_wasi_python(context, working_dir):
+ """Configure the WASI/host build."""
+ if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
+ raise ValueError(
+ "WASI-SDK not found; "
+ "download from "
+ "https://github.com/WebAssembly/wasi-sdk and/or "
+ "specify via $WASI_SDK_PATH or --wasi-sdk"
+ )
+
+ config_site = os.fsdecode(HERE / "config.site-wasm32-wasi")
+
+ wasi_build_dir = working_dir.relative_to(CHECKOUT)
+
+ python_build_dir = BUILD_DIR / "build"
+ lib_dirs = list(python_build_dir.glob("lib.*"))
+ assert len(lib_dirs) == 1, (
+ f"Expected a single lib.* directory in {python_build_dir}"
+ )
+ lib_dir = os.fsdecode(lib_dirs[0])
+ python_version = lib_dir.rpartition("-")[-1]
+ sysconfig_data_dir = (
+ f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
+ )
+
+ # Use PYTHONPATH to include sysconfig data which must be anchored to the
+ # WASI guest's `/` directory.
+ args = {
+ "PYTHONPATH": f"/{sysconfig_data_dir}",
+ "PYTHON_WASM": working_dir / "python.wasm",
+ }
+ # Check dynamically for wasmtime in case it was specified manually via
+ # `--host-runner`.
+ if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
+ if wasmtime := shutil.which("wasmtime"):
+ args[WASMTIME_VAR_NAME] = wasmtime
+ else:
+ raise FileNotFoundError(
+ "wasmtime not found; download from "
+ "https://github.com/bytecodealliance/wasmtime"
+ )
+ host_runner = context.host_runner.format_map(args)
+ env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
+ build_python = os.fsdecode(build_python_path())
+ # The path to `configure` MUST be relative, else `python.wasm` is unable
+ # to find the stdlib due to Python not recognizing that it's being
+ # executed from within a checkout.
+ configure = [
+ os.path.relpath(CHECKOUT / "configure", working_dir),
+ f"--host={context.host_triple}",
+ f"--build={BUILD_DIR.name}",
+ f"--with-build-python={build_python}",
+ ]
+ if build_python_is_pydebug():
+ configure.append("--with-pydebug")
+ if context.args:
+ configure.extend(context.args)
+ call(
+ configure,
+ env=updated_env(env_additions | wasi_sdk_env(context)),
+ context=context,
+ )
+
+ python_wasm = working_dir / "python.wasm"
+ exec_script = working_dir / "python.sh"
+ with exec_script.open("w", encoding="utf-8") as file:
+ file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
+ exec_script.chmod(0o755)
+ log("๐", f"Created {exec_script} (--host-runner)... ")
+ sys.stdout.flush()
+
+
+@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
+def make_wasi_python(context, working_dir):
+ """Run `make` for the WASI/host build."""
+ call(
+ ["make", "--jobs", str(cpu_count()), "all"],
+ env=updated_env(),
+ context=context,
+ )
+
+ exec_script = working_dir / "python.sh"
+ call([exec_script, "--version"], quiet=False)
+ log(
+ "๐",
+ f"Use `{exec_script.relative_to(context.init_dir)}` "
+ "to run CPython w/ the WASI host specified by --host-runner",
+ )
+
+
+def clean_contents(context):
+ """Delete all files created by this script."""
+ if CROSS_BUILD_DIR.exists():
+ log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...")
+ shutil.rmtree(CROSS_BUILD_DIR)
+
+ if LOCAL_SETUP.exists():
+ if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
+ log("๐งน", f"Deleting generated {LOCAL_SETUP} ...")
+
+
+def build_steps(*steps):
+ """Construct a command from other steps."""
+
+ def builder(context):
+ for step in steps:
+ step(context)
+
+ return builder
+
+
+def main():
+ with (HERE / "config.toml").open("rb") as file:
+ config = tomllib.load(file)
+ default_wasi_sdk = find_wasi_sdk(config)
+ default_host_triple = config["targets"]["host-triple"]
+ default_host_runner = (
+ f"{WASMTIME_HOST_RUNNER_VAR} run "
+ # For setting PYTHONPATH to the sysconfig data directory.
+ "--env PYTHONPATH={PYTHONPATH} "
+ # Map the checkout to / to load the stdlib from /Lib.
+ f"--dir {os.fsdecode(CHECKOUT)}::/ "
+ # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept
+ # in a config file.
+ # We are using such a file to act as defaults in case a user wants to override
+ # only some of the settings themselves, make it easy to modify settings
+ # post-build so that they immediately apply to the Makefile instead of having to
+ # regenerate it, and allow for easy copying of the settings for anyone else who
+ # may want to use them.
+ f"--config {os.fsdecode(HERE / 'wasmtime.toml')}"
+ )
+ default_logdir = pathlib.Path(tempfile.gettempdir())
+
+ parser = argparse.ArgumentParser()
+ subcommands = parser.add_subparsers(dest="subcommand")
+ build = subcommands.add_parser("build", help="Build everything")
+ configure_build = subcommands.add_parser(
+ "configure-build-python", help="Run `configure` for the build Python"
+ )
+ make_build = subcommands.add_parser(
+ "make-build-python", help="Run `make` for the build Python"
+ )
+ build_python = subcommands.add_parser(
+ "build-python", help="Build the build Python"
+ )
+ configure_host = subcommands.add_parser(
+ "configure-host",
+ help="Run `configure` for the "
+ "host/WASI (pydebug builds "
+ "are inferred from the build "
+ "Python)",
+ )
+ make_host = subcommands.add_parser(
+ "make-host", help="Run `make` for the host/WASI"
+ )
+ build_host = subcommands.add_parser(
+ "build-host", help="Build the host/WASI Python"
+ )
+ subcommands.add_parser(
+ "clean", help="Delete files and directories created by this script"
+ )
+ for subcommand in (
+ build,
+ configure_build,
+ make_build,
+ build_python,
+ configure_host,
+ make_host,
+ build_host,
+ ):
+ subcommand.add_argument(
+ "--quiet",
+ action="store_true",
+ default=False,
+ dest="quiet",
+ help="Redirect output from subprocesses to a log file",
+ )
+ subcommand.add_argument(
+ "--logdir",
+ type=pathlib.Path,
+ default=default_logdir,
+ help=f"Directory to store log files; defaults to {default_logdir}",
+ )
+ for subcommand in (
+ configure_build,
+ configure_host,
+ build_python,
+ build_host,
+ ):
+ subcommand.add_argument(
+ "--clean",
+ action="store_true",
+ default=False,
+ dest="clean",
+ help="Delete any relevant directories before building",
+ )
+ for subcommand in (
+ build,
+ configure_build,
+ configure_host,
+ build_python,
+ build_host,
+ ):
+ subcommand.add_argument(
+ "args", nargs="*", help="Extra arguments to pass to `configure`"
+ )
+ for subcommand in build, configure_host, build_host:
+ subcommand.add_argument(
+ "--wasi-sdk",
+ type=pathlib.Path,
+ dest="wasi_sdk_path",
+ default=default_wasi_sdk,
+ help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
+ )
+ subcommand.add_argument(
+ "--host-runner",
+ action="store",
+ default=default_host_runner,
+ dest="host_runner",
+ help="Command template for running the WASI host; defaults to "
+ f"`{default_host_runner}`",
+ )
+ for subcommand in build, configure_host, make_host, build_host:
+ subcommand.add_argument(
+ "--host-triple",
+ action="store",
+ default=default_host_triple,
+ help="The target triple for the WASI host build; "
+ f"defaults to {default_host_triple}",
+ )
+
+ context = parser.parse_args()
+ context.init_dir = pathlib.Path().absolute()
+
+ build_build_python = build_steps(configure_build_python, make_build_python)
+ build_wasi_python = build_steps(configure_wasi_python, make_wasi_python)
+
+ dispatch = {
+ "configure-build-python": configure_build_python,
+ "make-build-python": make_build_python,
+ "build-python": build_build_python,
+ "configure-host": configure_wasi_python,
+ "make-host": make_wasi_python,
+ "build-host": build_wasi_python,
+ "build": build_steps(build_build_python, build_wasi_python),
+ "clean": clean_contents,
+ }
+ dispatch[context.subcommand](context)
+
+
+if __name__ == "__main__":
+ main()
ANDROID_DIRS = frozenset({"Android"})
IOS_DIRS = frozenset({"Apple", "iOS"})
MACOS_DIRS = frozenset({"Mac"})
-WASI_DIRS = frozenset({Path("Tools", "wasm")})
+WASI_DIRS = frozenset({Path("Platforms", "WASI")})
LIBRARY_FUZZER_PATHS = frozenset({
# All C/CPP fuzzers.
return "ios"
if first_part in ANDROID_DIRS:
return "android"
- if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/
+ if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS:
return "wasi"
return None
-# Python WebAssembly (WASM) build
+# Python Emscripten (wasm32-emscripten) build
-**WASI support is [tier 2](https://peps.python.org/pep-0011/#tier-2).**
**Emscripten support is [tier 3](https://peps.python.org/pep-0011/#tier-3).**
This directory contains configuration and helpers to facilitate cross
-compilation of CPython to WebAssembly (WASM). Python supports Emscripten
-(*wasm32-emscripten*) and WASI (*wasm32-wasi*) targets. Emscripten builds
-run in modern browsers and JavaScript runtimes like *Node.js*. WASI builds
-use WASM runtimes such as *wasmtime*.
+compilation of CPython to WebAssembly (WASM) using Emscripten. Emscripten builds
+run in modern browsers and JavaScript runtimes like *Node.js*.
+
+For WASI builds, see [Platforms/WASI](../../Platforms/WASI/README.md).
**NOTE**: If you are looking for general information about WebAssembly that is
not directly related to CPython, please see https://github.com/psf/webassembly.
-## Emscripten (wasm32-emscripten)
-
### Build
See [the devguide instructions for building for Emscripten](https://devguide.python.org/getting-started/setup-building/#emscripten).
- Test modules are disabled by default. Use ``--enable-test-modules`` build
test modules like ``_testcapi``.
-## WASI (wasm32-wasi)
-
-See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi).
-
-## Detecting WebAssembly builds
+## Detecting Emscripten builds
### Python code
if sys.platform == "emscripten":
# Python on Emscripten
...
-if sys.platform == "wasi":
- # Python on WASI
- ...
if os.name == "posix":
# WASM platforms identify as POSIX-like.
)
```
-```python
->>> import os, sys
->>> os.uname()
-posix.uname_result(
- sysname='wasi',
- nodename='(none)',
- release='0.0.0',
- version='0.0.0',
- machine='wasm32'
-)
->>> os.name
-'posix'
->>> sys.platform
-'wasi'
-```
-
### C code
-Emscripten SDK and WASI SDK define several built-in macros. You can dump a
-full list of built-ins with ``emcc -dM -E - < /dev/null`` and
-``/path/to/wasi-sdk/bin/clang -dM -E - < /dev/null``.
+Emscripten SDK defines several built-in macros. You can dump a full list of
+built-ins with ``emcc -dM -E - < /dev/null``.
* WebAssembly ``__wasm__`` (also ``__wasm``)
* wasm32 ``__wasm32__`` (also ``__wasm32``)
* wasm64 ``__wasm64__``
* Emscripten ``__EMSCRIPTEN__`` (also ``EMSCRIPTEN``)
* Emscripten version ``__EMSCRIPTEN_major__``, ``__EMSCRIPTEN_minor__``, ``__EMSCRIPTEN_tiny__``
-* WASI ``__wasi__``
+
if __name__ == "__main__":
import pathlib
import runpy
- import sys
-
- print(
- "โ ๏ธ WARNING: This script is deprecated and slated for removal in Python 3.20; "
- "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n",
- file=sys.stderr,
- )
runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__")
-#!/usr/bin/env python3
-
-import argparse
-import contextlib
-import functools
-import os
-
-import tomllib
-
-try:
- from os import process_cpu_count as cpu_count
-except ImportError:
- from os import cpu_count
-import pathlib
-import shutil
-import subprocess
-import sys
-import sysconfig
-import tempfile
-
-HERE = pathlib.Path(__file__).parent
-
-# Path is: cpython/Tools/wasm/wasi
-CHECKOUT = HERE.parent.parent.parent
-assert (CHECKOUT / "configure").is_file(), (
- "Please update the location of the file"
-)
-
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
-# Build platform can also be found via `config.guess`.
-BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
-
-LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
-LOCAL_SETUP_MARKER = (
- b"# Generated by Tools/wasm/wasi .\n"
- b"# Required to statically build extension modules."
-)
-
-WASI_SDK_VERSION = 29
-
-WASMTIME_VAR_NAME = "WASMTIME"
-WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
-
-
-def separator():
- """Print a separator line across the terminal width."""
- try:
- tput_output = subprocess.check_output(
- ["tput", "cols"], encoding="utf-8"
- )
- except subprocess.CalledProcessError:
- terminal_width = 80
- else:
- terminal_width = int(tput_output.strip())
- print("โฏ" * terminal_width)
-
-
-def log(emoji, message, *, spacing=None):
- """Print a notification with an emoji.
-
- If 'spacing' is None, calculate the spacing based on the number of code points
- in the emoji as terminals "eat" a space when the emoji has multiple code points.
- """
- if spacing is None:
- spacing = " " if len(emoji) == 1 else " "
- print("".join([emoji, spacing, message]))
-
-
-def updated_env(updates={}):
- """Create a new dict representing the environment to use.
-
- The changes made to the execution environment are printed out.
- """
- env_defaults = {}
- # https://reproducible-builds.org/docs/source-date-epoch/
- git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
- try:
- epoch = subprocess.check_output(
- git_epoch_cmd, encoding="utf-8"
- ).strip()
- env_defaults["SOURCE_DATE_EPOCH"] = epoch
- except subprocess.CalledProcessError:
- pass # Might be building from a tarball.
- # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
- environment = env_defaults | os.environ | updates
-
- env_diff = {}
- for key, value in environment.items():
- if os.environ.get(key) != value:
- env_diff[key] = value
-
- env_vars = (
- f"\n {key}={item}" for key, item in sorted(env_diff.items())
- )
- log("๐", f"Environment changes:{''.join(env_vars)}")
-
- return environment
-
-
-def subdir(working_dir, *, clean_ok=False):
- """Decorator to change to a working directory."""
-
- def decorator(func):
- @functools.wraps(func)
- def wrapper(context):
- nonlocal working_dir
-
- if callable(working_dir):
- working_dir = working_dir(context)
- separator()
- log("๐", os.fsdecode(working_dir))
- if (
- clean_ok
- and getattr(context, "clean", False)
- and working_dir.exists()
- ):
- log("๐ฎ", "Deleting directory (--clean)...")
- shutil.rmtree(working_dir)
-
- working_dir.mkdir(parents=True, exist_ok=True)
-
- with contextlib.chdir(working_dir):
- return func(context, working_dir)
-
- return wrapper
-
- return decorator
-
-
-def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
- """Execute a command.
-
- If 'quiet' is true, then redirect stdout and stderr to a temporary file.
- """
- if context is not None:
- quiet = context.quiet
- logdir = context.logdir
- elif quiet and logdir is None:
- raise ValueError("When quiet is True, logdir must be specified")
-
- log("โฏ", " ".join(map(str, command)), spacing=" ")
- if not quiet:
- stdout = None
- stderr = None
- else:
- stdout = tempfile.NamedTemporaryFile(
- "w",
- encoding="utf-8",
- delete=False,
- dir=logdir,
- prefix="cpython-wasi-",
- suffix=".log",
- )
- stderr = subprocess.STDOUT
- log("๐", f"Logging output to {stdout.name} (--quiet)...")
-
- subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
-
-
-def build_python_path():
- """The path to the build Python binary."""
- binary = BUILD_DIR / "python"
- if not binary.is_file():
- binary = binary.with_suffix(".exe")
- if not binary.is_file():
- raise FileNotFoundError(
- f"Unable to find `python(.exe)` in {BUILD_DIR}"
- )
-
- return binary
-
-
-def build_python_is_pydebug():
- """Find out if the build Python is a pydebug build."""
- test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
- result = subprocess.run(
- [build_python_path(), "-c", test],
- capture_output=True,
- )
- return bool(result.returncode)
-
-
-@subdir(BUILD_DIR, clean_ok=True)
-def configure_build_python(context, working_dir):
- """Configure the build/host Python."""
- if LOCAL_SETUP.exists():
- if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
- log("๐", f"{LOCAL_SETUP} exists ...")
- else:
- log("โ ๏ธ", f"{LOCAL_SETUP} exists, but has unexpected contents")
- else:
- log("๐", f"Creating {LOCAL_SETUP} ...")
- LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
-
- configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
- if context.args:
- configure.extend(context.args)
-
- call(configure, context=context)
-
-
-@subdir(BUILD_DIR)
-def make_build_python(context, working_dir):
- """Make/build the build Python."""
- call(["make", "--jobs", str(cpu_count()), "all"], context=context)
-
- binary = build_python_path()
- cmd = [
- binary,
- "-c",
- "import sys; "
- "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
- ]
- version = subprocess.check_output(cmd, encoding="utf-8").strip()
-
- log("๐", f"{binary} {version}")
-
-
-def find_wasi_sdk(config):
- """Find the path to the WASI SDK."""
- wasi_sdk_path = None
- wasi_sdk_version = config["targets"]["wasi-sdk"]
-
- if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
- wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
- else:
- opt_path = pathlib.Path("/opt")
- # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
- # has said they don't plan to ever do a point release and all of their Git tags
- # lack the ``.0`` suffix.
- # Starting with WASI SDK 23, the tarballs went from containing a directory named
- # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
- # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
- potential_sdks = [
- path
- for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*")
- if path.is_dir()
- ]
- if len(potential_sdks) == 1:
- wasi_sdk_path = potential_sdks[0]
- elif (default_path := opt_path / "wasi-sdk").is_dir():
- wasi_sdk_path = default_path
-
- # Starting with WASI SDK 25, a VERSION file is included in the root
- # of the SDK directory that we can read to warn folks when they are using
- # an unsupported version.
- if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
- version_details = version_file.read_text(encoding="utf-8")
- found_version = version_details.splitlines()[0]
- # Make sure there's a trailing dot to avoid false positives if somehow the
- # supported version is a prefix of the found version (e.g. `25` and `2567`).
- if not found_version.startswith(f"{wasi_sdk_version}."):
- major_version = found_version.partition(".")[0]
- log(
- "โ ๏ธ",
- f" Found WASI SDK {major_version}, "
- f"but WASI SDK {wasi_sdk_version} is the supported version",
- )
-
- return wasi_sdk_path
-
-
-def wasi_sdk_env(context):
- """Calculate environment variables for building with wasi-sdk."""
- wasi_sdk_path = context.wasi_sdk_path
- sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
- env = {
- "CC": "clang",
- "CPP": "clang-cpp",
- "CXX": "clang++",
- "AR": "llvm-ar",
- "RANLIB": "ranlib",
- }
-
- for env_var, binary_name in list(env.items()):
- env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
-
- if not wasi_sdk_path.name.startswith("wasi-sdk"):
- for compiler in ["CC", "CPP", "CXX"]:
- env[compiler] += f" --sysroot={sysroot}"
-
- env["PKG_CONFIG_PATH"] = ""
- env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
- map(
- os.fsdecode,
- [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
- )
- )
- env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
-
- env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
- env["WASI_SYSROOT"] = os.fsdecode(sysroot)
-
- env["PATH"] = os.pathsep.join([
- os.fsdecode(wasi_sdk_path / "bin"),
- os.environ["PATH"],
- ])
-
- return env
-
-
-@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
-def configure_wasi_python(context, working_dir):
- """Configure the WASI/host build."""
- if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
- raise ValueError(
- "WASI-SDK not found; "
- "download from "
- "https://github.com/WebAssembly/wasi-sdk and/or "
- "specify via $WASI_SDK_PATH or --wasi-sdk"
- )
-
- config_site = os.fsdecode(HERE / "config.site-wasm32-wasi")
-
- wasi_build_dir = working_dir.relative_to(CHECKOUT)
-
- python_build_dir = BUILD_DIR / "build"
- lib_dirs = list(python_build_dir.glob("lib.*"))
- assert len(lib_dirs) == 1, (
- f"Expected a single lib.* directory in {python_build_dir}"
- )
- lib_dir = os.fsdecode(lib_dirs[0])
- python_version = lib_dir.rpartition("-")[-1]
- sysconfig_data_dir = (
- f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
- )
-
- # Use PYTHONPATH to include sysconfig data which must be anchored to the
- # WASI guest's `/` directory.
- args = {
- "PYTHONPATH": f"/{sysconfig_data_dir}",
- "PYTHON_WASM": working_dir / "python.wasm",
- }
- # Check dynamically for wasmtime in case it was specified manually via
- # `--host-runner`.
- if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
- if wasmtime := shutil.which("wasmtime"):
- args[WASMTIME_VAR_NAME] = wasmtime
- else:
- raise FileNotFoundError(
- "wasmtime not found; download from "
- "https://github.com/bytecodealliance/wasmtime"
- )
- host_runner = context.host_runner.format_map(args)
- env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
- build_python = os.fsdecode(build_python_path())
- # The path to `configure` MUST be relative, else `python.wasm` is unable
- # to find the stdlib due to Python not recognizing that it's being
- # executed from within a checkout.
- configure = [
- os.path.relpath(CHECKOUT / "configure", working_dir),
- f"--host={context.host_triple}",
- f"--build={BUILD_DIR.name}",
- f"--with-build-python={build_python}",
- ]
- if build_python_is_pydebug():
- configure.append("--with-pydebug")
- if context.args:
- configure.extend(context.args)
- call(
- configure,
- env=updated_env(env_additions | wasi_sdk_env(context)),
- context=context,
- )
-
- python_wasm = working_dir / "python.wasm"
- exec_script = working_dir / "python.sh"
- with exec_script.open("w", encoding="utf-8") as file:
- file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
- exec_script.chmod(0o755)
- log("๐", f"Created {exec_script} (--host-runner)... ")
- sys.stdout.flush()
-
-
-@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
-def make_wasi_python(context, working_dir):
- """Run `make` for the WASI/host build."""
- call(
- ["make", "--jobs", str(cpu_count()), "all"],
- env=updated_env(),
- context=context,
- )
-
- exec_script = working_dir / "python.sh"
- call([exec_script, "--version"], quiet=False)
- log(
- "๐",
- f"Use `{exec_script.relative_to(context.init_dir)}` "
- "to run CPython w/ the WASI host specified by --host-runner",
- )
-
-
-def clean_contents(context):
- """Delete all files created by this script."""
- if CROSS_BUILD_DIR.exists():
- log("๐งน", f"Deleting {CROSS_BUILD_DIR} ...")
- shutil.rmtree(CROSS_BUILD_DIR)
-
- if LOCAL_SETUP.exists():
- if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
- log("๐งน", f"Deleting generated {LOCAL_SETUP} ...")
-
-
-def build_steps(*steps):
- """Construct a command from other steps."""
-
- def builder(context):
- for step in steps:
- step(context)
-
- return builder
-
-
-def main():
- with (HERE / "config.toml").open("rb") as file:
- config = tomllib.load(file)
- default_wasi_sdk = find_wasi_sdk(config)
- default_host_triple = config["targets"]["host-triple"]
- default_host_runner = (
- f"{WASMTIME_HOST_RUNNER_VAR} run "
- # For setting PYTHONPATH to the sysconfig data directory.
- "--env PYTHONPATH={PYTHONPATH} "
- # Map the checkout to / to load the stdlib from /Lib.
- f"--dir {os.fsdecode(CHECKOUT)}::/ "
- # Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept
- # in a config file.
- # We are using such a file to act as defaults in case a user wants to override
- # only some of the settings themselves, make it easy to modify settings
- # post-build so that they immediately apply to the Makefile instead of having to
- # regenerate it, and allow for easy copying of the settings for anyone else who
- # may want to use them.
- f"--config {os.fsdecode(HERE / 'wasmtime.toml')}"
- )
- default_logdir = pathlib.Path(tempfile.gettempdir())
+if __name__ == "__main__":
+ import pathlib
+ import runpy
+ import sys
- parser = argparse.ArgumentParser()
- subcommands = parser.add_subparsers(dest="subcommand")
- build = subcommands.add_parser("build", help="Build everything")
- configure_build = subcommands.add_parser(
- "configure-build-python", help="Run `configure` for the build Python"
- )
- make_build = subcommands.add_parser(
- "make-build-python", help="Run `make` for the build Python"
- )
- build_python = subcommands.add_parser(
- "build-python", help="Build the build Python"
- )
- configure_host = subcommands.add_parser(
- "configure-host",
- help="Run `configure` for the "
- "host/WASI (pydebug builds "
- "are inferred from the build "
- "Python)",
- )
- make_host = subcommands.add_parser(
- "make-host", help="Run `make` for the host/WASI"
+ print(
+ "โ ๏ธ WARNING: This script is deprecated and slated for removal in Python 3.20; "
+ "execute the `Platforms/WASI/` directory instead (i.e. `python Platforms/WASI`)\n",
+ file=sys.stderr,
)
- build_host = subcommands.add_parser(
- "build-host", help="Build the host/WASI Python"
- )
- subcommands.add_parser(
- "clean", help="Delete files and directories created by this script"
- )
- for subcommand in (
- build,
- configure_build,
- make_build,
- build_python,
- configure_host,
- make_host,
- build_host,
- ):
- subcommand.add_argument(
- "--quiet",
- action="store_true",
- default=False,
- dest="quiet",
- help="Redirect output from subprocesses to a log file",
- )
- subcommand.add_argument(
- "--logdir",
- type=pathlib.Path,
- default=default_logdir,
- help=f"Directory to store log files; defaults to {default_logdir}",
- )
- for subcommand in (
- configure_build,
- configure_host,
- build_python,
- build_host,
- ):
- subcommand.add_argument(
- "--clean",
- action="store_true",
- default=False,
- dest="clean",
- help="Delete any relevant directories before building",
- )
- for subcommand in (
- build,
- configure_build,
- configure_host,
- build_python,
- build_host,
- ):
- subcommand.add_argument(
- "args", nargs="*", help="Extra arguments to pass to `configure`"
- )
- for subcommand in build, configure_host, build_host:
- subcommand.add_argument(
- "--wasi-sdk",
- type=pathlib.Path,
- dest="wasi_sdk_path",
- default=default_wasi_sdk,
- help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
- )
- subcommand.add_argument(
- "--host-runner",
- action="store",
- default=default_host_runner,
- dest="host_runner",
- help="Command template for running the WASI host; defaults to "
- f"`{default_host_runner}`",
- )
- for subcommand in build, configure_host, make_host, build_host:
- subcommand.add_argument(
- "--host-triple",
- action="store",
- default=default_host_triple,
- help="The target triple for the WASI host build; "
- f"defaults to {default_host_triple}",
- )
-
- context = parser.parse_args()
- context.init_dir = pathlib.Path().absolute()
- build_build_python = build_steps(configure_build_python, make_build_python)
- build_wasi_python = build_steps(configure_wasi_python, make_wasi_python)
+ checkout = pathlib.Path(__file__).parent.parent.parent.parent
- dispatch = {
- "configure-build-python": configure_build_python,
- "make-build-python": make_build_python,
- "build-python": build_build_python,
- "configure-host": configure_wasi_python,
- "make-host": make_wasi_python,
- "build-host": build_wasi_python,
- "build": build_steps(build_build_python, build_wasi_python),
- "clean": clean_contents,
- }
- dispatch[context.subcommand](context)
-
-
-if __name__ == "__main__":
- main()
+ runpy.run_path(checkout / "Platforms" / "WASI", run_name="__main__")
projects / thirdparty / Python / cpython.git / commitdiff
