]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-95853: Add script to automate WASM build (GH-95828) 95955/head
authorChristian Heimes <christian@python.org>
Sat, 13 Aug 2022 19:56:08 +0000 (21:56 +0200)
committerGitHub <noreply@github.com>
Sat, 13 Aug 2022 19:56:08 +0000 (21:56 +0200)
Automate WASM build with a new Python script. The script provides
several build profiles with configure flags for Emscripten flavors
and WASI. The script can detect and use Emscripten SDK and WASI SDK from
default locations or env vars.

``configure`` now detects Node arguments and creates HOSTRUNNER
arguments for Node 16. It also sets correct arguments for
``wasm64-emscripten``.

Co-authored-by: Brett Cannon <brett@python.org>
Lib/test/test_unicode_file_functions.py
Lib/test/test_warnings/__init__.py
Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst [new file with mode: 0644]
Python/sysmodule.c
Tools/wasm/README.md
Tools/wasm/wasi-env
Tools/wasm/wasm_build.py [new file with mode: 0755]
configure
configure.ac

index 54916dec4eafa3b78c750c5209ebf70776db647b..47619c8807bafe30c454ac683a5cae07e2ab8841 100644 (file)
@@ -6,6 +6,7 @@ import unittest
 import warnings
 from unicodedata import normalize
 from test.support import os_helper
+from test import support
 
 
 filenames = [
@@ -123,6 +124,10 @@ class UnicodeFileTests(unittest.TestCase):
     # NFKD in Python is useless, because darwin will normalize it later and so
     # open(), os.stat(), etc. don't raise any exception.
     @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X')
+    @unittest.skipIf(
+        support.is_emscripten or support.is_wasi,
+        "test fails on Emscripten/WASI when host platform is macOS."
+    )
     def test_normalize(self):
         files = set(self.files)
         others = set()
index b00ddd5df2f256735fbf593f68b525d1798fb422..9e473e923cad03111d98e2d9c90a055ee8694877 100644 (file)
@@ -487,7 +487,14 @@ class WarnTests(BaseTest):
                 module=self.module) as w:
             self.module.resetwarnings()
             self.module.filterwarnings("always", category=UserWarning)
-            for filename in ("nonascii\xe9\u20ac", "surrogate\udc80"):
+            filenames = ["nonascii\xe9\u20ac"]
+            if not support.is_emscripten:
+                # JavaScript does not like surrogates.
+                # Invalid UTF-8 leading byte 0x80 encountered when
+                # deserializing a UTF-8 string in wasm memory to a JS
+                # string!
+                filenames.append("surrogate\udc80")
+            for filename in filenames:
                 try:
                     os.fsencode(filename)
                 except UnicodeEncodeError:
diff --git a/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst b/Misc/NEWS.d/next/Tools-Demos/2022-08-10-17-08-43.gh-issue-95853.HCjC2m.rst
new file mode 100644 (file)
index 0000000..c38db3a
--- /dev/null
@@ -0,0 +1,2 @@
+The new tool ``Tools/wasm/wasm_builder.py`` automates configure, compile, and
+test steps for building CPython on WebAssembly platforms.
index e861d9cbce415c45b24e84fd15ffae7135078c56..b8009b2db45f7bebeb4f8437fbd06823fd23b188 100644 (file)
@@ -2789,14 +2789,18 @@ EM_JS(char *, _Py_emscripten_runtime, (void), {
     if (typeof navigator == 'object') {
         info = navigator.userAgent;
     } else if (typeof process == 'object') {
-        info = "Node.js ".concat(process.version)
+        info = "Node.js ".concat(process.version);
     } else {
-        info = "UNKNOWN"
+        info = "UNKNOWN";
     }
     var len = lengthBytesUTF8(info) + 1;
     var res = _malloc(len);
-    stringToUTF8(info, res, len);
+    if (res) stringToUTF8(info, res, len);
+#if __wasm64__
+    return BigInt(res);
+#else
     return res;
+#endif
 });
 
 static PyObject *
index 6496a29e6ff809f9d01cf4612bb8425dc9f6bfc2..c4c21b4f09dd2af7f004f2e211c6f33692b10fc9 100644 (file)
@@ -35,7 +35,13 @@ docker run --rm -ti -v $(pwd):/python-wasm/cpython -w /python-wasm/cpython quay.
 
 ### Compile a build Python interpreter
 
-From within the container, run the following commands:
+From within the container, run the following command:
+
+```shell
+./Tools/wasm/wasm_build.py build
+```
+
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/build
@@ -45,13 +51,13 @@ make -j$(nproc)
 popd
 ```
 
-### Fetch and build additional emscripten ports
+### Cross-compile to wasm32-emscripten for browser
 
 ```shell
-embuilder build zlib bzip2
+./Tools/wasm/wasm_build.py emscripten-browser
 ```
 
-### Cross compile to wasm32-emscripten for browser
+The command is roughly equivalent to:
 
 ```shell
 mkdir -p builddir/emscripten-browser
@@ -85,14 +91,21 @@ and header files with debug builds.
 ### Cross compile to wasm32-emscripten for node
 
 ```shell
-mkdir -p builddir/emscripten-node
-pushd builddir/emscripten-node
+./Tools/wasm/wasm_build.py emscripten-browser-dl
+```
+
+The command is roughly equivalent to:
+
+```shell
+mkdir -p builddir/emscripten-node-dl
+pushd builddir/emscripten-node-dl
 
 CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
   emconfigure ../../configure -C \
     --host=wasm32-unknown-emscripten \
     --build=$(../../config.guess) \
     --with-emscripten-target=node \
+    --enable-wasm-dynamic-linking \
     --with-build-python=$(pwd)/../build/python
 
 emmake make -j$(nproc)
@@ -100,7 +113,7 @@ popd
 ```
 
 ```shell
-node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node/python.js
+node --experimental-wasm-threads --experimental-wasm-bulk-memory --experimental-wasm-bigint builddir/emscripten-node-dl/python.js
 ```
 
 (``--experimental-wasm-bigint`` is not needed with recent NodeJS versions)
@@ -199,6 +212,15 @@ Node builds use ``NODERAWFS``.
 - Node RawFS allows direct access to the host file system without need to
   perform ``FS.mount()`` call.
 
+## wasm64-emscripten
+
+- wasm64 requires recent NodeJS and ``--experimental-wasm-memory64``.
+- ``EM_JS`` functions must return ``BigInt()``.
+- ``Py_BuildValue()`` format strings must match size of types. Confusing 32
+  and 64 bits types leads to memory corruption, see
+  [gh-95876](https://github.com/python/cpython/issues/95876) and
+  [gh-95878](https://github.com/python/cpython/issues/95878).
+
 # Hosting Python WASM builds
 
 The simple REPL terminal uses SharedArrayBuffer. For security reasons
@@ -234,6 +256,12 @@ The script ``wasi-env`` sets necessary compiler and linker flags as well as
 ``pkg-config`` overrides. The script assumes that WASI-SDK is installed in
 ``/opt/wasi-sdk`` or ``$WASI_SDK_PATH``.
 
+```shell
+./Tools/wasm/wasm_build.py wasi
+```
+
+The command is roughly equivalent to:
+
 ```shell
 mkdir -p builddir/wasi
 pushd builddir/wasi
index 6c2d56e0e5e32b16466cd15a35d30ff5d181cb32..48908b02e60b960aed514d69ce87f5f835e3ccd7 100755 (executable)
@@ -72,4 +72,5 @@ export CFLAGS LDFLAGS
 export PKG_CONFIG_PATH PKG_CONFIG_LIBDIR PKG_CONFIG_SYSROOT_DIR
 export PATH
 
-exec "$@"
+# no exec, it makes arvg[0] path absolute.
+"$@"
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
new file mode 100755 (executable)
index 0000000..e7a1f4a
--- /dev/null
@@ -0,0 +1,567 @@
+#!/usr/bin/env python3
+"""Build script for Python on WebAssembly platforms.
+
+  $ ./Tools/wasm/wasm_builder.py emscripten-browser compile
+  $ ./Tools/wasm/wasm_builder.py emscripten-node-dl test
+  $ ./Tools/wasm/wasm_builder.py wasi test
+
+Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
+"emscripten-browser", and "wasi".
+
+Emscripten builds require a recent Emscripten SDK. The tools looks for an
+activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
+(Debian, Homebrew) are not supported.
+
+WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
+and falls back to /opt/wasi-sdk.
+"""
+import argparse
+import enum
+import dataclasses
+import os
+import pathlib
+import shlex
+import shutil
+import subprocess
+import sysconfig
+
+# for Python 3.8
+from typing import Any, Dict, Callable, Iterable, List, Optional, Union
+
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+WASMTOOLS = SRCDIR / "Tools" / "wasm"
+BUILDDIR = SRCDIR / "builddir"
+CONFIGURE = SRCDIR / "configure"
+SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
+
+HAS_CCACHE = shutil.which("ccache") is not None
+
+# path to WASI-SDK root
+WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
+
+# path to Emscripten SDK config file.
+# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
+EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
+# 3.1.16 has broken utime()
+EMSDK_MIN_VERSION = (3, 1, 17)
+_MISSING = pathlib.PurePath("MISSING")
+
+# WASM_WEBSERVER = WASMTOOLS / "wasmwebserver.py"
+
+CLEAN_SRCDIR = f"""
+Builds require a clean source directory. Please use a clean checkout or
+run "make clean -C '{SRCDIR}'".
+"""
+
+INSTALL_EMSDK = """
+wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
+https://emscripten.org/docs/getting_started/downloads.html how to install
+Emscripten and how to activate the SDK with ". /path/to/emsdk/emsdk_env.sh".
+
+    git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
+    cd /path/to/emsdk
+    ./emsdk install latest
+    ./emsdk activate latest
+    source /path/to/emsdk_env.sh
+"""
+
+INSTALL_WASI_SDK = """
+wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
+https://github.com/WebAssembly/wasi-sdk/releases and install it to
+"/opt/wasi-sdk". Alternatively you can install the SDK in a different location
+and point the environment variable WASI_SDK_PATH to the root directory
+of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
+"""
+
+INSTALL_WASMTIME = """
+wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
+https://wasmtime.dev/ to install wasmtime.
+"""
+
+
+def get_emscripten_root(emconfig: pathlib.Path = EM_CONFIG) -> pathlib.PurePath:
+    """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT
+
+    The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
+    environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
+    subdirectory with tools like "emconfigure".
+    """
+    if not emconfig.exists():
+        return _MISSING
+    with open(emconfig, encoding="utf-8") as f:
+        code = f.read()
+    # EM_CONFIG file is a Python snippet
+    local: Dict[str, Any] = {}
+    exec(code, globals(), local)
+    return pathlib.Path(local["EMSCRIPTEN_ROOT"])
+
+
+EMSCRIPTEN_ROOT = get_emscripten_root()
+
+
+class ConditionError(ValueError):
+    def __init__(self, info: str, text: str):
+        self.info = info
+        self.text = text
+
+    def __str__(self):
+        return f"{type(self).__name__}: '{self.info}'\n{self.text}"
+
+
+class MissingDependency(ConditionError):
+    pass
+
+
+class DirtySourceDirectory(ConditionError):
+    pass
+
+
+@dataclasses.dataclass
+class Platform:
+    """Platform-specific settings
+
+    - CONFIG_SITE override
+    - configure wrapper (e.g. emconfigure)
+    - make wrapper (e.g. emmake)
+    - additional environment variables
+    - check function to verify SDK
+    """
+
+    name: str
+    pythonexe: str
+    config_site: Optional[pathlib.PurePath]
+    configure_wrapper: Optional[pathlib.PurePath]
+    make_wrapper: Optional[pathlib.PurePath]
+    environ: dict
+    check: Callable[[], None]
+
+    def getenv(self, profile: "BuildProfile") -> dict:
+        return self.environ.copy()
+
+
+def _check_clean_src():
+    candidates = [
+        SRCDIR / "Programs" / "python.o",
+        SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
+
+
+NATIVE = Platform(
+    "native",
+    # macOS has python.exe
+    pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
+    config_site=None,
+    configure_wrapper=None,
+    make_wrapper=None,
+    environ={},
+    check=_check_clean_src,
+)
+
+
+def _check_emscripten():
+    if EMSCRIPTEN_ROOT is _MISSING:
+        raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
+    # sanity check
+    emconfigure = EMSCRIPTEN.configure_wrapper
+    if not emconfigure.exists():
+        raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
+    # version check
+    version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
+    if not version_txt.exists():
+        raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
+    with open(version_txt) as f:
+        version = f.read().strip().strip('"')
+    version_tuple = tuple(int(v) for v in version.split("."))
+    if version_tuple < EMSDK_MIN_VERSION:
+        raise MissingDependency(
+            os.fspath(version_txt),
+            f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
+            "minimum required version "
+            f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
+        )
+    _check_clean_src()
+
+
+EMSCRIPTEN = Platform(
+    "emscripten",
+    pythonexe="python.js",
+    config_site=WASMTOOLS / "config.site-wasm32-emscripten",
+    configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
+    make_wrapper=EMSCRIPTEN_ROOT / "emmake",
+    environ={"EM_COMPILER_WRAPPER": "ccache"} if HAS_CCACHE else {},
+    check=_check_emscripten,
+)
+
+
+def _check_wasi():
+    wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
+    if not wasm_ld.exists():
+        raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
+    wasmtime = shutil.which("wasmtime")
+    if wasmtime is None:
+        raise MissingDependency("wasmtime", INSTALL_WASMTIME)
+    _check_clean_src()
+
+
+WASI = Platform(
+    "wasi",
+    pythonexe="python.wasm",
+    config_site=WASMTOOLS / "config.site-wasm32-wasi",
+    configure_wrapper=WASMTOOLS / "wasi-env",
+    make_wrapper=None,
+    environ={
+        "WASI_SDK_PATH": WASI_SDK_PATH,
+        # workaround for https://github.com/python/cpython/issues/95952
+        "HOSTRUNNER": (
+            "wasmtime run "
+            "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-$(VERSION):/Lib "
+            "--mapdir /::{srcdir} --"
+        ),
+    },
+    check=_check_wasi,
+)
+
+
+class Host(enum.Enum):
+    """Target host triplet"""
+
+    wasm32_emscripten = "wasm32-unknown-emscripten"
+    wasm64_emscripten = "wasm64-unknown-emscripten"
+    wasm32_wasi = "wasm32-unknown-wasi"
+    wasm64_wasi = "wasm64-unknown-wasi"
+    # current platform
+    build = sysconfig.get_config_var("BUILD_GNU_TYPE")
+
+    @property
+    def platform(self) -> Platform:
+        if self.is_emscripten:
+            return EMSCRIPTEN
+        elif self.is_wasi:
+            return WASI
+        else:
+            return NATIVE
+
+    @property
+    def is_emscripten(self) -> bool:
+        cls = type(self)
+        return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
+
+    @property
+    def is_wasi(self) -> bool:
+        cls = type(self)
+        return self in {cls.wasm32_wasi, cls.wasm64_wasi}
+
+
+class EmscriptenTarget(enum.Enum):
+    """Emscripten-specific targets (--with-emscripten-target)"""
+
+    browser = "browser"
+    browser_debug = "browser-debug"
+    node = "node"
+    node_debug = "node-debug"
+
+    @property
+    def can_execute(self) -> bool:
+        cls = type(self)
+        return self not in {cls.browser, cls.browser_debug}
+
+
+@dataclasses.dataclass
+class BuildProfile:
+    name: str
+    host: Host
+    target: Union[EmscriptenTarget, None] = None
+    dynamic_linking: Union[bool, None] = None
+    pthreads: Union[bool, None] = None
+    testopts: str = "-j2"
+
+    @property
+    def can_execute(self) -> bool:
+        """Can target run pythoninfo and tests?
+
+        Disabled for browser, enabled for all other targets
+        """
+        return self.target is None or self.target.can_execute
+
+    @property
+    def builddir(self) -> pathlib.Path:
+        """Path to build directory"""
+        return BUILDDIR / self.name
+
+    @property
+    def python_cmd(self) -> pathlib.Path:
+        """Path to python executable"""
+        return self.builddir / self.host.platform.pythonexe
+
+    @property
+    def makefile(self) -> pathlib.Path:
+        """Path to Makefile"""
+        return self.builddir / "Makefile"
+
+    @property
+    def configure_cmd(self) -> List[str]:
+        """Generate configure command"""
+        # use relative path, so WASI tests can find lib prefix.
+        # pathlib.Path.relative_to() does not work here.
+        configure = os.path.relpath(CONFIGURE, self.builddir)
+        cmd = [configure, "-C"]
+        platform = self.host.platform
+        if platform.configure_wrapper:
+            cmd.insert(0, os.fspath(platform.configure_wrapper))
+
+        cmd.append(f"--host={self.host.value}")
+        cmd.append(f"--build={Host.build.value}")
+
+        if self.target is not None:
+            assert self.host.is_emscripten
+            cmd.append(f"--with-emscripten-target={self.target.value}")
+
+        if self.dynamic_linking is not None:
+            assert self.host.is_emscripten
+            opt = "enable" if self.dynamic_linking else "disable"
+            cmd.append(f"--{opt}-wasm-dynamic-linking")
+
+        if self.pthreads is not None:
+            assert self.host.is_emscripten
+            opt = "enable" if self.pthreads else "disable"
+            cmd.append(f"--{opt}-wasm-pthreads")
+
+        if self.host != Host.build:
+            cmd.append(f"--with-build-python={BUILD.python_cmd}")
+
+        if platform.config_site is not None:
+            cmd.append(f"CONFIG_SITE={platform.config_site}")
+
+        return cmd
+
+    @property
+    def make_cmd(self) -> List[str]:
+        """Generate make command"""
+        cmd = ["make"]
+        platform = self.host.platform
+        if platform.make_wrapper:
+            cmd.insert(0, os.fspath(platform.make_wrapper))
+        return cmd
+
+    def getenv(self) -> dict:
+        """Generate environ dict for platform"""
+        env = os.environ.copy()
+        env.setdefault("MAKEFLAGS", f"-j{os.cpu_count()}")
+        platenv = self.host.platform.getenv(self)
+        for key, value in platenv.items():
+            if isinstance(value, str):
+                value = value.format(
+                    relbuilddir=self.builddir.relative_to(SRCDIR),
+                    srcdir=SRCDIR,
+                )
+            env[key] = value
+        return env
+
+    def _run_cmd(self, cmd: Iterable[str], args: Iterable[str]):
+        cmd = list(cmd)
+        cmd.extend(args)
+        return subprocess.check_call(
+            cmd,
+            cwd=os.fspath(self.builddir),
+            env=self.getenv(),
+        )
+
+    def _check_execute(self):
+        if not self.can_execute:
+            raise ValueError(f"Cannot execute on {self.target}")
+
+    def run_build(self, force_configure: bool = False):
+        """Run configure (if necessary) and make"""
+        if force_configure or not self.makefile.exists():
+            self.run_configure()
+        self.run_make()
+
+    def run_configure(self, *args):
+        """Run configure script to generate Makefile"""
+        os.makedirs(self.builddir, exist_ok=True)
+        return self._run_cmd(self.configure_cmd, args)
+
+    def run_make(self, *args):
+        """Run make (defaults to build all)"""
+        return self._run_cmd(self.make_cmd, args)
+
+    def run_pythoninfo(self):
+        """Run 'make pythoninfo'"""
+        self._check_execute()
+        return self.run_make("pythoninfo")
+
+    def run_test(self):
+        """Run buildbottests"""
+        self._check_execute()
+        return self.run_make("buildbottest", f"TESTOPTS={self.testopts}")
+
+    def run_py(self, *args):
+        """Run Python with hostrunner"""
+        self._check_execute()
+        self.run_make(
+            "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
+        )
+
+    def clean(self, all: bool = False):
+        """Clean build directory"""
+        if all:
+            if self.builddir.exists():
+                shutil.rmtree(self.builddir)
+        elif self.makefile.exists():
+            self.run_make("clean")
+
+
+# native build (build Python)
+BUILD = BuildProfile(
+    "build",
+    host=Host.build,
+)
+
+_profiles = [
+    BUILD,
+    # wasm32-emscripten
+    BuildProfile(
+        "emscripten-browser",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.browser,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-browser-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.browser_debug,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-dl",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-dl-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node_debug,
+        dynamic_linking=True,
+    ),
+    BuildProfile(
+        "emscripten-node-pthreads",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node,
+        pthreads=True,
+    ),
+    BuildProfile(
+        "emscripten-node-pthreads-debug",
+        host=Host.wasm32_emscripten,
+        target=EmscriptenTarget.node_debug,
+        pthreads=True,
+    ),
+    # wasm64-emscripten (currently not working)
+    BuildProfile(
+        "wasm64-emscripten-node-debug",
+        host=Host.wasm64_emscripten,
+        target=EmscriptenTarget.node_debug,
+        # MEMORY64 is not compatible with dynamic linking
+        dynamic_linking=False,
+        pthreads=False,
+    ),
+    # wasm32-wasi
+    BuildProfile(
+        "wasi",
+        host=Host.wasm32_wasi,
+        # skip sysconfig test_srcdir
+        testopts="-i '*.test_srcdir' -j2",
+    ),
+    # no SDK available yet
+    # BuildProfile(
+    #    "wasm64-wasi",
+    #    host=Host.wasm64_wasi,
+    # ),
+]
+
+PROFILES = {p.name: p for p in _profiles}
+
+parser = argparse.ArgumentParser(
+    "wasm_build.py",
+    description=__doc__,
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+)
+parser.add_argument(
+    "--clean", "-c", help="Clean build directories first", action="store_true"
+)
+
+platforms = list(PROFILES) + ["cleanall"]
+parser.add_argument(
+    "platform",
+    metavar="PLATFORM",
+    help=f"Build platform: {', '.join(platforms)}",
+    choices=platforms,
+)
+
+ops = ["compile", "pythoninfo", "test", "repl", "clean", "cleanall"]
+parser.add_argument(
+    "op",
+    metavar="OP",
+    help=f"operation: {', '.join(ops)}",
+    choices=ops,
+    default="compile",
+    nargs="?",
+)
+
+
+def main():
+    args = parser.parse_args()
+    if args.platform == "cleanall":
+        for builder in PROFILES.values():
+            builder.clean(all=True)
+        parser.exit(0)
+
+    builder = PROFILES[args.platform]
+    try:
+        builder.host.platform.check()
+    except ConditionError as e:
+        parser.error(str(e))
+
+    # hack for WASI
+    if builder.host.is_wasi and not SETUP_LOCAL.exists():
+        SETUP_LOCAL.touch()
+
+    if args.op in {"compile", "pythoninfo", "repl", "test"}:
+        # all targets need a build Python
+        if builder is not BUILD:
+            if args.clean:
+                BUILD.clean(all=False)
+                BUILD.run_build()
+            elif not BUILD.python_cmd.exists():
+                BUILD.run_build()
+
+        if args.clean:
+            builder.clean(all=False)
+
+        if args.op == "compile":
+            builder.run_build(force_configure=True)
+        else:
+            if not builder.makefile.exists():
+                builder.run_configure()
+            if args.op == "pythoninfo":
+                builder.run_pythoninfo()
+            elif args.op == "repl":
+                builder.run_py()
+            elif args.op == "test":
+                builder.run_test()
+    elif args.op == "clean":
+        builder.clean(all=False)
+    elif args.op == "cleanall":
+        builder.clean(all=True)
+    else:
+        raise ValueError(args.op)
+
+    print(builder.builddir)
+    parser.exit(0)
+
+
+if __name__ == "__main__":
+    main()
index 3f25d43dde6f0cd172b0b2e85c8d239d987e83bf..82b55a3745d5751aedc04ed9c08817854dac4cbd 100755 (executable)
--- a/configure
+++ b/configure
@@ -906,6 +906,7 @@ AR
 LINK_PYTHON_OBJS
 LINK_PYTHON_DEPS
 LIBRARY_DEPS
+NODE
 HOSTRUNNER
 STATIC_LIBPYTHON
 GNULD
@@ -4079,6 +4080,16 @@ if test -z "$CFLAGS"; then
         CFLAGS=
 fi
 
+case $host in #(
+  wasm64-*-emscripten) :
+
+    as_fn_append CFLAGS " -sMEMORY64=1"
+    as_fn_append LDFLAGS " -sMEMORY64=1"
+   ;; #(
+  *) :
+     ;;
+esac
+
 if test "$ac_sys_system" = "Darwin"
 then
     # Extract the first word of "xcrun", so it can be a program name with args.
@@ -6220,7 +6231,7 @@ cat > conftest.c <<EOF
 #    error unknown wasm32 platform
 #  endif
 #elif defined(__wasm64__)
-#  if defined(__EMSCRIPTEN)
+#  if defined(__EMSCRIPTEN__)
        wasm64-emscripten
 #  elif defined(__wasi__)
        wasm64-wasi
@@ -6840,19 +6851,162 @@ if test "$cross_compiling" = yes; then
 fi
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5
-$as_echo_n "checking HOSTRUNNER... " >&6; }
 if test -z "$HOSTRUNNER"
 then
   case $ac_sys_system/$ac_sys_emscripten_target in #(
   Emscripten/node*) :
 
+      if test -n "$ac_tool_prefix"; then
+  # Extract the first word of "${ac_tool_prefix}node", so it can be a program name with args.
+set dummy ${ac_tool_prefix}node; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_NODE+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $NODE in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_NODE="$NODE" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_NODE="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  ;;
+esac
+fi
+NODE=$ac_cv_path_NODE
+if test -n "$NODE"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NODE" >&5
+$as_echo "$NODE" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+fi
+if test -z "$ac_cv_path_NODE"; then
+  ac_pt_NODE=$NODE
+  # Extract the first word of "node", so it can be a program name with args.
+set dummy node; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_path_ac_pt_NODE+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  case $ac_pt_NODE in
+  [\\/]* | ?:[\\/]*)
+  ac_cv_path_ac_pt_NODE="$ac_pt_NODE" # Let the user override the test with a path.
+  ;;
+  *)
+  as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_path_ac_pt_NODE="$as_dir/$ac_word$ac_exec_ext"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  ;;
+esac
+fi
+ac_pt_NODE=$ac_cv_path_ac_pt_NODE
+if test -n "$ac_pt_NODE"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_NODE" >&5
+$as_echo "$ac_pt_NODE" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+  if test "x$ac_pt_NODE" = x; then
+    NODE="node"
+  else
+    case $cross_compiling:$ac_tool_warned in
+yes:)
+{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5
+$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;}
+ac_tool_warned=yes ;;
+esac
+    NODE=$ac_pt_NODE
+  fi
+else
+  NODE="$ac_cv_path_NODE"
+fi
+
+      HOSTRUNNER="$NODE"
       # bigint for ctypes c_longlong, c_longdouble
-      HOSTRUNNER="node --experimental-wasm-bigint"
+      # no longer available in Node 16
+      { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bigint" >&5
+$as_echo_n "checking for node --experimental-wasm-bigint... " >&6; }
+if ${ac_cv_tool_node_wasm_bigint+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+        if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then
+          ac_cv_tool_node_wasm_bigint=yes
+        else
+          ac_cv_tool_node_wasm_bigint=no
+        fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bigint" >&5
+$as_echo "$ac_cv_tool_node_wasm_bigint" >&6; }
+      if test "x$ac_cv_tool_node_wasm_bigint" = xyes; then :
+
+        as_fn_append HOSTRUNNER " --experimental-wasm-bigint"
+
+fi
+
       if test "x$enable_wasm_pthreads" = xyes; then :
 
-        HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory"
+        as_fn_append HOSTRUNNER " --experimental-wasm-threads"
+        # no longer available in Node 16
+        { $as_echo "$as_me:${as_lineno-$LINENO}: checking for node --experimental-wasm-bulk-memory" >&5
+$as_echo_n "checking for node --experimental-wasm-bulk-memory... " >&6; }
+if ${ac_cv_tool_node_wasm_bulk_memory+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+          if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then
+            ac_cv_tool_node_wasm_bulk_memory=yes
+          else
+            ac_cv_tool_node_wasm_bulk_memory=no
+          fi
+
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_tool_node_wasm_bulk_memory" >&5
+$as_echo "$ac_cv_tool_node_wasm_bulk_memory" >&6; }
+        if test "x$ac_cv_tool_node_wasm_bulk_memory" = xyes; then :
+
+          as_fn_append HOSTRUNNER " --experimental-wasm-bulk-memory"
 
+fi
+
+fi
+
+      if test "x$host_cpu" = xwasm64; then :
+  as_fn_append HOSTRUNNER " --experimental-wasm-memory64"
 fi
      ;; #(
               WASI/*) :
@@ -6863,6 +7017,8 @@ fi
 esac
 fi
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5
+$as_echo_n "checking HOSTRUNNER... " >&6; }
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5
 $as_echo "$HOSTRUNNER" >&6; }
 
index 8decd9ebae84cf33a59fd2f3bd8496b160ff05c2..85d9e8011835edbc9dad5e22bac4105bee0a3bbe 100644 (file)
@@ -753,6 +753,16 @@ if test -z "$CFLAGS"; then
         CFLAGS=
 fi
 
+dnl Emscripten SDK and WASI SDK default to wasm32.
+dnl On Emscripten use MEMORY64 setting to build target wasm64-emscripten.
+dnl for wasm64.
+AS_CASE([$host],
+  [wasm64-*-emscripten], [
+    AS_VAR_APPEND([CFLAGS], [" -sMEMORY64=1"])
+    AS_VAR_APPEND([LDFLAGS], [" -sMEMORY64=1"])
+  ],
+)
+
 if test "$ac_sys_system" = "Darwin"
 then
   dnl look for SDKROOT
@@ -1048,7 +1058,7 @@ cat > conftest.c <<EOF
 #    error unknown wasm32 platform
 #  endif
 #elif defined(__wasm64__)
-#  if defined(__EMSCRIPTEN)
+#  if defined(__EMSCRIPTEN__)
        wasm64-emscripten
 #  elif defined(__wasi__)
        wasm64-wasi
@@ -1515,16 +1525,41 @@ if test "$cross_compiling" = yes; then
 fi
 
 AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform])
-AC_MSG_CHECKING([HOSTRUNNER])
 if test -z "$HOSTRUNNER"
 then
   AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
     [Emscripten/node*], [
+      AC_PATH_TOOL([NODE], [node], [node])
+      HOSTRUNNER="$NODE"
       # bigint for ctypes c_longlong, c_longdouble
-      HOSTRUNNER="node --experimental-wasm-bigint"
+      # no longer available in Node 16
+      AC_CACHE_CHECK([for node --experimental-wasm-bigint], [ac_cv_tool_node_wasm_bigint], [
+        if $NODE -v --experimental-wasm-bigint > /dev/null 2>&1; then
+          ac_cv_tool_node_wasm_bigint=yes
+        else
+          ac_cv_tool_node_wasm_bigint=no
+        fi
+      ])
+      AS_VAR_IF([ac_cv_tool_node_wasm_bigint], [yes], [
+        AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bigint"])
+      ])
+
       AS_VAR_IF([enable_wasm_pthreads], [yes], [
-        HOSTRUNNER="$HOSTRUNNER --experimental-wasm-threads --experimental-wasm-bulk-memory"
+        AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-threads"])
+        # no longer available in Node 16
+        AC_CACHE_CHECK([for node --experimental-wasm-bulk-memory], [ac_cv_tool_node_wasm_bulk_memory], [
+          if $NODE -v --experimental-wasm-bulk-memory > /dev/null 2>&1; then
+            ac_cv_tool_node_wasm_bulk_memory=yes
+          else
+            ac_cv_tool_node_wasm_bulk_memory=no
+          fi
+        ])
+        AS_VAR_IF([ac_cv_tool_node_wasm_bulk_memory], [yes], [
+          AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-bulk-memory"])
+        ])
       ])
+
+      AS_VAR_IF([host_cpu], [wasm64], [AS_VAR_APPEND([HOSTRUNNER], [" --experimental-wasm-memory64"])])
     ],
     dnl TODO: support other WASI runtimes
     dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the
@@ -1534,6 +1569,7 @@ then
   )
 fi
 AC_SUBST([HOSTRUNNER])
+AC_MSG_CHECKING([HOSTRUNNER])
 AC_MSG_RESULT([$HOSTRUNNER])
 
 if test -n "$HOSTRUNNER"; then