PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix"
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
-LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8")
+LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode(
+ "utf-8"
+)
def updated_env(updates={}):
# https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try:
- epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
+ epoch = subprocess.check_output(
+ git_epoch_cmd, encoding="utf-8"
+ ).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError:
pass # Might be building from a tarball.
terminal_width = 80
print("⎯" * terminal_width)
print("📁", working_dir)
- if clean_ok and getattr(context, "clean", False) and working_dir.exists():
+ if (
+ clean_ok
+ and getattr(context, "clean", False)
+ and working_dir.exists()
+ ):
print("🚮 Deleting directory (--clean)...")
shutil.rmtree(working_dir)
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
- raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}")
+ raise FileNotFoundError(
+ f"Unable to find `python(.exe)` in {NATIVE_BUILD_DIR}"
+ )
return binary
cmd = [
binary,
"-c",
- "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+ "import sys; "
+ "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
]
version = subprocess.check_output(cmd, encoding="utf-8").strip()
def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
- with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete_on_close=False) as tmp_file:
+ with tempfile.NamedTemporaryFile(
+ suffix=".tar.gz", delete_on_close=False
+ ) as tmp_file:
with urlopen(url) as response:
shutil.copyfileobj(response, tmp_file)
tmp_file.close()
ver = "3.4.6"
libffi_dir = working_dir / f"libffi-{ver}"
shutil.rmtree(libffi_dir, ignore_errors=True)
- download_and_unpack(working_dir, f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz", "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e")
+ download_and_unpack(
+ working_dir,
+ f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz",
+ "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e",
+ )
call(
[EMSCRIPTEN_DIR / "make_libffi.sh"],
env=updated_env({"PREFIX": PREFIX_DIR}),
ver = "4.0.1"
mpdec_dir = working_dir / f"mpdecimal-{ver}"
shutil.rmtree(mpdec_dir, ignore_errors=True)
- download_and_unpack(working_dir, f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz", "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8")
+ download_and_unpack(
+ working_dir,
+ f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz",
+ "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
+ )
call(
[
"emconfigure",
quiet=context.quiet,
)
call(
- [
- "make",
- "install"
- ],
+ ["make", "install"],
cwd=mpdec_dir,
quiet=context.quiet,
)
@subdir(HOST_DIR, clean_ok=True)
def configure_emscripten_python(context, working_dir):
"""Configure the emscripten/host build."""
- config_site = os.fsdecode(
- EMSCRIPTEN_DIR / "config.site-wasm32-emscripten"
- )
+ config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten")
emscripten_build_dir = working_dir.relative_to(CHECKOUT)
python_build_dir = NATIVE_BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*"))
- assert (
- len(lib_dirs) == 1
- ), f"Expected a single lib.* directory in {python_build_dir}"
+ assert len(lib_dirs) == 1, (
+ f"Expected a single lib.* directory in {python_build_dir}"
+ )
lib_dir = os.fsdecode(lib_dirs[0])
pydebug = lib_dir.endswith("-pydebug")
python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
quiet=context.quiet,
)
- shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs")
+ shutil.copy(
+ EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs"
+ )
node_entry = working_dir / "node_entry.mjs"
exec_script = working_dir / "python.sh"
subcommands = parser.add_subparsers(dest="subcommand")
build = subcommands.add_parser("build", help="Build everything")
configure_build = subcommands.add_parser(
- "configure-build-python", help="Run `configure` for the " "build Python"
+ "configure-build-python", help="Run `configure` for the build Python"
)
make_mpdec_cmd = subcommands.add_parser(
- "make-mpdec", help="Clone mpdec repo, configure and build it for emscripten"
+ "make-mpdec",
+ help="Clone mpdec repo, configure and build it for emscripten",
)
make_libffi_cmd = subcommands.add_parser(
- "make-libffi", help="Clone libffi repo, configure and build it for emscripten"
+ "make-libffi",
+ help="Clone libffi repo, configure and build it for emscripten",
)
make_build = subcommands.add_parser(
"make-build-python", help="Run `make` for the build Python"
if not context.subcommand:
# No command provided, display help and exit
- print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr)
+ print(
+ "Expected one of",
+ ", ".join(sorted(dispatch.keys())),
+ file=sys.stderr,
+ )
parser.print_help(sys.stderr)
sys.exit(1)
dispatch[context.subcommand](context)
import contextlib
import functools
import os
+
try:
from os import process_cpu_count as cpu_count
except ImportError:
CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent
-assert (CHECKOUT / "configure").is_file(), "Please update the location of the file"
+assert (CHECKOUT / "configure").is_file(), (
+ "Please update the location of the file"
+)
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
# Build platform can also be found via `config.guess`.
BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
-LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n"
- "# Required to statically build extension modules.").encode("utf-8")
+LOCAL_SETUP_MARKER = (
+ "# Generated by Tools/wasm/wasi .\n"
+ "# Required to statically build extension modules."
+).encode("utf-8")
WASI_SDK_VERSION = 24
# https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try:
- epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
+ epoch = subprocess.check_output(
+ git_epoch_cmd, encoding="utf-8"
+ ).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError:
pass # Might be building from a tarball.
def subdir(working_dir, *, clean_ok=False):
"""Decorator to change to a working directory."""
+
def decorator(func):
@functools.wraps(func)
def wrapper(context):
if callable(working_dir):
working_dir = working_dir(context)
try:
- tput_output = subprocess.check_output(["tput", "cols"],
- encoding="utf-8")
+ tput_output = subprocess.check_output(
+ ["tput", "cols"], encoding="utf-8"
+ )
except subprocess.CalledProcessError:
terminal_width = 80
else:
terminal_width = int(tput_output.strip())
print("⎯" * terminal_width)
print("📁", working_dir)
- if (clean_ok and getattr(context, "clean", False) and
- working_dir.exists()):
+ if (
+ clean_ok
+ and getattr(context, "clean", False)
+ and working_dir.exists()
+ ):
print("🚮 Deleting directory (--clean)...")
shutil.rmtree(working_dir)
stdout = None
stderr = None
else:
- stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8",
- delete=False,
- dir=logdir,
- prefix="cpython-wasi-",
- suffix=".log")
+ stdout = tempfile.NamedTemporaryFile(
+ "w",
+ encoding="utf-8",
+ delete=False,
+ dir=logdir,
+ prefix="cpython-wasi-",
+ suffix=".log",
+ )
stderr = subprocess.STDOUT
print(f"📝 Logging output to {stdout.name} (--quiet)...")
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
- raise FileNotFoundError("Unable to find `python(.exe)` in "
- f"{BUILD_DIR}")
+ raise FileNotFoundError(
+ f"Unable to find `python(.exe)` in {BUILD_DIR}"
+ )
return binary
def build_python_is_pydebug():
"""Find out if the build Python is a pydebug build."""
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
- result = subprocess.run([build_python_path(), "-c", test],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ result = subprocess.run(
+ [build_python_path(), "-c", test],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
return bool(result.returncode)
print(f"📝 Creating {LOCAL_SETUP} ...")
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
- configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)]
+ configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
if context.args:
configure.extend(context.args)
@subdir(BUILD_DIR)
def make_build_python(context, working_dir):
"""Make/build the build Python."""
- call(["make", "--jobs", str(cpu_count()), "all"],
- context=context)
+ call(["make", "--jobs", str(cpu_count()), "all"], context=context)
binary = build_python_path()
- cmd = [binary, "-c",
- "import sys; "
- "print(f'{sys.version_info.major}.{sys.version_info.minor}')"]
+ cmd = [
+ binary,
+ "-c",
+ "import sys; "
+ "print(f'{sys.version_info.major}.{sys.version_info.minor}')",
+ ]
version = subprocess.check_output(cmd, encoding="utf-8").strip()
print(f"🎉 {binary} {version}")
# Starting with WASI SDK 23, the tarballs went from containing a directory named
# ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
# ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
- potential_sdks = [path for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*")
- if path.is_dir()]
+ potential_sdks = [
+ path
+ for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*")
+ if path.is_dir()
+ ]
if len(potential_sdks) == 1:
return potential_sdks[0]
elif (default_path := opt_path / "wasi-sdk").is_dir():
"""Calculate environment variables for building with wasi-sdk."""
wasi_sdk_path = context.wasi_sdk_path
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
- env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++",
- "AR": "llvm-ar", "RANLIB": "ranlib"}
+ env = {
+ "CC": "clang",
+ "CPP": "clang-cpp",
+ "CXX": "clang++",
+ "AR": "llvm-ar",
+ "RANLIB": "ranlib",
+ }
for env_var, binary_name in list(env.items()):
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
env["PKG_CONFIG_PATH"] = ""
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
- map(os.fsdecode,
- [sysroot / "lib" / "pkgconfig",
- sysroot / "share" / "pkgconfig"]))
+ map(
+ os.fsdecode,
+ [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
+ )
+ )
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
- env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"),
- os.environ["PATH"]])
+ env["PATH"] = os.pathsep.join([
+ os.fsdecode(wasi_sdk_path / "bin"),
+ os.environ["PATH"],
+ ])
return env
def configure_wasi_python(context, working_dir):
"""Configure the WASI/host build."""
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
- raise ValueError("WASI-SDK not found; "
- "download from "
- "https://github.com/WebAssembly/wasi-sdk and/or "
- "specify via $WASI_SDK_PATH or --wasi-sdk")
-
- config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi")
+ raise ValueError(
+ "WASI-SDK not found; "
+ "download from "
+ "https://github.com/WebAssembly/wasi-sdk and/or "
+ "specify via $WASI_SDK_PATH or --wasi-sdk"
+ )
+
+ config_site = os.fsdecode(
+ CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi"
+ )
wasi_build_dir = working_dir.relative_to(CHECKOUT)
python_build_dir = BUILD_DIR / "build"
lib_dirs = list(python_build_dir.glob("lib.*"))
- assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
+ assert len(lib_dirs) == 1, (
+ f"Expected a single lib.* directory in {python_build_dir}"
+ )
lib_dir = os.fsdecode(lib_dirs[0])
python_version = lib_dir.rpartition("-")[-1]
- sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
+ sysconfig_data_dir = (
+ f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
+ )
# Use PYTHONPATH to include sysconfig data which must be anchored to the
# WASI guest's `/` directory.
- args = {"GUEST_DIR": "/",
- "HOST_DIR": CHECKOUT,
- "ENV_VAR_NAME": "PYTHONPATH",
- "ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
- "PYTHON_WASM": working_dir / "python.wasm"}
+ args = {
+ "GUEST_DIR": "/",
+ "HOST_DIR": CHECKOUT,
+ "ENV_VAR_NAME": "PYTHONPATH",
+ "ENV_VAR_VALUE": f"/{sysconfig_data_dir}",
+ "PYTHON_WASM": working_dir / "python.wasm",
+ }
# Check dynamically for wasmtime in case it was specified manually via
# `--host-runner`.
if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
if wasmtime := shutil.which("wasmtime"):
args[WASMTIME_VAR_NAME] = wasmtime
else:
- raise FileNotFoundError("wasmtime not found; download from "
- "https://github.com/bytecodealliance/wasmtime")
+ raise FileNotFoundError(
+ "wasmtime not found; download from "
+ "https://github.com/bytecodealliance/wasmtime"
+ )
host_runner = context.host_runner.format_map(args)
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
build_python = os.fsdecode(build_python_path())
# The path to `configure` MUST be relative, else `python.wasm` is unable
# to find the stdlib due to Python not recognizing that it's being
# executed from within a checkout.
- configure = [os.path.relpath(CHECKOUT / 'configure', working_dir),
- f"--host={context.host_triple}",
- f"--build={BUILD_DIR.name}",
- f"--with-build-python={build_python}"]
+ configure = [
+ os.path.relpath(CHECKOUT / "configure", working_dir),
+ f"--host={context.host_triple}",
+ f"--build={BUILD_DIR.name}",
+ f"--with-build-python={build_python}",
+ ]
if build_python_is_pydebug():
configure.append("--with-pydebug")
if context.args:
configure.extend(context.args)
- call(configure,
- env=updated_env(env_additions | wasi_sdk_env(context)),
- context=context)
+ call(
+ configure,
+ env=updated_env(env_additions | wasi_sdk_env(context)),
+ context=context,
+ )
python_wasm = working_dir / "python.wasm"
exec_script = working_dir / "python.sh"
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
def make_wasi_python(context, working_dir):
"""Run `make` for the WASI/host build."""
- call(["make", "--jobs", str(cpu_count()), "all"],
- env=updated_env(),
- context=context)
+ call(
+ ["make", "--jobs", str(cpu_count()), "all"],
+ env=updated_env(),
+ context=context,
+ )
exec_script = working_dir / "python.sh"
call([exec_script, "--version"], quiet=False)
def build_all(context):
"""Build everything."""
- steps = [configure_build_python, make_build_python, configure_wasi_python,
- make_wasi_python]
+ steps = [
+ configure_build_python,
+ make_build_python,
+ configure_wasi_python,
+ make_wasi_python,
+ ]
for step in steps:
step(context)
+
def clean_contents(context):
"""Delete all files created by this script."""
if CROSS_BUILD_DIR.exists():
def main():
default_host_triple = "wasm32-wasip1"
default_wasi_sdk = find_wasi_sdk()
- default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run "
- # Make sure the stack size will work for a pydebug
- # build.
- # Use 16 MiB stack.
- "--wasm max-wasm-stack=16777216 "
- # Enable thread support; causes use of preview1.
- #"--wasm threads=y --wasi threads=y "
- # Map the checkout to / to load the stdlib from /Lib.
- "--dir {HOST_DIR}::{GUEST_DIR} "
- # Set PYTHONPATH to the sysconfig data.
- "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}")
+ default_host_runner = (
+ f"{WASMTIME_HOST_RUNNER_VAR} run "
+ # Make sure the stack size will work for a pydebug
+ # build.
+ # Use 16 MiB stack.
+ "--wasm max-wasm-stack=16777216 "
+ # Enable thread support; causes use of preview1.
+ # "--wasm threads=y --wasi threads=y "
+ # Map the checkout to / to load the stdlib from /Lib.
+ "--dir {HOST_DIR}::{GUEST_DIR} "
+ # Set PYTHONPATH to the sysconfig data.
+ "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}"
+ )
default_logdir = pathlib.Path(tempfile.gettempdir())
parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand")
build = subcommands.add_parser("build", help="Build everything")
- configure_build = subcommands.add_parser("configure-build-python",
- help="Run `configure` for the "
- "build Python")
- make_build = subcommands.add_parser("make-build-python",
- help="Run `make` for the build Python")
- configure_host = subcommands.add_parser("configure-host",
- help="Run `configure` for the "
- "host/WASI (pydebug builds "
- "are inferred from the build "
- "Python)")
- make_host = subcommands.add_parser("make-host",
- help="Run `make` for the host/WASI")
- subcommands.add_parser("clean", help="Delete files and directories "
- "created by this script")
- for subcommand in build, configure_build, make_build, configure_host, make_host:
- subcommand.add_argument("--quiet", action="store_true", default=False,
- dest="quiet",
- help="Redirect output from subprocesses to a log file")
- subcommand.add_argument("--logdir", type=pathlib.Path, default=default_logdir,
- help="Directory to store log files; "
- f"defaults to {default_logdir}")
+ configure_build = subcommands.add_parser(
+ "configure-build-python", help="Run `configure` for the build Python"
+ )
+ make_build = subcommands.add_parser(
+ "make-build-python", help="Run `make` for the build Python"
+ )
+ configure_host = subcommands.add_parser(
+ "configure-host",
+ help="Run `configure` for the "
+ "host/WASI (pydebug builds "
+ "are inferred from the build "
+ "Python)",
+ )
+ make_host = subcommands.add_parser(
+ "make-host", help="Run `make` for the host/WASI"
+ )
+ subcommands.add_parser(
+ "clean", help="Delete files and directories created by this script"
+ )
+ for subcommand in (
+ build,
+ configure_build,
+ make_build,
+ configure_host,
+ make_host,
+ ):
+ subcommand.add_argument(
+ "--quiet",
+ action="store_true",
+ default=False,
+ dest="quiet",
+ help="Redirect output from subprocesses to a log file",
+ )
+ subcommand.add_argument(
+ "--logdir",
+ type=pathlib.Path,
+ default=default_logdir,
+ help=f"Directory to store log files; defaults to {default_logdir}",
+ )
for subcommand in configure_build, configure_host:
- subcommand.add_argument("--clean", action="store_true", default=False,
- dest="clean",
- help="Delete any relevant directories before building")
+ subcommand.add_argument(
+ "--clean",
+ action="store_true",
+ default=False,
+ dest="clean",
+ help="Delete any relevant directories before building",
+ )
for subcommand in build, configure_build, configure_host:
- subcommand.add_argument("args", nargs="*",
- help="Extra arguments to pass to `configure`")
+ subcommand.add_argument(
+ "args", nargs="*", help="Extra arguments to pass to `configure`"
+ )
for subcommand in build, configure_host:
- subcommand.add_argument("--wasi-sdk", type=pathlib.Path,
- dest="wasi_sdk_path",
- default=default_wasi_sdk,
- help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}")
- subcommand.add_argument("--host-runner", action="store",
- default=default_host_runner, dest="host_runner",
- help="Command template for running the WASI host; defaults to "
- f"`{default_host_runner}`")
+ subcommand.add_argument(
+ "--wasi-sdk",
+ type=pathlib.Path,
+ dest="wasi_sdk_path",
+ default=default_wasi_sdk,
+ help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
+ )
+ subcommand.add_argument(
+ "--host-runner",
+ action="store",
+ default=default_host_runner,
+ dest="host_runner",
+ help="Command template for running the WASI host; defaults to "
+ f"`{default_host_runner}`",
+ )
for subcommand in build, configure_host, make_host:
- subcommand.add_argument("--host-triple", action="store",
- default=default_host_triple,
- help="The target triple for the WASI host build; "
- f"defaults to {default_host_triple}")
+ subcommand.add_argument(
+ "--host-triple",
+ action="store",
+ default=default_host_triple,
+ help="The target triple for the WASI host build; "
+ f"defaults to {default_host_triple}",
+ )
context = parser.parse_args()
context.init_dir = pathlib.Path().absolute()
- dispatch = {"configure-build-python": configure_build_python,
- "make-build-python": make_build_python,
- "configure-host": configure_wasi_python,
- "make-host": make_wasi_python,
- "build": build_all,
- "clean": clean_contents}
+ dispatch = {
+ "configure-build-python": configure_build_python,
+ "make-build-python": make_build_python,
+ "configure-host": configure_wasi_python,
+ "make-host": make_wasi_python,
+ "build": build_all,
+ "clean": clean_contents,
+ }
dispatch[context.subcommand](context)
-if __name__ == "__main__":
+if __name__ == "__main__":
main()
projects / thirdparty / Python / cpython.git / commitdiff
