PublicArchive/cpython
2
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2026-05-06 04:37:33 -04:00
Code Issues Packages Projects Releases Wiki Activity

Refactor Platforms/WASI/__main__.py for lazy importing and future new subcommands (#145404)

Browse Source 
Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com>
This commit is contained in:
Brett Cannon
2026-03-04 13:27:23 -08:00
committed by GitHub
parent ae208d5665
commit b63dc8abdf
4 changed files with 462 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+8
View File
@@ -14,6 +14,10 @@ repos:
name: Run Ruff (lint) on Lib/test/
args: [--exit-non-zero-on-fix]
files: ^Lib/test/
- id: ruff-check
name: Run Ruff (lint) on Platforms/WASI/
args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml]
files: ^Platforms/WASI/
- id: ruff-check
name: Run Ruff (lint) on Tools/build/
args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
@@ -42,6 +46,10 @@ repos:
name: Run Ruff (format) on Doc/
args: [--exit-non-zero-on-fix]
files: ^Doc/
- id: ruff-format
name: Run Ruff (format) on Platforms/WASI/
args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml]
files: ^Platforms/WASI/
- id: ruff-format
name: Run Ruff (format) on Tools/build/check_warnings.py
args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
Platforms/WASI/.ruff.toml
+2
View File
@@ -1,5 +1,7 @@
extend = "../../.ruff.toml" # Inherit the project-wide settings
target-version = "py314"
[format]
preview = true
docstring-code-format = true
Platforms/WASI/__main__.py
+39 -424
View File
@@ -1,417 +1,23 @@
#!/usr/bin/env python3
__lazy_modules__ = ["_build"]
import argparse
import contextlib
import functools
import os
import tomllib
try:
from os import process_cpu_count as cpu_count
except ImportError:
from os import cpu_count
import pathlib
import shutil
import subprocess
import sys
import sysconfig
import tempfile
CHECKOUT = HERE = pathlib.Path(__file__).parent
import _build
while CHECKOUT != CHECKOUT.parent:
if (CHECKOUT / "configure").is_file():
break
CHECKOUT = CHECKOUT.parent
else:
raise FileNotFoundError(
"Unable to find the root of the CPython checkout by looking for 'configure'"
)
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
# Build platform can also be found via `config.guess`.
BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = (
b"# Generated by Platforms/WASI .\n"
b"# Required to statically build extension modules."
)
WASI_SDK_VERSION = 29
WASMTIME_VAR_NAME = "WASMTIME"
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
def separator():
"""Print a separator line across the terminal width."""
try:
tput_output = subprocess.check_output(
["tput", "cols"], encoding="utf-8"
)
except subprocess.CalledProcessError:
terminal_width = 80
else:
terminal_width = int(tput_output.strip())
print("⎯" * terminal_width)
def log(emoji, message, *, spacing=None):
"""Print a notification with an emoji.
If 'spacing' is None, calculate the spacing based on the number of code points
in the emoji as terminals "eat" a space when the emoji has multiple code points.
"""
if spacing is None:
spacing = " " if len(emoji) == 1 else " "
print("".join([emoji, spacing, message]))
def updated_env(updates={}):
"""Create a new dict representing the environment to use.
The changes made to the execution environment are printed out.
"""
env_defaults = {}
# https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try:
epoch = subprocess.check_output(
git_epoch_cmd, encoding="utf-8"
).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError:
pass # Might be building from a tarball.
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
environment = env_defaults | os.environ | updates
env_diff = {}
for key, value in environment.items():
if os.environ.get(key) != value:
env_diff[key] = value
env_vars = (
f"\n {key}={item}" for key, item in sorted(env_diff.items())
)
log("🌎", f"Environment changes:{''.join(env_vars)}")
return environment
def subdir(working_dir, *, clean_ok=False):
"""Decorator to change to a working directory."""
def decorator(func):
@functools.wraps(func)
def wrapper(context):
nonlocal working_dir
if callable(working_dir):
working_dir = working_dir(context)
separator()
log("📁", os.fsdecode(working_dir))
if (
clean_ok
and getattr(context, "clean", False)
and working_dir.exists()
):
log("🚮", "Deleting directory (--clean)...")
shutil.rmtree(working_dir)
working_dir.mkdir(parents=True, exist_ok=True)
with contextlib.chdir(working_dir):
return func(context, working_dir)
return wrapper
return decorator
def call(command, *, context=None, quiet=False, logdir=None, **kwargs):
"""Execute a command.
If 'quiet' is true, then redirect stdout and stderr to a temporary file.
"""
if context is not None:
quiet = context.quiet
logdir = context.logdir
elif quiet and logdir is None:
raise ValueError("When quiet is True, logdir must be specified")
log("❯", " ".join(map(str, command)), spacing=" ")
if not quiet:
stdout = None
stderr = None
else:
stdout = tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
delete=False,
dir=logdir,
prefix="cpython-wasi-",
suffix=".log",
)
stderr = subprocess.STDOUT
log("📝", f"Logging output to {stdout.name} (--quiet)...")
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
def build_python_path():
"""The path to the build Python binary."""
binary = BUILD_DIR / "python"
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
raise FileNotFoundError(
f"Unable to find `python(.exe)` in {BUILD_DIR}"
)
return binary
def build_python_is_pydebug():
"""Find out if the build Python is a pydebug build."""
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
result = subprocess.run(
[build_python_path(), "-c", test],
capture_output=True,
)
return bool(result.returncode)
@subdir(BUILD_DIR, clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
if LOCAL_SETUP.exists():
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
log("👍", f"{LOCAL_SETUP} exists ...")
else:
log("⚠️", f"{LOCAL_SETUP} exists, but has unexpected contents")
else:
log("📝", f"Creating {LOCAL_SETUP} ...")
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
if context.args:
configure.extend(context.args)
call(configure, context=context)
@subdir(BUILD_DIR)
def make_build_python(context, working_dir):
"""Make/build the build Python."""
call(["make", "--jobs", str(cpu_count()), "all"], context=context)
binary = build_python_path()
cmd = [
binary,
"-c",
"import sys; "
"print(f'{sys.version_info.major}.{sys.version_info.minor}')",
]
version = subprocess.check_output(cmd, encoding="utf-8").strip()
log("🎉", f"{binary} {version}")
def find_wasi_sdk(config):
"""Find the path to the WASI SDK."""
wasi_sdk_path = None
wasi_sdk_version = config["targets"]["wasi-sdk"]
if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
else:
opt_path = pathlib.Path("/opt")
# WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
# has said they don't plan to ever do a point release and all of their Git tags
# lack the ``.0`` suffix.
# Starting with WASI SDK 23, the tarballs went from containing a directory named
# ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
# ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
potential_sdks = [
path
for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*")
if path.is_dir()
]
if len(potential_sdks) == 1:
wasi_sdk_path = potential_sdks[0]
elif (default_path := opt_path / "wasi-sdk").is_dir():
wasi_sdk_path = default_path
# Starting with WASI SDK 25, a VERSION file is included in the root
# of the SDK directory that we can read to warn folks when they are using
# an unsupported version.
if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
version_details = version_file.read_text(encoding="utf-8")
found_version = version_details.splitlines()[0]
# Make sure there's a trailing dot to avoid false positives if somehow the
# supported version is a prefix of the found version (e.g. `25` and `2567`).
if not found_version.startswith(f"{wasi_sdk_version}."):
major_version = found_version.partition(".")[0]
log(
"⚠️",
f" Found WASI SDK {major_version}, "
f"but WASI SDK {wasi_sdk_version} is the supported version",
)
return wasi_sdk_path
def wasi_sdk_env(context):
"""Calculate environment variables for building with wasi-sdk."""
wasi_sdk_path = context.wasi_sdk_path
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
env = {
"CC": "clang",
"CPP": "clang-cpp",
"CXX": "clang++",
"AR": "llvm-ar",
"RANLIB": "ranlib",
}
for env_var, binary_name in list(env.items()):
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
if not wasi_sdk_path.name.startswith("wasi-sdk"):
for compiler in ["CC", "CPP", "CXX"]:
env[compiler] += f" --sysroot={sysroot}"
env["PKG_CONFIG_PATH"] = ""
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
map(
os.fsdecode,
[sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
)
)
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
env["PATH"] = os.pathsep.join([
os.fsdecode(wasi_sdk_path / "bin"),
os.environ["PATH"],
])
return env
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
def configure_wasi_python(context, working_dir):
"""Configure the WASI/host build."""
if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
raise ValueError(
"WASI-SDK not found; "
"download from "
"https://github.com/WebAssembly/wasi-sdk and/or "
"specify via $WASI_SDK_PATH or --wasi-sdk"
)
config_site = os.fsdecode(HERE / "config.site-wasm32-wasi")
wasi_build_dir = working_dir.relative_to(CHECKOUT)
args = {
"ARGV0": f"/{wasi_build_dir}/python.wasm",
"PYTHON_WASM": working_dir / "python.wasm",
}
# Check dynamically for wasmtime in case it was specified manually via
# `--host-runner`.
if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
if wasmtime := shutil.which("wasmtime"):
args[WASMTIME_VAR_NAME] = wasmtime
else:
raise FileNotFoundError(
"wasmtime not found; download from "
"https://github.com/bytecodealliance/wasmtime"
)
host_runner = context.host_runner.format_map(args)
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
build_python = os.fsdecode(build_python_path())
# The path to `configure` MUST be relative, else `python.wasm` is unable
# to find the stdlib due to Python not recognizing that it's being
# executed from within a checkout.
configure = [
os.path.relpath(CHECKOUT / "configure", working_dir),
f"--host={context.host_triple}",
f"--build={BUILD_DIR.name}",
f"--with-build-python={build_python}",
]
if build_python_is_pydebug():
configure.append("--with-pydebug")
if context.args:
configure.extend(context.args)
call(
configure,
env=updated_env(env_additions | wasi_sdk_env(context)),
context=context,
)
python_wasm = working_dir / "python.wasm"
exec_script = working_dir / "python.sh"
with exec_script.open("w", encoding="utf-8") as file:
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
exec_script.chmod(0o755)
log("🏃", f"Created {exec_script} (--host-runner)... ")
sys.stdout.flush()
@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
def make_wasi_python(context, working_dir):
"""Run `make` for the WASI/host build."""
call(
["make", "--jobs", str(cpu_count()), "all"],
env=updated_env(),
context=context,
)
exec_script = working_dir / "python.sh"
call([exec_script, "--version"], quiet=False)
log(
"🎉",
f"Use `{exec_script.relative_to(context.init_dir)}` "
"to run CPython w/ the WASI host specified by --host-runner",
)
def clean_contents(context):
"""Delete all files created by this script."""
if CROSS_BUILD_DIR.exists():
log("🧹", f"Deleting {CROSS_BUILD_DIR} ...")
shutil.rmtree(CROSS_BUILD_DIR)
if LOCAL_SETUP.exists():
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
log("🧹", f"Deleting generated {LOCAL_SETUP} ...")
def build_steps(*steps):
"""Construct a command from other steps."""
def builder(context):
for step in steps:
step(context)
return builder
HERE = pathlib.Path(__file__).parent
def main():
with (HERE / "config.toml").open("rb") as file:
config = tomllib.load(file)
default_wasi_sdk = find_wasi_sdk(config)
default_host_triple = config["targets"]["host-triple"]
default_host_runner = (
f"{WASMTIME_HOST_RUNNER_VAR} run "
# Set argv0 so that getpath.py can auto-discover the sysconfig data directory
"{WASMTIME} run "
# Set argv0 so that getpath.py can auto-discover the sysconfig data directory.
"--argv0 {ARGV0} "
# Map the checkout to / to load the stdlib from /Lib.
f"--dir {os.fsdecode(CHECKOUT)}::/ "
"--dir {CHECKOUT}::/ "
# Flags involving --optimize, --codegen, --debug, --wasm, and --wasi can be kept
# in a config file.
# We are using such a file to act as defaults in case a user wants to override
@@ -419,9 +25,8 @@ def main():
# post-build so that they immediately apply to the Makefile instead of having to
# regenerate it, and allow for easy copying of the settings for anyone else who
# may want to use them.
f"--config {os.fsdecode(HERE / 'wasmtime.toml')}"
"--config {WASMTIME_CONFIG_PATH}"
)
default_logdir = pathlib.Path(tempfile.gettempdir())
parser = argparse.ArgumentParser()
subcommands = parser.add_subparsers(dest="subcommand")
@@ -470,8 +75,8 @@ def main():
subcommand.add_argument(
"--logdir",
type=pathlib.Path,
default=default_logdir,
help=f"Directory to store log files; defaults to {default_logdir}",
default=None,
help="Directory to store log files",
)
for subcommand in (
configure_build,
@@ -501,8 +106,9 @@ def main():
"--wasi-sdk",
type=pathlib.Path,
dest="wasi_sdk_path",
default=default_wasi_sdk,
help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}",
default=None,
help="Path to the WASI SDK; defaults to WASI_SDK_PATH environment variable "
"or the appropriate version found in /opt",
)
subcommand.add_argument(
"--host-runner",
@@ -516,28 +122,37 @@ def main():
subcommand.add_argument(
"--host-triple",
action="store",
default=default_host_triple,
default=None,
help="The target triple for the WASI host build; "
f"defaults to {default_host_triple}",
f"defaults to the value found in {os.fsdecode(HERE / 'config.toml')}",
)
context = parser.parse_args()
context.init_dir = pathlib.Path().absolute()
build_build_python = build_steps(configure_build_python, make_build_python)
build_wasi_python = build_steps(configure_wasi_python, make_wasi_python)
dispatch = {
"configure-build-python": configure_build_python,
"make-build-python": make_build_python,
"build-python": build_build_python,
"configure-host": configure_wasi_python,
"make-host": make_wasi_python,
"build-host": build_wasi_python,
"build": build_steps(build_build_python, build_wasi_python),
"clean": clean_contents,
}
dispatch[context.subcommand](context)
match context.subcommand:
case "configure-build-python":
_build.configure_build_python(context)
case "make-build-python":
_build.make_build_python(context)
case "build-python":
_build.configure_build_python(context)
_build.make_build_python(context)
case "configure-host":
_build.configure_wasi_python(context)
case "make-host":
_build.make_wasi_python(context)
case "build-host":
_build.configure_wasi_python(context)
_build.make_wasi_python(context)
case "build":
_build.configure_build_python(context)
_build.make_build_python(context)
_build.configure_wasi_python(context)
_build.make_wasi_python(context)
case "clean":
_build.clean_contents(context)
case _:
raise ValueError(f"Unknown subcommand {context.subcommand!r}")
if __name__ == "__main__":
Platforms/WASI/_build.py
+413
View File
@@ -0,0 +1,413 @@
#!/usr/bin/env python3
__lazy_modules__ = ["shutil", "sys", "tempfile", "tomllib"]
import contextlib
import functools
import os
import pathlib
import shutil
import subprocess
import sys
import sysconfig
import tempfile
import tomllib
try:
from os import process_cpu_count as cpu_count
except ImportError:
from os import cpu_count
CHECKOUT = HERE = pathlib.Path(__file__).parent
while CHECKOUT != CHECKOUT.parent:
if (CHECKOUT / "configure").is_file():
break
CHECKOUT = CHECKOUT.parent
else:
raise FileNotFoundError(
"Unable to find the root of the CPython checkout by looking for 'configure'"
)
CROSS_BUILD_DIR = CHECKOUT / "cross-build"
# Build platform can also be found via `config.guess`.
BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE")
LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
LOCAL_SETUP_MARKER = (
b"# Generated by Platforms/WASI .\n"
b"# Required to statically build extension modules."
)
WASMTIME_VAR_NAME = "WASMTIME"
WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
def separator():
"""Print a separator line across the terminal width."""
try:
tput_output = subprocess.check_output(
["tput", "cols"], encoding="utf-8"
)
except subprocess.CalledProcessError:
terminal_width = 80
else:
terminal_width = int(tput_output.strip())
print("⎯" * terminal_width)
def log(emoji, message, *, spacing=None):
"""Print a notification with an emoji.
If 'spacing' is None, calculate the spacing based on the number of code points
in the emoji as terminals "eat" a space when the emoji has multiple code points.
"""
if spacing is None:
spacing = " " if len(emoji) == 1 else " "
print("".join([emoji, spacing, message]))
def updated_env(updates={}):
"""Create a new dict representing the environment to use.
The changes made to the execution environment are printed out.
"""
env_defaults = {}
# https://reproducible-builds.org/docs/source-date-epoch/
git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
try:
epoch = subprocess.check_output(
git_epoch_cmd, encoding="utf-8"
).strip()
env_defaults["SOURCE_DATE_EPOCH"] = epoch
except subprocess.CalledProcessError:
pass # Might be building from a tarball.
# This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
environment = env_defaults | os.environ | updates
env_diff = {}
for key, value in environment.items():
if os.environ.get(key) != value:
env_diff[key] = value
env_vars = [
f"\n {key}={item}" for key, item in sorted(env_diff.items())
]
log("🌎", f"Environment changes:{''.join(env_vars)}")
return environment
def subdir(working_dir, *, clean_ok=False):
"""Decorator to change to a working directory."""
def decorator(func):
@functools.wraps(func)
def wrapper(context):
nonlocal working_dir
if callable(working_dir):
working_dir = working_dir(context)
separator()
log("📁", os.fsdecode(working_dir))
if (
clean_ok
and getattr(context, "clean", False)
and working_dir.exists()
):
log("🚮", "Deleting directory (--clean)...")
shutil.rmtree(working_dir)
working_dir.mkdir(parents=True, exist_ok=True)
with contextlib.chdir(working_dir):
return func(context, working_dir)
return wrapper
return decorator
def call(command, *, context=None, quiet=False, **kwargs):
"""Execute a command.
If 'quiet' is true, then redirect stdout and stderr to a temporary file.
"""
if context is not None:
quiet = context.quiet
log("❯", " ".join(map(str, command)), spacing=" ")
if not quiet:
stdout = None
stderr = None
else:
if (logdir := getattr(context, "logdir", None)) is None:
logdir = pathlib.Path(tempfile.gettempdir())
stdout = tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
delete=False,
dir=logdir,
prefix="cpython-wasi-",
suffix=".log",
)
stderr = subprocess.STDOUT
log("📝", f"Logging output to {stdout.name} (--quiet)...")
subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
def build_python_path():
"""The path to the build Python binary."""
binary = BUILD_DIR / "python"
if not binary.is_file():
binary = binary.with_suffix(".exe")
if not binary.is_file():
raise FileNotFoundError(
f"Unable to find `python(.exe)` in {BUILD_DIR}"
)
return binary
def build_python_is_pydebug():
"""Find out if the build Python is a pydebug build."""
test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)"
result = subprocess.run(
[build_python_path(), "-c", test],
capture_output=True,
)
return bool(result.returncode)
@subdir(BUILD_DIR, clean_ok=True)
def configure_build_python(context, working_dir):
"""Configure the build/host Python."""
if LOCAL_SETUP.exists():
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
log("👍", f"{LOCAL_SETUP} exists ...")
else:
log("⚠️", f"{LOCAL_SETUP} exists, but has unexpected contents")
else:
log("📝", f"Creating {LOCAL_SETUP} ...")
LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
configure = [os.path.relpath(CHECKOUT / "configure", working_dir)]
if context.args:
configure.extend(context.args)
call(configure, context=context)
@subdir(BUILD_DIR)
def make_build_python(context, working_dir):
"""Make/build the build Python."""
call(["make", "--jobs", str(cpu_count()), "all"], context=context)
binary = build_python_path()
cmd = [
binary,
"-c",
"import sys; "
"print(f'{sys.version_info.major}.{sys.version_info.minor}')",
]
version = subprocess.check_output(cmd, encoding="utf-8").strip()
log("🎉", f"{binary} {version}")
def wasi_sdk(context):
"""Find the path to the WASI SDK."""
if wasi_sdk_path := context.wasi_sdk_path:
if not wasi_sdk_path.exists():
raise ValueError(
"WASI SDK not found; "
"download from "
"https://github.com/WebAssembly/wasi-sdk and/or "
"specify via $WASI_SDK_PATH or --wasi-sdk"
)
return wasi_sdk_path
with (HERE / "config.toml").open("rb") as file:
config = tomllib.load(file)
wasi_sdk_version = config["targets"]["wasi-sdk"]
if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"):
wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var)
if not wasi_sdk_path.exists():
raise ValueError(
f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})"
)
else:
opt_path = pathlib.Path("/opt")
# WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team
# has said they don't plan to ever do a point release and all of their Git tags
# lack the ``.0`` suffix.
# Starting with WASI SDK 23, the tarballs went from containing a directory named
# ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g.
# ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``.
potential_sdks = [
path
for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*")
if path.is_dir()
]
if len(potential_sdks) == 1:
wasi_sdk_path = potential_sdks[0]
elif (default_path := opt_path / "wasi-sdk").is_dir():
wasi_sdk_path = default_path
# Starting with WASI SDK 25, a VERSION file is included in the root
# of the SDK directory that we can read to warn folks when they are using
# an unsupported version.
if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file():
version_details = version_file.read_text(encoding="utf-8")
found_version = version_details.splitlines()[0]
# Make sure there's a trailing dot to avoid false positives if somehow the
# supported version is a prefix of the found version (e.g. `25` and `2567`).
if not found_version.startswith(f"{wasi_sdk_version}."):
major_version = found_version.partition(".")[0]
log(
"⚠️",
f" Found WASI SDK {major_version}, "
f"but WASI SDK {wasi_sdk_version} is the supported version",
)
# Cache the result.
context.wasi_sdk_path = wasi_sdk_path
return wasi_sdk_path
def wasi_sdk_env(context):
"""Calculate environment variables for building with wasi-sdk."""
wasi_sdk_path = wasi_sdk(context)
sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
env = {
"CC": "clang",
"CPP": "clang-cpp",
"CXX": "clang++",
"AR": "llvm-ar",
"RANLIB": "ranlib",
}
for env_var, binary_name in list(env.items()):
env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
if not wasi_sdk_path.name.startswith("wasi-sdk"):
for compiler in ["CC", "CPP", "CXX"]:
env[compiler] += f" --sysroot={sysroot}"
env["PKG_CONFIG_PATH"] = ""
env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
map(
os.fsdecode,
[sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"],
)
)
env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
env["WASI_SYSROOT"] = os.fsdecode(sysroot)
env["PATH"] = os.pathsep.join([
os.fsdecode(wasi_sdk_path / "bin"),
os.environ["PATH"],
])
return env
def host_triple(context):
"""Determine the target triple for the WASI host build."""
if context.host_triple:
return context.host_triple
with (HERE / "config.toml").open("rb") as file:
config = tomllib.load(file)
# Cache the result.
context.host_triple = config["targets"]["host-triple"]
return context.host_triple
@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context), clean_ok=True)
def configure_wasi_python(context, working_dir):
"""Configure the WASI/host build."""
config_site = os.fsdecode(HERE / "config.site-wasm32-wasi")
wasi_build_dir = working_dir.relative_to(CHECKOUT)
args = {
"WASMTIME": "wasmtime",
"ARGV0": f"/{wasi_build_dir}/python.wasm",
"CHECKOUT": os.fsdecode(CHECKOUT),
"WASMTIME_CONFIG_PATH": os.fsdecode(HERE / "wasmtime.toml"),
}
# Check dynamically for wasmtime in case it was specified manually via
# `--host-runner`.
if "{WASMTIME}" in context.host_runner:
if wasmtime := shutil.which("wasmtime"):
args["WASMTIME"] = wasmtime
else:
raise FileNotFoundError(
"wasmtime not found; download from "
"https://github.com/bytecodealliance/wasmtime"
)
host_runner = context.host_runner.format_map(args)
env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
build_python = os.fsdecode(build_python_path())
# The path to `configure` MUST be relative, else `python.wasm` is unable
# to find the stdlib due to Python not recognizing that it's being
# executed from within a checkout.
configure = [
os.path.relpath(CHECKOUT / "configure", working_dir),
f"--host={host_triple(context)}",
f"--build={BUILD_DIR.name}",
f"--with-build-python={build_python}",
]
if build_python_is_pydebug():
configure.append("--with-pydebug")
if context.args:
configure.extend(context.args)
call(
configure,
env=updated_env(env_additions | wasi_sdk_env(context)),
context=context,
)
python_wasm = working_dir / "python.wasm"
exec_script = working_dir / "python.sh"
with exec_script.open("w", encoding="utf-8") as file:
file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
exec_script.chmod(0o755)
log("🏃", f"Created {exec_script} (--host-runner)... ")
sys.stdout.flush()
@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context))
def make_wasi_python(context, working_dir):
"""Run `make` for the WASI/host build."""
call(
["make", "--jobs", str(cpu_count()), "all"],
env=updated_env(),
context=context,
)
exec_script = working_dir / "python.sh"
call([exec_script, "--version"], quiet=False)
log(
"🎉",
f"Use `{exec_script.relative_to(pathlib.Path().absolute())}` "
"to run CPython w/ the WASI host specified by --host-runner",
)
def clean_contents(context):
"""Delete all files created by this script."""
if CROSS_BUILD_DIR.exists():
log("🧹", f"Deleting {CROSS_BUILD_DIR} ...")
shutil.rmtree(CROSS_BUILD_DIR)
if LOCAL_SETUP.exists():
if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER:
log("🧹", f"Deleting generated {LOCAL_SETUP} ...")