* _add_python_opts() now handles cross compilation and HOSTRUNNER.
* display_header() now tells if Python is cross-compiled, display
HOSTRUNNER, and get the host platform.
* Remove Tools/scripts/run_tests.py script.
* Remove "make hostrunnertest": use "make buildbottest"
or "make test" instead.
import re
import shlex
import sys
+import sysconfig
import time
from test import support
strip_py_suffix, count, format_duration,
printlist, get_temp_dir, get_work_dir, exit_timeout,
display_header, cleanup_temp_dir, print_warning,
+ is_cross_compiled, get_host_runner,
MS_WINDOWS, EXIT_TIMEOUT)
self.want_rerun: bool = ns.rerun
self.want_run_leaks: bool = ns.runleaks
- ci_mode = (ns.fast_ci or ns.slow_ci)
+ self.ci_mode: bool = (ns.fast_ci or ns.slow_ci)
self.want_add_python_opts: bool = (_add_python_opts
- and ns._add_python_opts
- and ci_mode)
+ and ns._add_python_opts)
# Select tests
if ns.match_tests:
if (self.want_header
or not(self.pgo or self.quiet or self.single_test_run
or tests or self.cmdline_args)):
- display_header(self.use_resources)
+ display_header(self.use_resources, self.python_cmd)
if self.randomize:
print("Using random seed", self.random_seed)
# processes.
return self._run_tests(selected, tests)
- def _add_python_opts(self):
- python_opts = []
+ def _add_cross_compile_opts(self, regrtest_opts):
+ # WASM/WASI buildbot builders pass multiple PYTHON environment
+ # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
+ keep_environ = bool(self.python_cmd)
+ environ = None
+
+ # Are we using cross-compilation?
+ cross_compile = is_cross_compiled()
+
+ # Get HOSTRUNNER
+ hostrunner = get_host_runner()
+
+ if cross_compile:
+ # emulate -E, but keep PYTHONPATH + cross compile env vars,
+ # so test executable can load correct sysconfigdata file.
+ keep = {
+ '_PYTHON_PROJECT_BASE',
+ '_PYTHON_HOST_PLATFORM',
+ '_PYTHON_SYSCONFIGDATA_NAME',
+ 'PYTHONPATH'
+ }
+ old_environ = os.environ
+ new_environ = {
+ name: value for name, value in os.environ.items()
+ if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
+ }
+ # Only set environ if at least one variable was removed
+ if new_environ != old_environ:
+ environ = new_environ
+ keep_environ = True
+
+ if cross_compile and hostrunner:
+ if self.num_workers == 0:
+ # For now use only two cores for cross-compiled builds;
+ # hostrunner can be expensive.
+ regrtest_opts.extend(['-j', '2'])
+
+ # If HOSTRUNNER is set and -p/--python option is not given, then
+ # use hostrunner to execute python binary for tests.
+ if not self.python_cmd:
+ buildpython = sysconfig.get_config_var("BUILDPYTHON")
+ python_cmd = f"{hostrunner} {buildpython}"
+ regrtest_opts.extend(["--python", python_cmd])
+ keep_environ = True
+
+ return (environ, keep_environ)
+
+ def _add_ci_python_opts(self, python_opts, keep_environ):
+ # --fast-ci and --slow-ci add options to Python:
+ # "-u -W default -bb -E"
# Unbuffered stdout and stderr
if not sys.stdout.write_through:
if sys.flags.bytes_warning < 2:
python_opts.append('-bb')
- # WASM/WASI buildbot builders pass multiple PYTHON environment
- # variables such as PYTHONPATH and _PYTHON_HOSTRUNNER.
- if not self.python_cmd:
+ if not keep_environ:
# Ignore PYTHON* environment variables
if not sys.flags.ignore_environment:
python_opts.append('-E')
- if not python_opts:
- return
-
- cmd = [*sys.orig_argv, "--dont-add-python-opts"]
- cmd[1:1] = python_opts
-
+ def _execute_python(self, cmd, environ):
# Make sure that messages before execv() are logged
sys.stdout.flush()
sys.stderr.flush()
cmd_text = shlex.join(cmd)
try:
+ print(f"+ {cmd_text}", flush=True)
+
if hasattr(os, 'execv') and not MS_WINDOWS:
os.execv(cmd[0], cmd)
# On success, execv() do no return.
# On error, it raises an OSError.
else:
import subprocess
- with subprocess.Popen(cmd) as proc:
+ with subprocess.Popen(cmd, env=environ) as proc:
try:
proc.wait()
except KeyboardInterrupt:
f"Command: {cmd_text}")
# continue executing main()
+ def _add_python_opts(self):
+ python_opts = []
+ regrtest_opts = []
+
+ environ, keep_environ = self._add_cross_compile_opts(regrtest_opts)
+ if self.ci_mode:
+ self._add_ci_python_opts(python_opts, keep_environ)
+
+ if (not python_opts) and (not regrtest_opts) and (environ is None):
+ # Nothing changed: nothing to do
+ return
+
+ # Create new command line
+ cmd = list(sys.orig_argv)
+ if python_opts:
+ cmd[1:1] = python_opts
+ if regrtest_opts:
+ cmd.extend(regrtest_opts)
+ cmd.append("--dont-add-python-opts")
+
+ self._execute_python(cmd, environ)
+
def _init(self):
# Set sys.stdout encoder error handler to backslashreplace,
# similar to sys.stderr error handler, to avoid UnicodeEncodeError
import os.path
import platform
import random
+import shlex
import signal
+import subprocess
import sys
import sysconfig
import tempfile
f"{new_fd_limit}: {err}.")
-def display_header(use_resources: tuple[str, ...]):
+def get_host_runner():
+ if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
+ hostrunner = sysconfig.get_config_var("HOSTRUNNER")
+ return hostrunner
+
+
+def is_cross_compiled():
+ return ('_PYTHON_HOST_PLATFORM' in os.environ)
+
+
+def display_header(use_resources: tuple[str, ...],
+ python_cmd: tuple[str, ...] | None):
# Print basic platform information
print("==", platform.python_implementation(), *sys.version.split())
print("==", platform.platform(aliased=True),
print("== encodings: locale=%s, FS=%s"
% (locale.getencoding(), sys.getfilesystemencoding()))
-
if use_resources:
print(f"== resources ({len(use_resources)}): "
f"{', '.join(sorted(use_resources))}")
else:
print("== resources: (all disabled, use -u option)")
+ cross_compile = is_cross_compiled()
+ if cross_compile:
+ print("== cross compiled: Yes")
+ if python_cmd:
+ cmd = shlex.join(python_cmd)
+ print(f"== host python: {cmd}")
+
+ get_cmd = [*python_cmd, '-m', 'platform']
+ proc = subprocess.run(
+ get_cmd,
+ stdout=subprocess.PIPE,
+ text=True,
+ cwd=os_helper.SAVEDCWD)
+ stdout = proc.stdout.replace('\n', ' ').strip()
+ if stdout:
+ print(f"== host platform: {stdout}")
+ elif proc.returncode:
+ print(f"== host platform: <command failed with exit code {proc.returncode}>")
+ else:
+ hostrunner = get_host_runner()
+ if hostrunner:
+ print(f"== host runner: {hostrunner}")
+
# This makes it easier to remember what to set in your local
# environment when trying to reproduce a sanitizer failure.
asan = support.check_sanitizer(address=True)
args = [*self.python_args, script, *self.regrtest_args, *self.tests]
self.run_tests(args)
- @unittest.skipUnless(sysconfig.is_python_build(),
- 'run_tests.py script is not installed')
- def test_tools_script_run_tests(self):
- # Tools/scripts/run_tests.py
- script = os.path.join(ROOT_DIR, 'Tools', 'scripts', 'run_tests.py')
- args = [script, *self.regrtest_args, *self.tests]
- self.run_tests(args)
-
def run_batch(self, *args):
proc = self.run_command(args)
self.check_output(proc.stdout)
TESTOPTS= $(EXTRATESTOPTS)
TESTPYTHON= $(RUNSHARED) $(PYTHON_FOR_BUILD) $(TESTPYTHONOPTS)
-TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
+TESTRUNNER= $(TESTPYTHON) -m test
TESTTIMEOUT=
# Remove "test_python_*" directories of previous failed test jobs.
fi
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
-# Like buildbottest, but run Python tests with HOSTRUNNER directly.
-.PHONY: hostrunnertest
-hostrunnertest: all
- $(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
-
.PHONY: pythoninfo
pythoninfo: all
$(RUNSHARED) $(HOSTRUNNER) ./$(BUILDPYTHON) -m test.pythoninfo
--- /dev/null
+Remove ``Tools/scripts/run_tests.py`` and ``make hostrunnertest``. Just run
+``./python -m test --slow-ci``, ``make buildbottest`` or ``make test`` instead.
+Python test runner (regrtest) now handles cross-compilation and HOSTRUNNER. It
+also adds options to Python such fast ``-u -E -W default -bb`` when
+``--fast-ci`` or ``--slow-ci`` option is used. Patch by Victor Stinner.
+++ /dev/null
-"""Run Python's test suite in a fast, rigorous way.
-
-The defaults are meant to be reasonably thorough, while skipping certain
-tests that can be time-consuming or resource-intensive (e.g. largefile),
-or distracting (e.g. audio and gui). These defaults can be overridden by
-simply passing a -u option to this script.
-
-"""
-
-import os
-import shlex
-import sys
-import sysconfig
-import test.support
-
-
-def is_multiprocess_flag(arg):
- return arg.startswith('-j') or arg.startswith('--multiprocess')
-
-
-def is_python_flag(arg):
- return arg.startswith('-p') or arg.startswith('--python')
-
-
-def main(regrtest_args):
- args = [sys.executable]
-
- cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
- if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
- hostrunner = sysconfig.get_config_var("HOSTRUNNER")
- if cross_compile:
- # emulate -E, but keep PYTHONPATH + cross compile env vars, so
- # test executable can load correct sysconfigdata file.
- keep = {
- '_PYTHON_PROJECT_BASE',
- '_PYTHON_HOST_PLATFORM',
- '_PYTHON_SYSCONFIGDATA_NAME',
- 'PYTHONPATH'
- }
- environ = {
- name: value for name, value in os.environ.items()
- if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
- }
- else:
- environ = os.environ.copy()
-
- # Allow user-specified interpreter options to override our defaults.
- args.extend(test.support.args_from_interpreter_flags())
-
- args.extend(['-m', 'test', # Run the test suite
- '--fast-ci', # Fast Continuous Integration mode
- ])
- if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
- if cross_compile and hostrunner:
- # For now use only two cores for cross-compiled builds;
- # hostrunner can be expensive.
- args.extend(['-j', '2'])
-
- if cross_compile and hostrunner:
- # If HOSTRUNNER is set and -p/--python option is not given, then
- # use hostrunner to execute python binary for tests.
- if not any(is_python_flag(arg) for arg in regrtest_args):
- buildpython = sysconfig.get_config_var("BUILDPYTHON")
- args.extend(["--python", f"{hostrunner} {buildpython}"])
-
- args.extend(regrtest_args)
-
- print(shlex.join(args), flush=True)
-
- if sys.platform == 'win32':
- from subprocess import call
- sys.exit(call(args))
- else:
- os.execve(sys.executable, args, environ)
-
-
-if __name__ == '__main__':
- main(sys.argv[1:])
projects / thirdparty / Python / cpython.git / commitdiff
