gh-109276: libregrtest calls random.seed() before each test (#109279)

libregrtest now calls random.seed() before running each test file
when -r/--randomize command line option is used. Moreover, it's also
called in worker processes. It should help to make tests more
deterministic. Previously, it was only called once in the main
process before running all test files and it was not called in worker
processes.

* Convert some f-strings to regular strings in test_regrtest when
  f-string is not needed.
* Remove unused all_methods variable from test_regrtest.
* Add RunTests members are now mandatory.

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 2c0a6c204373cce9006ccf742f441c330b5c2e12..f52deac329dc84d153976191f764e8d4fae8a309 100644 (file)
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -112,8 +112,11 @@ class Regrtest:
         self.junit_filename: StrPath | None = ns.xmlpath
         self.memory_limit: str | None = ns.memlimit
         self.gc_threshold: int | None = ns.threshold
-        self.use_resources: list[str] = ns.use_resources
-        self.python_cmd: list[str] | None = ns.python
+        self.use_resources: tuple[str] = tuple(ns.use_resources)
+        if ns.python:
+            self.python_cmd: tuple[str] = tuple(ns.python)
+        else:
+            self.python_cmd = None
         self.coverage: bool = ns.trace
         self.coverage_dir: StrPath | None = ns.coverdir
         self.tmp_dir: StrPath | None = ns.tempdir
@@ -377,8 +380,11 @@ class Regrtest:
         return RunTests(
             tests,
             fail_fast=self.fail_fast,
+            fail_env_changed=self.fail_env_changed,
             match_tests=self.match_tests,
             ignore_tests=self.ignore_tests,
+            match_tests_dict=None,
+            rerun=None,
             forever=self.forever,
             pgo=self.pgo,
             pgo_extended=self.pgo_extended,
@@ -393,6 +399,9 @@ class Regrtest:
             gc_threshold=self.gc_threshold,
             use_resources=self.use_resources,
             python_cmd=self.python_cmd,
+            randomize=self.randomize,
+            random_seed=self.random_seed,
+            json_fd=None,
         )
 
     def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py
index 64f8f6ab0ff30578135719fb9a2e5dc4c6a1b44d..656958fa71312fabf786d8a9cbd087af36646ef7 100644 (file)
--- a/Lib/test/libregrtest/runtests.py
+++ b/Lib/test/libregrtest/runtests.py
@@ -16,29 +16,31 @@ class HuntRefleak:
 @dataclasses.dataclass(slots=True, frozen=True)
 class RunTests:
     tests: TestTuple
-    fail_fast: bool = False
-    fail_env_changed: bool = False
-    match_tests: FilterTuple | None = None
-    ignore_tests: FilterTuple | None = None
-    match_tests_dict: FilterDict | None = None
-    rerun: bool = False
-    forever: bool = False
-    pgo: bool = False
-    pgo_extended: bool = False
-    output_on_failure: bool = False
-    timeout: float | None = None
-    verbose: int = 0
-    quiet: bool = False
-    hunt_refleak: HuntRefleak | None = None
-    test_dir: StrPath | None = None
-    use_junit: bool = False
-    memory_limit: str | None = None
-    gc_threshold: int | None = None
-    use_resources: list[str] = dataclasses.field(default_factory=list)
-    python_cmd: list[str] | None = None
+    fail_fast: bool
+    fail_env_changed: bool
+    match_tests: FilterTuple | None
+    ignore_tests: FilterTuple | None
+    match_tests_dict: FilterDict | None
+    rerun: bool
+    forever: bool
+    pgo: bool
+    pgo_extended: bool
+    output_on_failure: bool
+    timeout: float | None
+    verbose: int
+    quiet: bool
+    hunt_refleak: HuntRefleak | None
+    test_dir: StrPath | None
+    use_junit: bool
+    memory_limit: str | None
+    gc_threshold: int | None
+    use_resources: tuple[str]
+    python_cmd: tuple[str] | None
+    randomize: bool
+    random_seed: int | None
     # On Unix, it's a file descriptor.
     # On Windows, it's a handle.
-    json_fd: int | None = None
+    json_fd: int | None
 
     def copy(self, **override):
         state = dataclasses.asdict(self)

diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py
index 353a0f70b94ab2df1be1d5a68b02a6d40e4f4d38..1c40b7c7b3bbfd49d2ab8e9be9cd7a3a65a3e7ff 100644 (file)
--- a/Lib/test/libregrtest/setup.py
+++ b/Lib/test/libregrtest/setup.py
@@ -1,5 +1,6 @@
 import faulthandler
 import os
+import random
 import signal
 import sys
 import unittest
@@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests):
 
     if runtests.gc_threshold is not None:
         gc.set_threshold(runtests.gc_threshold)
+
+    if runtests.randomize:
+        random.seed(runtests.random_seed)

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 466b6f66797b7aff0fd8fb5d298db155016d9316..7cf3d05a6e6d7089ba7f490e88bef21feab920af 100644 (file)
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -11,6 +11,7 @@ import io
 import locale
 import os.path
 import platform
+import random
 import re
 import subprocess
 import sys
@@ -504,7 +505,7 @@ class BaseTestCase(unittest.TestCase):
         if rerun is not None:
             regex = list_regex('%s re-run test%s', [rerun.name])
             self.check_line(output, regex)
-            regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode"
+            regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
             self.check_line(output, regex)
             regex = fr"Re-running {rerun.name} in verbose mode"
             if rerun.match:
@@ -1019,13 +1020,13 @@ class ArgsTestCase(BaseTestCase):
                                   forever=True)
 
     @without_optimizer
-    def check_leak(self, code, what, *, multiprocessing=False):
+    def check_leak(self, code, what, *, run_workers=False):
         test = self.create_test('huntrleaks', code=code)
 
         filename = 'reflog.txt'
         self.addCleanup(os_helper.unlink, filename)
         cmd = ['--huntrleaks', '3:3:']
-        if multiprocessing:
+        if run_workers:
             cmd.append('-j1')
         cmd.append(test)
         output = self.run_tests(*cmd,
@@ -1044,7 +1045,7 @@ class ArgsTestCase(BaseTestCase):
             self.assertIn(line2, reflog)
 
     @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
-    def check_huntrleaks(self, *, multiprocessing: bool):
+    def check_huntrleaks(self, *, run_workers: bool):
         # test --huntrleaks
         code = textwrap.dedent("""
             import unittest
@@ -1055,13 +1056,13 @@ class ArgsTestCase(BaseTestCase):
                 def test_leak(self):
                     GLOBAL_LIST.append(object())
         """)
-        self.check_leak(code, 'references', multiprocessing=multiprocessing)
+        self.check_leak(code, 'references', run_workers=run_workers)
 
     def test_huntrleaks(self):
-        self.check_huntrleaks(multiprocessing=False)
+        self.check_huntrleaks(run_workers=False)
 
     def test_huntrleaks_mp(self):
-        self.check_huntrleaks(multiprocessing=True)
+        self.check_huntrleaks(run_workers=True)
 
     @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
     def test_huntrleaks_fd_leak(self):
@@ -1139,8 +1140,6 @@ class ArgsTestCase(BaseTestCase):
                 def test_method4(self):
                     pass
         """)
-        all_methods = ['test_method1', 'test_method2',
-                       'test_method3', 'test_method4']
         testname = self.create_test(code=code)
 
         # only run a subset
@@ -1762,7 +1761,7 @@ class ArgsTestCase(BaseTestCase):
             if encoding is None:
                 encoding = sys.__stdout__.encoding
                 if encoding is None:
-                    self.skipTest(f"cannot get regrtest worker encoding")
+                    self.skipTest("cannot get regrtest worker encoding")
 
         nonascii = b"byte:\xa0\xa9\xff\n"
         try:
@@ -1789,7 +1788,7 @@ class ArgsTestCase(BaseTestCase):
                                   stats=0)
 
     def test_doctest(self):
-        code = textwrap.dedent(fr'''
+        code = textwrap.dedent(r'''
             import doctest
             import sys
             from test import support
@@ -1827,6 +1826,46 @@ class ArgsTestCase(BaseTestCase):
                                   randomize=True,
                                   stats=TestStats(1, 1, 0))
 
+    def _check_random_seed(self, run_workers: bool):
+        # gh-109276: When -r/--randomize is used, random.seed() is called
+        # with the same random seed before running each test file.
+        code = textwrap.dedent(r'''
+            import random
+            import unittest
+
+            class RandomSeedTest(unittest.TestCase):
+                def test_randint(self):
+                    numbers = [random.randint(0, 1000) for _ in range(10)]
+                    print(f"Random numbers: {numbers}")
+        ''')
+        tests = [self.create_test(name=f'test_random{i}', code=code)
+                 for i in range(1, 3+1)]
+
+        random_seed = 856_656_202
+        cmd = ["--randomize", f"--randseed={random_seed}"]
+        if run_workers:
+            # run as many worker processes than the number of tests
+            cmd.append(f'-j{len(tests)}')
+        cmd.extend(tests)
+        output = self.run_tests(*cmd)
+
+        random.seed(random_seed)
+        # Make the assumption that nothing consume entropy between libregrest
+        # setup_tests() which calls random.seed() and RandomSeedTest calling
+        # random.randint().
+        numbers = [random.randint(0, 1000) for _ in range(10)]
+        expected = f"Random numbers: {numbers}"
+
+        regex = r'^Random numbers: .*$'
+        matches = re.findall(regex, output, flags=re.MULTILINE)
+        self.assertEqual(matches, [expected] * len(tests))
+
+    def test_random_seed(self):
+        self._check_random_seed(run_workers=False)
+
+    def test_random_seed_workers(self):
+        self._check_random_seed(run_workers=True)
+
 
 class TestUtils(unittest.TestCase):
     def test_format_duration(self):

diff --git a/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst
new file mode 100644 (file)
index 0000000..cf4074b
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst
@@ -0,0 +1,6 @@
+libregrtest now calls :func:`random.seed()` before running each test file
+when ``-r/--randomize`` command line option is used. Moreover, it's also
+called in worker processes.  It should help to make tests more
+deterministic. Previously, it was only called once in the main process before
+running all test files and it was not called in worker processes.  Patch by
+Victor Stinner.