[3.11] bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (GH-30895) (GH...

(cherry picked from commit 995386071f96e4cfebfa027a71ca9134e4651d2a)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>

diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 655e4d2e56f8f04e72dcbd28a254e5a7e0a04364..0125227bf11755a7b0391b32712fb7ac760f08a2 100644 (file)
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -28,6 +28,19 @@ from test.support import threading_helper
 # Must be smaller than buildbot "1200 seconds without output" limit.
 EXIT_TIMEOUT = 120.0
 
+# gh-90681: When rerunning tests, we might need to rerun the whole
+# class or module suite if some its life-cycle hooks fail.
+# Test level hooks are not affected.
+_TEST_LIFECYCLE_HOOKS = frozenset((
+    'setUpClass', 'tearDownClass',
+    'setUpModule', 'tearDownModule',
+))
+
+EXITCODE_BAD_TEST = 2
+EXITCODE_INTERRUPTED = 130
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+
 
 class Regrtest:
     """Execute a test suite.
@@ -331,8 +344,12 @@ class Regrtest:
 
             errors = result.errors or []
             failures = result.failures or []
-            error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
-            failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
+            error_names = [
+                self.normalize_test_name(test_full_name, is_error=True)
+                for (test_full_name, *_) in errors]
+            failure_names = [
+                self.normalize_test_name(test_full_name)
+                for (test_full_name, *_) in failures]
             self.ns.verbose = True
             orig_match_tests = self.ns.match_tests
             if errors or failures:
@@ -358,6 +375,21 @@ class Regrtest:
 
         self.display_result()
 
+    def normalize_test_name(self, test_full_name, *, is_error=False):
+        short_name = test_full_name.split(" ")[0]
+        if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
+            # This means that we have a failure in a life-cycle hook,
+            # we need to rerun the whole module or class suite.
+            # Basically the error looks like this:
+            #    ERROR: setUpClass (test.test_reg_ex.RegTest)
+            # or
+            #    ERROR: setUpModule (test.test_reg_ex)
+            # So, we need to parse the class / module name.
+            lpar = test_full_name.index('(')
+            rpar = test_full_name.index(')')
+            return test_full_name[lpar + 1: rpar].split('.')[-1]
+        return short_name
+
     def display_result(self):
         # If running the test suite for PGO then no one cares about results.
         if self.ns.pgo:

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index c33f90d8071df97b65934e14ae2195ac400f0bc8..98be9cdd0e1652572409b76936ca7da8cb63bdc0 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1097,7 +1097,7 @@ def _run_suite(suite):
     if junit_xml_list is not None:
         junit_xml_list.append(result.get_xml_element())
 
-    if not result.testsRun and not result.skipped:
+    if not result.testsRun and not result.skipped and not result.errors:
         raise TestDidNotRun
     if not result.wasSuccessful():
         if len(result.errors) == 1 and not result.failures:

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 15e2f89ee20c1d4f76792b5448eecda332be6db8..f692dd1e3e65b47673d3f2ec552287050977e3f8 100644 (file)
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -30,6 +30,11 @@ ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..')
 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
 LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
 
+EXITCODE_BAD_TEST = 2
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+EXITCODE_INTERRUPTED = 130
+
 TEST_INTERRUPTED = textwrap.dedent("""
     from signal import SIGINT, raise_signal
     try:
@@ -1115,6 +1120,160 @@ class ArgsTestCase(BaseTestCase):
         self.check_executed_tests(output, [testname],
                                   rerun={testname: "test_fail_once"})
 
+    def test_rerun_setup_class_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.TestCase):
+                @classmethod
+                def setUpClass(self):
+                    raise RuntimeError('Fail')
+
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "ExampleTests"})
+
+    def test_rerun_teardown_class_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.TestCase):
+                @classmethod
+                def tearDownClass(self):
+                    raise RuntimeError('Fail')
+
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "ExampleTests"})
+
+    def test_rerun_setup_module_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            def setUpModule():
+                raise RuntimeError('Fail')
+
+            class ExampleTests(unittest.TestCase):
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: testname})
+
+    def test_rerun_teardown_module_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            def tearDownModule():
+                raise RuntimeError('Fail')
+
+            class ExampleTests(unittest.TestCase):
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: testname})
+
+    def test_rerun_setup_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.TestCase):
+                def setUp(self):
+                    raise RuntimeError('Fail')
+
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "test_success"})
+
+    def test_rerun_teardown_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.TestCase):
+                def tearDown(self):
+                    raise RuntimeError('Fail')
+
+                def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "test_success"})
+
+    def test_rerun_async_setup_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.IsolatedAsyncioTestCase):
+                async def asyncSetUp(self):
+                    raise RuntimeError('Fail')
+
+                async def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "test_success"})
+
+    def test_rerun_async_teardown_hook_failure(self):
+        # FAILURE then FAILURE
+        code = textwrap.dedent("""
+            import unittest
+
+            class ExampleTests(unittest.IsolatedAsyncioTestCase):
+                async def asyncTearDown(self):
+                    raise RuntimeError('Fail')
+
+                async def test_success(self):
+                    return
+        """)
+        testname = self.create_test(code=code)
+
+        output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+        self.check_executed_tests(output, testname,
+                                  failed=[testname],
+                                  rerun={testname: "test_success"})
+
     def test_no_tests_ran(self):
         code = textwrap.dedent("""
             import unittest