[3.11] gh-106300: Improve errors testing in test_unittest.test_runner (GH-106737...

Use a custom exception to prevent unintentional silence of actual errors.
(cherry picked from commit fd9d70a94de5b0756b52b9ae21e236e25545db4f)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>

diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py
index e02b2dadc8fdbbb6b661bc20f199372a08818e25..3aefc1e0a5fdd11f3dbdfbe04050223b6469ee27 100644 (file)
--- a/Lib/unittest/test/test_runner.py
+++ b/Lib/unittest/test/test_runner.py
@@ -21,6 +21,13 @@ def getRunner():
                                    stream=io.StringIO())
 
 
+class CustomError(Exception):
+    pass
+
+# For test output compat:
+CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError"
+
+
 def runTests(*cases):
     suite = unittest.TestSuite()
     for case in cases:
@@ -43,7 +50,7 @@ def cleanup(ordering, blowUp=False):
         ordering.append('cleanup_good')
     else:
         ordering.append('cleanup_exc')
-        raise Exception('CleanUpExc')
+        raise CustomError('CleanUpExc')
 
 
 class TestCM:
@@ -105,8 +112,8 @@ class TestCleanUp(unittest.TestCase):
         result = unittest.TestResult()
         outcome = test._outcome = _Outcome(result=result)
 
-        CleanUpExc = Exception('foo')
-        exc2 = Exception('bar')
+        CleanUpExc = CustomError('foo')
+        exc2 = CustomError('bar')
         def cleanup1():
             raise CleanUpExc
 
@@ -122,10 +129,10 @@ class TestCleanUp(unittest.TestCase):
         (_, msg2), (_, msg1) = result.errors
         self.assertIn('in cleanup1', msg1)
         self.assertIn('raise CleanUpExc', msg1)
-        self.assertIn('Exception: foo', msg1)
+        self.assertIn(f'{CustomErrorRepr}: foo', msg1)
         self.assertIn('in cleanup2', msg2)
         self.assertIn('raise exc2', msg2)
-        self.assertIn('Exception: bar', msg2)
+        self.assertIn(f'{CustomErrorRepr}: bar', msg2)
 
     def testCleanupInRun(self):
         blowUp = False
@@ -136,7 +143,7 @@ class TestCleanUp(unittest.TestCase):
                 ordering.append('setUp')
                 test.addCleanup(cleanup2)
                 if blowUp:
-                    raise Exception('foo')
+                    raise CustomError('foo')
 
             def testNothing(self):
                 ordering.append('test')
@@ -277,7 +284,7 @@ class TestClassCleanup(unittest.TestCase):
                 ordering.append('setUpClass')
                 cls.addClassCleanup(cleanup, ordering)
                 if blowUp:
-                    raise Exception()
+                    raise CustomError()
             def testNothing(self):
                 ordering.append('test')
             @classmethod
@@ -303,7 +310,7 @@ class TestClassCleanup(unittest.TestCase):
                 ordering.append('setUpClass')
                 cls.addClassCleanup(cleanup, ordering)
                 if blowUp:
-                    raise Exception()
+                    raise CustomError()
             def testNothing(self):
                 ordering.append('test')
             @classmethod
@@ -343,7 +350,7 @@ class TestClassCleanup(unittest.TestCase):
         ordering = []
         blowUp = True
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'CleanUpExc')
         self.assertEqual(ordering,
@@ -363,10 +370,10 @@ class TestClassCleanup(unittest.TestCase):
             @classmethod
             def tearDownClass(cls):
                 ordering.append('tearDownClass')
-                raise Exception('TearDownClassExc')
+                raise CustomError('TearDownClassExc')
 
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'TearDownClassExc')
         self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass'])
@@ -376,7 +383,7 @@ class TestClassCleanup(unittest.TestCase):
         ordering = []
         blowUp = True
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'TearDownClassExc')
         self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass'])
@@ -389,16 +396,22 @@ class TestClassCleanup(unittest.TestCase):
                 pass
 
         def cleanup1():
-            raise Exception('cleanup1')
+            raise CustomError('cleanup1')
 
         def cleanup2():
-            raise Exception('cleanup2')
+            raise CustomError('cleanup2')
 
         TestableTest.addClassCleanup(cleanup1)
         TestableTest.addClassCleanup(cleanup2)
-        with self.assertRaises(Exception) as e:
-            TestableTest.doClassCleanups()
-            self.assertEqual(e, 'cleanup1')
+        TestableTest.doClassCleanups()
+
+        self.assertEqual(len(TestableTest.tearDown_exceptions), 2)
+
+        e1, e2 = TestableTest.tearDown_exceptions
+        self.assertIsInstance(e1[1], CustomError)
+        self.assertEqual(str(e1[1]), 'cleanup2')
+        self.assertIsInstance(e2[1], CustomError)
+        self.assertEqual(str(e2[1]), 'cleanup1')
 
     def test_with_errors_addCleanUp(self):
         ordering = []
@@ -418,7 +431,7 @@ class TestClassCleanup(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'setUp', 'cleanup_exc',
                           'tearDownClass', 'cleanup_good'])
@@ -441,7 +454,7 @@ class TestClassCleanup(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'setUp', 'test', 'cleanup_good',
                           'tearDownClass', 'cleanup_exc'])
@@ -457,11 +470,11 @@ class TestClassCleanup(unittest.TestCase):
                 ordering.append('setUpClass')
                 cls.addClassCleanup(cleanup, ordering, blowUp=True)
                 if class_blow_up:
-                    raise Exception('ClassExc')
+                    raise CustomError('ClassExc')
             def setUp(self):
                 ordering.append('setUp')
                 if method_blow_up:
-                    raise Exception('MethodExc')
+                    raise CustomError('MethodExc')
             def testNothing(self):
                 ordering.append('test')
             @classmethod
@@ -470,7 +483,7 @@ class TestClassCleanup(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'setUp', 'test',
                           'tearDownClass', 'cleanup_exc'])
@@ -480,9 +493,9 @@ class TestClassCleanup(unittest.TestCase):
         method_blow_up = False
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: ClassExc')
+                         f'{CustomErrorRepr}: ClassExc')
         self.assertEqual(result.errors[1][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'cleanup_exc'])
 
@@ -491,9 +504,9 @@ class TestClassCleanup(unittest.TestCase):
         method_blow_up = True
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: MethodExc')
+                         f'{CustomErrorRepr}: MethodExc')
         self.assertEqual(result.errors[1][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'setUp', 'tearDownClass',
                           'cleanup_exc'])
@@ -510,11 +523,11 @@ class TestClassCleanup(unittest.TestCase):
             @classmethod
             def tearDownClass(cls):
                 ordering.append('tearDownClass')
-                raise Exception('TearDownExc')
+                raise CustomError('TearDownExc')
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: TearDownExc')
+                         f'{CustomErrorRepr}: TearDownExc')
         self.assertEqual(ordering,
                          ['setUpClass', 'test', 'tearDownClass', 'cleanup_good'])
 
@@ -607,7 +620,7 @@ class TestModuleCleanUp(unittest.TestCase):
             module_cleanups.append((3, args, kwargs))
 
         def module_cleanup_bad(*args, **kwargs):
-            raise Exception('CleanUpExc')
+            raise CustomError('CleanUpExc')
 
         class Module(object):
             unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3,
@@ -617,7 +630,7 @@ class TestModuleCleanUp(unittest.TestCase):
                          [(module_cleanup_good, (1, 2, 3),
                            dict(four='hello', five='goodbye')),
                           (module_cleanup_bad, (), {})])
-        with self.assertRaises(Exception) as e:
+        with self.assertRaises(CustomError) as e:
             unittest.case.doModuleCleanups()
         self.assertEqual(str(e.exception), 'CleanUpExc')
         self.assertEqual(unittest.case._module_cleanups, [])
@@ -646,7 +659,7 @@ class TestModuleCleanUp(unittest.TestCase):
                 ordering.append('setUpModule')
                 unittest.addModuleCleanup(cleanup, ordering)
                 if blowUp:
-                    raise Exception('setUpModule Exc')
+                    raise CustomError('setUpModule Exc')
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule')
@@ -666,7 +679,7 @@ class TestModuleCleanUp(unittest.TestCase):
         result = runTests(TestableTest)
         self.assertEqual(ordering, ['setUpModule', 'cleanup_good'])
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: setUpModule Exc')
+                         f'{CustomErrorRepr}: setUpModule Exc')
 
         ordering = []
         blowUp = False
@@ -686,7 +699,7 @@ class TestModuleCleanUp(unittest.TestCase):
                 ordering.append('setUpModule')
                 unittest.addModuleCleanup(cleanup, ordering)
                 if blowUp:
-                    raise Exception()
+                    raise CustomError()
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule')
@@ -697,7 +710,7 @@ class TestModuleCleanUp(unittest.TestCase):
                 ordering.append('setUpModule2')
                 unittest.addModuleCleanup(cleanup, ordering)
                 if blowUp2:
-                    raise Exception()
+                    raise CustomError()
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule2')
@@ -786,7 +799,7 @@ class TestModuleCleanUp(unittest.TestCase):
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule')
-                raise Exception('CleanUpExc')
+                raise CustomError('CleanUpExc')
 
         class TestableTest(unittest.TestCase):
             @classmethod
@@ -802,7 +815,7 @@ class TestModuleCleanUp(unittest.TestCase):
         sys.modules['Module'] = Module
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
                                     'tearDownClass', 'tearDownModule',
                                     'cleanup_good'])
@@ -842,7 +855,7 @@ class TestModuleCleanUp(unittest.TestCase):
         ordering = []
         blowUp = True
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'CleanUpExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
@@ -860,7 +873,7 @@ class TestModuleCleanUp(unittest.TestCase):
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule')
-                raise Exception('TearDownModuleExc')
+                raise CustomError('TearDownModuleExc')
 
         class TestableTest(unittest.TestCase):
             @classmethod
@@ -875,7 +888,7 @@ class TestModuleCleanUp(unittest.TestCase):
         TestableTest.__module__ = 'Module'
         sys.modules['Module'] = Module
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'TearDownModuleExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
@@ -886,7 +899,7 @@ class TestModuleCleanUp(unittest.TestCase):
         ordering = []
         blowUp = True
         suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest)
-        with self.assertRaises(Exception) as cm:
+        with self.assertRaises(CustomError) as cm:
             suite.debug()
         self.assertEqual(str(cm.exception), 'TearDownModuleExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test',
@@ -965,7 +978,7 @@ class TestModuleCleanUp(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpModule', 'setUpClass', 'test', 'tearDownClass',
                           'cleanup_exc', 'tearDownModule', 'cleanup_good'])
@@ -995,7 +1008,7 @@ class TestModuleCleanUp(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpModule', 'setUp', 'test', 'tearDown',
                           'cleanup_exc', 'tearDownModule', 'cleanup_good'])
@@ -1011,7 +1024,7 @@ class TestModuleCleanUp(unittest.TestCase):
                 ordering.append('setUpModule')
                 unittest.addModuleCleanup(cleanup, ordering, blowUp=True)
                 if module_blow_up:
-                    raise Exception('ModuleExc')
+                    raise CustomError('ModuleExc')
             @staticmethod
             def tearDownModule():
                 ordering.append('tearDownModule')
@@ -1021,11 +1034,11 @@ class TestModuleCleanUp(unittest.TestCase):
             def setUpClass(cls):
                 ordering.append('setUpClass')
                 if class_blow_up:
-                    raise Exception('ClassExc')
+                    raise CustomError('ClassExc')
             def setUp(self):
                 ordering.append('setUp')
                 if method_blow_up:
-                    raise Exception('MethodExc')
+                    raise CustomError('MethodExc')
             def testNothing(self):
                 ordering.append('test')
             @classmethod
@@ -1037,7 +1050,7 @@ class TestModuleCleanUp(unittest.TestCase):
 
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering,
                          ['setUpModule', 'setUpClass', 'setUp', 'test',
                           'tearDownClass', 'tearDownModule',
@@ -1049,9 +1062,9 @@ class TestModuleCleanUp(unittest.TestCase):
         method_blow_up = False
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: ModuleExc')
+                         f'{CustomErrorRepr}: ModuleExc')
         self.assertEqual(result.errors[1][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering, ['setUpModule', 'cleanup_exc'])
 
         ordering = []
@@ -1060,9 +1073,9 @@ class TestModuleCleanUp(unittest.TestCase):
         method_blow_up = False
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: ClassExc')
+                         f'{CustomErrorRepr}: ClassExc')
         self.assertEqual(result.errors[1][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass',
                                     'tearDownModule', 'cleanup_exc'])
 
@@ -1072,9 +1085,9 @@ class TestModuleCleanUp(unittest.TestCase):
         method_blow_up = True
         result = runTests(TestableTest)
         self.assertEqual(result.errors[0][1].splitlines()[-1],
-                         'Exception: MethodExc')
+                         f'{CustomErrorRepr}: MethodExc')
         self.assertEqual(result.errors[1][1].splitlines()[-1],
-                         'Exception: CleanUpExc')
+                         f'{CustomErrorRepr}: CleanUpExc')
         self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp',
                                     'tearDownClass', 'tearDownModule',
                                     'cleanup_exc'])