[2.7] bpo-33873: Backport regrtest from master (GH-7936)

* bpo-33718: regrtest: use format_duration() to display failed tests (GH-7686)

* Enhance also format_duration(): work on integers and rounds towards
  +infinity (math.ceil).
* Write unit tests on format_duration()

(cherry picked from commit 4ffe9c2b251f6e027b26250b7a2618e78d4edd22)

* bpo-33873: regrtest: Add warning on -R 1:3 (GH-7736)

regrtest: Add warning when using less than 3 warmup runs like -R 1:3.

(cherry picked from commit cac4fef8860e66a9da67d09762f5b614b9471a12)

* bpo-33873: Fix bug in `runtest.py` and add checks for invalid `-R` parameters (GH-7735)

Fix bug in `Lib/test/libregrtest/runtest.py` that makes running tests an extra time than the specified number of runs.

Add check for invalid --huntrleaks/-R parameters.

(cherry picked from commit 58ed7307ea0b5c5aa052291ebc3030f314f938d8)

diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index 64aa46555621d2e41a57141fb6fdfc7b1b1b707d..6b49dafd9368f49a7574f412cf878ffade7494a4 100755 (executable)
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -174,20 +174,21 @@ Pattern examples:
 import StringIO
 import datetime
 import getopt
+import imp
 import json
+import math
 import os
+import platform
 import random
 import re
 import shutil
 import sys
+import sysconfig
+import tempfile
 import time
 import traceback
-import warnings
 import unittest
-import tempfile
-import imp
-import platform
-import sysconfig
+import warnings
 
 
 # Some times __path__ and __file__ are not absolute (e.g. while running from
@@ -270,17 +271,25 @@ def usage(code, msg=''):
 
 
 def format_duration(seconds):
-    if seconds < 1.0:
-        return '%.0f ms' % (seconds * 1e3)
-    if seconds < 60.0:
-        return '%.0f sec' % seconds
+    ms = int(math.ceil(seconds * 1e3))
+    seconds, ms = divmod(ms, 1000)
+    minutes, seconds = divmod(seconds, 60)
+    hours, minutes = divmod(minutes, 60)
 
-    minutes, seconds = divmod(seconds, 60.0)
-    hours, minutes = divmod(minutes, 60.0)
+    parts = []
     if hours:
-        return '%.0f hour %.0f min' % (hours, minutes)
-    else:
-        return '%.0f min %.0f sec' % (minutes, seconds)
+        parts.append('%s hour' % hours)
+    if minutes:
+        parts.append('%s min' % minutes)
+    if seconds:
+        parts.append('%s sec' % seconds)
+    if ms:
+        parts.append('%s ms' % ms)
+    if not parts:
+        return '0 ms'
+
+    parts = parts[:2]
+    return ' '.join(parts)
 
 
 _FORMAT_TEST_RESULT = {
@@ -507,6 +516,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         except ValueError:
             pass
 
+    if huntrleaks:
+        warmup, repetitions, _ = huntrleaks
+        if warmup < 1 or repetitions < 1:
+            msg = ("Invalid values for the --huntrleaks/-R parameters. The "
+                   "number of warmups and repetitions must be at least 1 "
+                   "each (1:1).")
+            print >>sys.stderr, msg
+            sys.exit(2)
+
     if slaveargs is not None:
         args, kwargs = json.loads(slaveargs)
         if kwargs['huntrleaks']:
@@ -682,6 +700,12 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
         if ncpu:
             print "== CPU count:", ncpu
 
+    if huntrleaks:
+        warmup, repetitions, _ = huntrleaks
+        if warmup < 3:
+            print("WARNING: Running tests with --huntrleaks/-R and less than "
+                  "3 warmup repetitions can give false positives!")
+
     if randomize:
         random.seed(random_seed)
         print "Using random seed", random_seed
@@ -809,7 +833,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
                     if (ok not in (CHILD_ERROR, INTERRUPTED)
                         and test_time >= PROGRESS_MIN_TIME
                         and not pgo):
-                        text += ' (%.0f sec)' % test_time
+                        text += ' (%s)' % format_duration(test_time)
                     running = get_running(workers)
                     if running and not pgo:
                         text += ' -- running: %s' % ', '.join(running)
@@ -1293,11 +1317,12 @@ def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=Non
                 # being imported.  For tests based on unittest or doctest,
                 # explicitly invoke their test_main() function (if it exists).
                 indirect_test = getattr(the_module, "test_main", None)
-                if indirect_test is not None:
-                    indirect_test()
                 if huntrleaks:
                     refleak = dash_R(the_module, test, indirect_test,
                         huntrleaks)
+                else:
+                    if indirect_test is not None:
+                        indirect_test()
                 test_time = time.time() - start_time
             post_test_cleanup()
         finally:

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 1dd7bb581868ab3dd0c41091bdbd11f62cbea6bc..a459504e445e032a2ff645acfb8b4f25fc1d91da 100644 (file)
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -17,6 +17,8 @@ import tempfile
 import textwrap
 import unittest
 from test import support
+# Use utils alias to use the same code for TestUtils in master and 2.7 branches
+import regrtest as utils
 
 
 Py_DEBUG = hasattr(sys, 'getobjects')
@@ -683,8 +685,32 @@ class ArgsTestCase(BaseTestCase):
                                   failed=testname, rerun=testname)
 
 
+class TestUtils(unittest.TestCase):
+    def test_format_duration(self):
+        self.assertEqual(utils.format_duration(0),
+                         '0 ms')
+        self.assertEqual(utils.format_duration(1e-9),
+                         '1 ms')
+        self.assertEqual(utils.format_duration(10e-3),
+                         '10 ms')
+        self.assertEqual(utils.format_duration(1.5),
+                         '1 sec 500 ms')
+        self.assertEqual(utils.format_duration(1),
+                         '1 sec')
+        self.assertEqual(utils.format_duration(2 * 60),
+                         '2 min')
+        self.assertEqual(utils.format_duration(2 * 60 + 1),
+                         '2 min 1 sec')
+        self.assertEqual(utils.format_duration(3 * 3600),
+                         '3 hour')
+        self.assertEqual(utils.format_duration(3 * 3600  + 2 * 60 + 1),
+                         '3 hour 2 min')
+        self.assertEqual(utils.format_duration(3 * 3600 + 1),
+                         '3 hour 1 sec')
+
+
 def test_main():
-    support.run_unittest(ProgramsTestCase, ArgsTestCase)
+    support.run_unittest(ProgramsTestCase, ArgsTestCase, TestUtils)
 
 
 if __name__ == "__main__":

diff --git a/Misc/NEWS.d/next/Tests/2018-06-16-01-37-31.bpo-33873.d86vab.rst b/Misc/NEWS.d/next/Tests/2018-06-16-01-37-31.bpo-33873.d86vab.rst
new file mode 100644 (file)
index 0000000..f4f4255
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2018-06-16-01-37-31.bpo-33873.d86vab.rst
@@ -0,0 +1,4 @@
+Fix a bug in ``regrtest`` that caused an extra test to run if
+--huntrleaks/-R was used. Exit with error in case that invalid
+parameters are specified to --huntrleaks/-R (at least one warmup
+run and one repetition must be used).