From 64856ad8b7279718ff10a9fb32003c2221af7228 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Jun 2018 12:36:31 +0200 Subject: [PATCH] bpo-18174: regrtest -R 3:3 now also detects FD leak (#7409) "python -m test --huntrleaks ..." now also checks for leak of file descriptors. Co-Authored-By: Richard Oudkerk --- Lib/test/regrtest.py | 62 ++++++++++++++++++++++++------------ Lib/test/support/__init__.py | 57 +++++++++++++++++++++++++++++++++ Lib/test/test_regrtest.py | 18 +++++++++++ 3 files changed, 117 insertions(+), 20 deletions(-) diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index e834394ce952..d19be8878a46 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -1416,37 +1416,59 @@ def dash_R(the_module, test, indirect_test, huntrleaks): nwarmup, ntracked, fname = huntrleaks fname = os.path.join(support.SAVEDCWD, fname) repcount = nwarmup + ntracked + rc_deltas = [0] * ntracked + fd_deltas = [0] * ntracked + print >> sys.stderr, "beginning", repcount, "repetitions" print >> sys.stderr, ("1234567890"*(repcount//10 + 1))[:repcount] dash_R_cleanup(fs, ps, pic, zdc, abcs) + # initialize variables to make pyflakes quiet + rc_before = fd_before = 0 for i in range(repcount): - rc_before = sys.gettotalrefcount() run_the_test() sys.stderr.write('.') dash_R_cleanup(fs, ps, pic, zdc, abcs) rc_after = sys.gettotalrefcount() + fd_after = support.fd_count() if i >= nwarmup: - deltas.append(rc_after - rc_before) + rc_deltas[i - nwarmup] = rc_after - rc_before + fd_deltas[i - nwarmup] = fd_after - fd_before + rc_before = rc_after + fd_before = fd_after print >> sys.stderr - # bpo-30776: Try to ignore false positives: - # - # [3, 0, 0] - # [0, 1, 0] - # [8, -8, 1] - # - # Expected leaks: - # - # [5, 5, 6] - # [10, 1, 1] - if all(delta >= 1 for delta in deltas): - msg = '%s leaked %s references, sum=%s' % (test, deltas, sum(deltas)) - print >> sys.stderr, msg - with open(fname, "a") as refrep: - print >> refrep, msg - refrep.flush() - return True - return False + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + # Checker for reference counters and memomry blocks. + # + # bpo-30776: Try to ignore false positives: + # + # [3, 0, 0] + # [0, 1, 0] + # [8, -8, 1] + # + # Expected leaks: + # + # [5, 5, 6] + # [10, 1, 1] + return all(delta >= 1 for delta in deltas) + + def check_fd_deltas(deltas): + return any(deltas) + + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (fd_deltas, 'file descriptors', check_fd_deltas) + ]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % (test, deltas, item_name, sum(deltas)) + print >> sys.stderr, msg + with open(fname, "a") as refrep: + print >> refrep, msg + refrep.flush() + failed = True + return failed def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copy_reg diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 47af6b39465f..8ffe1f861e69 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2062,6 +2062,63 @@ def _crash_python(): _testcapi._read_null() +def fd_count(): + """Count the number of open file descriptors. + """ + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + old_modes = None + if sys.platform == 'win32': + # bpo-25306, bpo-31009: Call CrtSetReportMode() to not kill the process + # on invalid file descriptor if Python is compiled in debug mode + try: + import msvcrt + msvcrt.CrtSetReportMode + except (AttributeError, ImportError): + # no msvcrt or a release build + pass + else: + old_modes = {} + for report_type in (msvcrt.CRT_WARN, + msvcrt.CRT_ERROR, + msvcrt.CRT_ASSERT): + old_modes[report_type] = msvcrt.CrtSetReportMode(report_type, 0) + + MAXFD = 256 + if hasattr(os, 'sysconf'): + try: + MAXFD = os.sysconf("SC_OPEN_MAX") + except OSError: + pass + + try: + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + finally: + if old_modes is not None: + for report_type in (msvcrt.CRT_WARN, + msvcrt.CRT_ERROR, + msvcrt.CRT_ASSERT): + msvcrt.CrtSetReportMode(report_type, old_modes[report_type]) + + return count + + class SaveSignals: """ Save an restore signal handlers. diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 988a72c10996..94ada9a545e6 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -511,6 +511,24 @@ class ArgsTestCase(BaseTestCase): """) self.check_leak(code, 'references') + @unittest.skipUnless(Py_DEBUG, 'need a debug build') + def test_huntrleaks_fd_leak(self): + # test --huntrleaks for file descriptor leak + code = textwrap.dedent(""" + import os + import unittest + from test import support + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + fd = os.open(__file__, os.O_RDONLY) + # bug: never close the file descriptor + + def test_main(): + support.run_unittest(FDLeakTest) + """) + self.check_leak(code, 'file descriptors') + def test_list_tests(self): # test --list-tests tests = [self.create_test() for i in range(5)] -- 2.47.3