bpo-43014: Improve performance of tokenize.tokenize by 20-30%

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1aee21b5e18fa716dfaa5306fc6aa8a96d253641..42c1f10373de9b5819547f75145bfac79dfd5cee 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -27,6 +27,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
 from builtins import open as _builtin_open
 from codecs import lookup, BOM_UTF8
 import collections
+import functools
 from io import TextIOWrapper
 import itertools as _itertools
 import re
@@ -95,6 +96,7 @@ def _all_string_prefixes():
                 result.add(''.join(u))
     return result
 
+@functools.lru_cache
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 

diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
new file mode 100644 (file)
index 0000000..02898e4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
@@ -0,0 +1 @@
+Improve performance of :mod:`tokenize` by 20-30%.  Patch by Anthony Sottile.