From: Anthony Sottile <asottile@umich.edu>
Date: Sun, 24 Jan 2021 09:23:17 +0000 (-0800)
Subject: bpo-43014: Improve performance of tokenize.tokenize by 20-30%
X-Git-Tag: v3.10.0a5~58
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=15bd9efd01e44087664e78bf766865a6d2e06626;p=thirdparty%2FPython%2Fcpython.git

bpo-43014: Improve performance of tokenize.tokenize by 20-30%
---

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1aee21b5e18f..42c1f10373de 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -27,6 +27,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
 from builtins import open as _builtin_open
 from codecs import lookup, BOM_UTF8
 import collections
+import functools
 from io import TextIOWrapper
 import itertools as _itertools
 import re
@@ -95,6 +96,7 @@ def _all_string_prefixes():
                 result.add(''.join(u))
     return result
 
+@functools.lru_cache
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 
diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
new file mode 100644
index 000000000000..02898e4a3a42
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst
@@ -0,0 +1 @@
+Improve performance of :mod:`tokenize` by 20-30%.  Patch by Anthony Sottile.