From 15bd9efd01e44087664e78bf766865a6d2e06626 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 24 Jan 2021 01:23:17 -0800 Subject: [PATCH] bpo-43014: Improve performance of tokenize.tokenize by 20-30% --- Lib/tokenize.py | 2 ++ .../next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1aee21b5e18f..42c1f10373de 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -27,6 +27,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections +import functools from io import TextIOWrapper import itertools as _itertools import re @@ -95,6 +96,7 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache def _compile(expr): return re.compile(expr, re.UNICODE) diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst new file mode 100644 index 000000000000..02898e4a3a42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst @@ -0,0 +1 @@ +Improve performance of :mod:`tokenize` by 20-30%. Patch by Anthony Sottile. -- 2.47.3