[3.12] gh-108303: Move tokenize-related data to Lib/test/tokenizedata (GH-109265...

* gh-108303: Move tokenize-related data to Lib/test/tokenizedata (GH-109265)

(cherry picked from commit 1110c5bc828218086f6397ec05a9312fb73ea30a)

* gh-108303: Add `Lib/test/tokenizedata` to `TESTSUBDIRS` (#109314)

(cherry picked from commit 42ab2cbd7b5e76e919b70883ae683e789dbd913d)

---------

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>

diff --git a/.gitattributes b/.gitattributes
index 4ed95069442f3d1485737b56f8f45e60b68b4ea8..2bfd4bf9f9b20200384411cdd60de77663976cd1 100644 (file)
--- a/.gitattributes
+++ b/.gitattributes
@@ -25,7 +25,7 @@ PC/classicAppCompat.* binary
 [attr]noeol -text
 
 Lib/test/cjkencodings/*                    noeol
-Lib/test/coding20731.py                    noeol
+Lib/test/tokenizedata/coding20731.py       noeol
 Lib/test/decimaltestdata/*.decTest         noeol
 Lib/test/test_email/data/*.txt             noeol
 Lib/test/test_importlib/resources/data01/*           noeol

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 19f6a03745d2d0b46894dae1bc1e7b593d27af34..4c1fd20ea921b86a7927a68d1f4eef29183cd686 100644 (file)
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
       - id: check-yaml
       - id: end-of-file-fixer
         types: [python]
-        exclude: Lib/test/coding20731.py
+        exclude: Lib/test/tokenizedata/coding20731.py
       - id: trailing-whitespace
         types_or: [c, python, rst]
 

diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index 5e0a44ad9691ec30b270b91827a8cbe244320989..c4e6551f6057822fcc2cecf371c304ef51257502 100644 (file)
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -132,7 +132,9 @@ class PyCompileTestsBase:
             os.chmod(self.directory, mode.st_mode)
 
     def test_bad_coding(self):
-        bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py')
+        bad_coding = os.path.join(os.path.dirname(__file__),
+                                  'tokenizedata',
+                                  'bad_coding2.py')
         with support.captured_stderr():
             self.assertIsNone(py_compile.compile(bad_coding, doraise=False))
         self.assertFalse(os.path.exists(
@@ -195,7 +197,9 @@ class PyCompileTestsBase:
         self.assertEqual(flags, 0b1)
 
     def test_quiet(self):
-        bad_coding = os.path.join(os.path.dirname(__file__), 'bad_coding2.py')
+        bad_coding = os.path.join(os.path.dirname(__file__),
+                                  'tokenizedata',
+                                  'bad_coding2.py')
         with support.captured_stderr() as stderr:
             self.assertIsNone(py_compile.compile(bad_coding, doraise=False, quiet=2))
             self.assertIsNone(py_compile.compile(bad_coding, doraise=True, quiet=2))
@@ -260,14 +264,18 @@ class PyCompileCLITestCase(unittest.TestCase):
         self.assertTrue(os.path.exists(self.cache_path))
 
     def test_bad_syntax(self):
-        bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py')
+        bad_syntax = os.path.join(os.path.dirname(__file__),
+                                  'tokenizedata',
+                                  'badsyntax_3131.py')
         rc, stdout, stderr = self.pycompilecmd_failure(bad_syntax)
         self.assertEqual(rc, 1)
         self.assertEqual(stdout, b'')
         self.assertIn(b'SyntaxError', stderr)
 
     def test_bad_syntax_with_quiet(self):
-        bad_syntax = os.path.join(os.path.dirname(__file__), 'badsyntax_3131.py')
+        bad_syntax = os.path.join(os.path.dirname(__file__),
+                                  'tokenizedata',
+                                  'badsyntax_3131.py')
         rc, stdout, stderr = self.pycompilecmd_failure('-q', bad_syntax)
         self.assertEqual(rc, 1)
         self.assertEqual(stdout, b'')

diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index 72c2b47779e00533ecdef52a2d64bbd21571c5bb..27871378f1c79e146133745374bc3a93a51b891a 100644 (file)
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -68,6 +68,7 @@ class MiscSourceEncodingTest(unittest.TestCase):
     def test_20731(self):
         sub = subprocess.Popen([sys.executable,
                         os.path.join(os.path.dirname(__file__),
+                                     'tokenizedata',
                                      'coding20731.py')],
                         stderr=subprocess.PIPE)
         err = sub.communicate()[1]
@@ -100,10 +101,10 @@ class MiscSourceEncodingTest(unittest.TestCase):
         self.verify_bad_module(module_name)
 
     def verify_bad_module(self, module_name):
-        self.assertRaises(SyntaxError, __import__, 'test.' + module_name)
+        self.assertRaises(SyntaxError, __import__, 'test.tokenizedata.' + module_name)
 
         path = os.path.dirname(__file__)
-        filename = os.path.join(path, module_name + '.py')
+        filename = os.path.join(path, 'tokenizedata', module_name + '.py')
         with open(filename, "rb") as fp:
             bytes = fp.read()
         self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')

diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
index 013c62630bc44b6b02f4a79494c1e36fdea0bf26..5d9714eb287e10630af5ff0a64ab96151bf31c92 100644 (file)
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -2564,16 +2564,17 @@ class CommandLineTest(unittest.TestCase):
         return script_helper.assert_python_failure('-m', 'tarfile', *args)
 
     def make_simple_tarfile(self, tar_name):
-        files = [support.findfile('tokenize_tests.txt'),
+        files = [support.findfile('tokenize_tests.txt',
+                                  subdir='tokenizedata'),
                  support.findfile('tokenize_tests-no-coding-cookie-'
-                                  'and-utf8-bom-sig-only.txt')]
+                                  'and-utf8-bom-sig-only.txt',
+                                  subdir='tokenizedata')]
         self.addCleanup(os_helper.unlink, tar_name)
         with tarfile.open(tar_name, 'w') as tf:
             for tardata in files:
                 tf.add(tardata, arcname=os.path.basename(tardata))
 
     def make_evil_tarfile(self, tar_name):
-        files = [support.findfile('tokenize_tests.txt')]
         self.addCleanup(os_helper.unlink, tar_name)
         with tarfile.open(tar_name, 'w') as tf:
             benign = tarfile.TarInfo('benign')
@@ -2654,9 +2655,11 @@ class CommandLineTest(unittest.TestCase):
         self.assertEqual(rc, 1)
 
     def test_create_command(self):
-        files = [support.findfile('tokenize_tests.txt'),
+        files = [support.findfile('tokenize_tests.txt',
+                                  subdir='tokenizedata'),
                  support.findfile('tokenize_tests-no-coding-cookie-'
-                                  'and-utf8-bom-sig-only.txt')]
+                                  'and-utf8-bom-sig-only.txt',
+                                  subdir='tokenizedata')]
         for opt in '-c', '--create':
             try:
                 out = self.tarfilecmd(opt, tmpname, *files)
@@ -2667,9 +2670,11 @@ class CommandLineTest(unittest.TestCase):
                 os_helper.unlink(tmpname)
 
     def test_create_command_verbose(self):
-        files = [support.findfile('tokenize_tests.txt'),
+        files = [support.findfile('tokenize_tests.txt',
+                                  subdir='tokenizedata'),
                  support.findfile('tokenize_tests-no-coding-cookie-'
-                                  'and-utf8-bom-sig-only.txt')]
+                                  'and-utf8-bom-sig-only.txt',
+                                  subdir='tokenizedata')]
         for opt in '-v', '--verbose':
             try:
                 out = self.tarfilecmd(opt, '-c', tmpname, *files,
@@ -2681,7 +2686,7 @@ class CommandLineTest(unittest.TestCase):
                 os_helper.unlink(tmpname)
 
     def test_create_command_dotless_filename(self):
-        files = [support.findfile('tokenize_tests.txt')]
+        files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
         try:
             out = self.tarfilecmd('-c', dotlessname, *files)
             self.assertEqual(out, b'')
@@ -2692,7 +2697,7 @@ class CommandLineTest(unittest.TestCase):
 
     def test_create_command_dot_started_filename(self):
         tar_name = os.path.join(TEMPDIR, ".testtar")
-        files = [support.findfile('tokenize_tests.txt')]
+        files = [support.findfile('tokenize_tests.txt', subdir='tokenizedata')]
         try:
             out = self.tarfilecmd('-c', tar_name, *files)
             self.assertEqual(out, b'')
@@ -2702,9 +2707,11 @@ class CommandLineTest(unittest.TestCase):
             os_helper.unlink(tar_name)
 
     def test_create_command_compressed(self):
-        files = [support.findfile('tokenize_tests.txt'),
+        files = [support.findfile('tokenize_tests.txt',
+                                  subdir='tokenizedata'),
                  support.findfile('tokenize_tests-no-coding-cookie-'
-                                  'and-utf8-bom-sig-only.txt')]
+                                  'and-utf8-bom-sig-only.txt',
+                                  subdir='tokenizedata')]
         for filetype in (GzipTest, Bz2Test, LzmaTest):
             if not filetype.open:
                 continue

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index c320478cf3a7bf683bdd879f44a7a076e069f16b..40680f004723c532c3023fd658cdb2688ff1b8c0 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1198,7 +1198,7 @@ class TestTokenizerAdheresToPep0263(TestCase):
     """
 
     def _testFile(self, filename):
-        path = os.path.join(os.path.dirname(__file__), filename)
+        path = os.path.join(os.path.dirname(__file__), 'tokenizedata', filename)
         with open(path, 'rb') as f:
             TestRoundtrip.check_roundtrip(self, f)
 
@@ -1791,7 +1791,7 @@ class TestRoundtrip(TestCase):
 
         self.check_roundtrip("if x == 1 : \n"
                              "  print(x)\n")
-        fn = support.findfile("tokenize_tests.txt")
+        fn = support.findfile("tokenize_tests.txt", subdir="tokenizedata")
         with open(fn, 'rb') as f:
             self.check_roundtrip(f)
         self.check_roundtrip("if x == 1:\n"
@@ -1846,8 +1846,7 @@ class TestRoundtrip(TestCase):
         # pass the '-ucpu' option to process the full directory.
 
         import glob, random
-        fn = support.findfile("tokenize_tests.txt")
-        tempdir = os.path.dirname(fn) or os.curdir
+        tempdir = os.path.dirname(__file__) or os.curdir
         testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py"))
 
         # Tokenize is broken on test_pep3131.py because regular expressions are

diff --git a/Lib/test/test_tools/test_reindent.py b/Lib/test/test_tools/test_reindent.py
index 3b0c793a38e4daba9eec7cb1246374b3dc30c567..64e31c2b7703c01bc37f466b2ded40ab462acfe0 100644 (file)
--- a/Lib/test/test_tools/test_reindent.py
+++ b/Lib/test/test_tools/test_reindent.py
@@ -25,7 +25,7 @@ class ReindentTests(unittest.TestCase):
         self.assertGreater(err, b'')
 
     def test_reindent_file_with_bad_encoding(self):
-        bad_coding_path = findfile('bad_coding.py')
+        bad_coding_path = findfile('bad_coding.py', subdir='tokenizedata')
         rc, out, err = assert_python_ok(self.script, '-r', bad_coding_path)
         self.assertEqual(out, b'')
         self.assertNotEqual(err, b'')

diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py
index 5b9ced5d1cb837f3f3874bb1b605f30373dadebc..63c6c055824b20f1428bea3a066984271c64b031 100644 (file)
--- a/Lib/test/test_unicode_identifiers.py
+++ b/Lib/test/test_unicode_identifiers.py
@@ -19,7 +19,7 @@ class PEP3131Test(unittest.TestCase):
 
     def test_invalid(self):
         try:
-            from test import badsyntax_3131
+            from test.tokenizedata import badsyntax_3131
         except SyntaxError as err:
             self.assertEqual(str(err),
               "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)")

diff --git a/Lib/test/tokenizedata/__init__.py b/Lib/test/tokenizedata/__init__.py
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/Lib/test/bad_coding.py b/Lib/test/tokenizedata/bad_coding.py
similarity index 100%
rename from Lib/test/bad_coding.py
rename to Lib/test/tokenizedata/bad_coding.py

diff --git a/Lib/test/bad_coding2.py b/Lib/test/tokenizedata/bad_coding2.py
similarity index 100%
rename from Lib/test/bad_coding2.py
rename to Lib/test/tokenizedata/bad_coding2.py

diff --git a/Lib/test/badsyntax_3131.py b/Lib/test/tokenizedata/badsyntax_3131.py
similarity index 100%
rename from Lib/test/badsyntax_3131.py
rename to Lib/test/tokenizedata/badsyntax_3131.py

diff --git a/Lib/test/coding20731.py b/Lib/test/tokenizedata/coding20731.py
similarity index 100%
rename from Lib/test/coding20731.py
rename to Lib/test/tokenizedata/coding20731.py

diff --git a/Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
similarity index 100%
rename from Lib/test/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt
rename to Lib/test/tokenizedata/tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt

diff --git a/Lib/test/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt b/Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt
similarity index 100%
rename from Lib/test/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt
rename to Lib/test/tokenizedata/tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt

diff --git a/Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
similarity index 100%
rename from Lib/test/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt
rename to Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt

diff --git a/Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt b/Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt
similarity index 100%
rename from Lib/test/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt
rename to Lib/test/tokenizedata/tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt

diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenizedata/tokenize_tests.txt
similarity index 100%
rename from Lib/test/tokenize_tests.txt
rename to Lib/test/tokenizedata/tokenize_tests.txt

diff --git a/Makefile.pre.in b/Makefile.pre.in
index cf054c19de6cf25df2bd74d3c036bf3863c76d55..7418ddf1500dc991edadb77a06622276988fb2d7 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -2226,6 +2226,7 @@ TESTSUBDIRS=      idlelib/idle_test \
                test/test_zipfile/_path \
                test/test_zoneinfo \
                test/test_zoneinfo/data \
+               test/tokenizedata \
                test/tracedmodules \
                test/typinganndata \
                test/xmltestdata \