From: Jason R. Coombs <jaraco@jaraco.com>
Date: Sat, 20 Jun 2015 23:52:22 +0000 (-0400)
Subject: Issue #20387: Add test capturing failure to roundtrip indented code in tokenize module.
X-Git-Tag: v3.5.0b3~35^2^2^2~3
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5713b3c5bf0c27d5443e6d3a1cd2ce3495778597;p=thirdparty%2FPython%2Fcpython.git

Issue #20387: Add test capturing failure to roundtrip indented code in tokenize module.
---

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 3e8a65474ed8..00a2c2b30b5f 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1229,6 +1229,22 @@ class UntokenizeTest(TestCase):
         self.assertEqual(untokenize(iter(tokens)), b'Hello ')
 
 
+class TestRoundtrip(TestCase):
+    def roundtrip(self, code):
+        if isinstance(code, str):
+            code = code.encode('utf-8')
+        return untokenize(tokenize(BytesIO(code).readline))
+
+    def test_indentation_semantics_retained(self):
+        """
+        Ensure that although whitespace might be mutated in a roundtrip,
+        the semantic meaning of the indentation remains consistent.
+        """
+        code = "if False:\n\tx=3\n\tx=3\n"
+        codelines = roundtrip(code).split('\n')
+        self.assertEqual(codelines[1], codelines[2])
+
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 def test_main():
@@ -1239,6 +1255,7 @@ def test_main():
     support.run_unittest(TestDetectEncoding)
     support.run_unittest(TestTokenize)
     support.run_unittest(UntokenizeTest)
+    support.run_unittest(TestRoundtrip)
 
 if __name__ == "__main__":
     test_main()