From: Pablo Galindo <Pablogsal@gmail.com>
Date: Mon, 6 Jan 2020 15:59:09 +0000 (+0000)
Subject: bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)
X-Git-Tag: v3.9.0a3~134
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=5ec91f78d59d9c39b984f284e00cd04b96ddb5db;p=thirdparty%2FPython%2Fcpython.git

bpo-39209: Manage correctly multi-line tokens in interactive mode (GH-17860)
---

diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py
index 9efd459a6f07..71f192f90d9a 100644
--- a/Lib/test/test_repl.py
+++ b/Lib/test/test_repl.py
@@ -58,5 +58,41 @@ class TestInteractiveInterpreter(unittest.TestCase):
         # Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.
         self.assertIn(p.returncode, (1, 120))
 
+    @cpython_only
+    def test_multiline_string_parsing(self):
+        # bpo-39209: Multiline string tokens need to be handled in the tokenizer
+        # in two places: the interactive path and the non-interactive path.
+        user_input = '''\
+        x = """<?xml version="1.0" encoding="iso-8859-1"?>
+        <test>
+            <Users>
+                <fun25>
+                    <limits>
+                        <total>0KiB</total>
+                        <kbps>0</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </limits>
+                    <usages>
+                        <total>16738211KiB</total>
+                        <kbps>237.15</kbps>
+                        <rps>1.3</rps>
+                        <connections>0</connections>
+                    </usages>
+                    <time_to_refresh>never</time_to_refresh>
+                    <limit_exceeded_URL>none</limit_exceeded_URL>
+                </fun25>
+            </Users>
+        </test>"""
+        '''
+        user_input = dedent(user_input)
+        user_input = user_input.encode()
+        p = spawn_repl()
+        with SuppressCrashReport():
+            p.stdin.write(user_input)
+        output = kill_python(p)
+        self.assertEqual(p.returncode, 0)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst
new file mode 100644
index 000000000000..c05b3f8dfa4d
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst	
@@ -0,0 +1,2 @@
+Correctly handle multi-line tokens in interactive mode. Patch by Pablo
+Galindo.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f84093dae5b6..f73c32684c7b 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)
                 size_t start = tok->start - tok->buf;
                 size_t oldlen = tok->cur - tok->buf;
                 size_t newlen = oldlen + strlen(newtok);
+                Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
                 char *buf = tok->buf;
                 buf = (char *)PyMem_REALLOC(buf, newlen+1);
                 tok->lineno++;
@@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)
                 }
                 tok->buf = buf;
                 tok->cur = tok->buf + oldlen;
+                tok->multi_line_start = tok->buf + cur_multi_line_start;
                 tok->line_start = tok->cur;
                 strcpy(tok->buf + oldlen, newtok);
                 PyMem_FREE(newtok);