if not isinstance(src, str):
src = src.decode(encoding, 'replace')
line = src.split('\n')[lineno-1]
+ if lineno == 1:
+ line = line.removeprefix('\ufeff')
self.assertIn(line, cm.exception.text)
def test_error_offset_continuation_characters(self):
check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +',
2, 19, encoding='cp1251')
- check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 10)
+ check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 12)
+ check(b'\n\n\nPython = "\xcf\xb3\xf2\xee\xed" +', 4, 12)
+ check(b'\xef\xbb\xbfPython = "\xcf\xb3\xf2\xee\xed" +', 1, 12)
check('x = "a', 1, 5)
check('lambda x: x = 2', 1, 1)
check('f{a + b + c}', 1, 2)
check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4)
check("(1+)", 1, 4)
check("[interesting\nfoo()\n", 1, 1)
- check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1)
+ check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 1, 0)
check("""f'''
{
(123_a)
# -*- coding: utf-8 -*-
import unittest
-from test.support import script_helper, captured_stdout, requires_subprocess, requires_resource
+from test import support
+from test.support import script_helper
from test.support.os_helper import TESTFN, unlink, rmtree
from test.support.import_helper import unload
import importlib
# two bytes in common with the UTF-8 BOM
self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20')
- @requires_subprocess()
+ @support.requires_subprocess()
def test_20731(self):
sub = subprocess.Popen([sys.executable,
os.path.join(os.path.dirname(__file__),
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xc3\u20ac'")
+ def test_first_utf8_coding_line_error(self):
+ src = (b'#coding:ascii \xc3\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
+
+ def test_second_utf8_coding_line_error(self):
+ src = (b'#!/usr/bin/python\n'
+ b'#coding:ascii \xc3\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
+
def test_utf8_bom(self):
src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
b'print(ascii("\xc3\xa4"))\n')
self.check_script_output(src, br"'\xe4'")
- def test_utf8_non_utf8_comment_line_error(self):
+ def test_utf8_bom_and_non_utf8_first_coding_line(self):
+ src = (b'\xef\xbb\xbf#coding:iso-8859-15\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"encoding problem: iso-8859-15 with BOM",
+ lineno=1)
+
+ def test_utf8_bom_and_non_utf8_second_coding_line(self):
+ src = (b'\xef\xbb\xbf#first\n'
+ b'#coding:iso-8859-15\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"encoding problem: iso-8859-15 with BOM",
+ lineno=2)
+
+ def test_non_utf8_shebang(self):
+ src = (b'#!/home/\xa4/bin/python\n'
+ b'#coding:iso-8859-15\n'
+ b'print(ascii("\xc3\xa4"))\n')
+ self.check_script_output(src, br"'\xc3\u20ac'")
+
+ def test_utf8_shebang_error(self):
+ src = (b'#!/home/\xc3\xa4/bin/python\n'
+ b'#coding:ascii\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte")
+
+ def test_non_utf8_shebang_error(self):
+ src = (b'#!/home/\xa4/bin/python\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src, br"Non-UTF-8 code starting with .* on line 1",
+ lineno=1)
+
+ def test_non_utf8_second_line_error(self):
+ src = (b'#first\n'
+ b'#second\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"Non-UTF-8 code starting with .* on line 2",
+ lineno=2)
+
+ def test_non_utf8_third_line_error(self):
+ src = (b'#first\n'
+ b'#second\n'
+ b'#third\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"Non-UTF-8 code starting with .* on line 3",
+ lineno=3)
+
+ def test_utf8_bom_non_utf8_third_line_error(self):
+ src = (b'\xef\xbb\xbf#first\n'
+ b'#second\n'
+ b'#third\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"Non-UTF-8 code starting with .* on line 3|"
+ br"'utf-8' codec can't decode byte",
+ lineno=3)
+
+ def test_utf_8_non_utf8_third_line_error(self):
+ src = (b'#coding: utf-8\n'
+ b'#second\n'
+ b'#third\xa4\n'
+ b'raise RuntimeError\n')
+ self.check_script_error(src,
+ br"Non-UTF-8 code starting with .* on line 3|"
+ br"'utf-8' codec can't decode byte",
+ lineno=3)
+
+ def test_utf8_non_utf8_third_line_error(self):
src = (b'#coding: utf8\n'
- b'#\n'
- b'#\xa4\n'
+ b'#second\n'
+ b'#third\xa4\n'
b'raise RuntimeError\n')
self.check_script_error(src,
br"'utf-8' codec can't decode byte|"
class UTF8ValidatorTest(unittest.TestCase):
@unittest.skipIf(not sys.platform.startswith("linux"),
"Too slow to run on non-Linux platforms")
- @requires_resource('cpu')
+ @support.requires_resource('cpu')
def test_invalid_utf8(self):
# This is a port of test_utf8_decode_invalid_sequences in
# test_unicode.py to exercise the separate utf8 validator in
check(b'\xF4'+cb+b'\xBF\xBF')
+@support.force_not_colorized_test_class
class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
def check_script_output(self, src, expected):
- with captured_stdout() as stdout:
+ with support.captured_stdout() as stdout:
exec(src)
out = stdout.getvalue().encode('latin1')
self.assertEqual(out.rstrip(), expected)
- def check_script_error(self, src, expected):
- with self.assertRaisesRegex(SyntaxError, expected.decode()) as cm:
+ def check_script_error(self, src, expected, lineno=...):
+ with self.assertRaises(SyntaxError) as cm:
exec(src)
+ exc = cm.exception
+ self.assertRegex(str(exc), expected.decode())
+ if lineno is not ...:
+ self.assertEqual(exc.lineno, lineno)
+ line = src.splitlines()[lineno-1].decode(errors='replace')
+ if lineno == 1:
+ line = line.removeprefix('\ufeff')
+ self.assertEqual(line, exc.text)
+@support.force_not_colorized_test_class
class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase):
def check_script_output(self, src, expected):
res = script_helper.assert_python_ok(fn)
self.assertEqual(res.out.rstrip(), expected)
- def check_script_error(self, src, expected):
+ def check_script_error(self, src, expected, lineno=...):
with tempfile.TemporaryDirectory() as tmpd:
fn = os.path.join(tmpd, 'test.py')
with open(fn, 'wb') as fp:
fp.write(src)
res = script_helper.assert_python_failure(fn)
- self.assertRegex(res.err.rstrip().splitlines()[-1], b'SyntaxError.*?' + expected)
+ err = res.err.rstrip()
+ self.assertRegex(err.splitlines()[-1], b'SyntaxError: ' + expected)
+ if lineno is not ...:
+ self.assertIn(f', line {lineno}\n'.encode(),
+ err.replace(os.linesep.encode(), b'\n'))
+ line = src.splitlines()[lineno-1].decode(errors='replace')
+ if lineno == 1:
+ line = line.removeprefix('\ufeff')
+ line = line.encode(sys.stderr.encoding, sys.stderr.errors)
+ self.assertIn(line, err)
+
if __name__ == "__main__":
--- /dev/null
+Support non-UTF-8 shebang and comments in Python source files if non-UTF-8
+encoding is specified. Detect decoding error in comments for default (UTF-8)
+encoding. Show the line and position of decoding error for default encoding
+in a traceback. Show the line containing the coding cookie when it conflicts
+with the BOM in a traceback.
#include <errcode.h>
#include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject()
+#include "pycore_runtime.h" // _Py_ID()
#include "lexer/state.h"
#include "lexer/lexer.h"
#include "pegen.h"
PyObject *value;
PyObject *tback;
PyErr_Fetch(&type, &value, &tback);
+ if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) {
+ if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) {
+ goto error;
+ }
+ PyErr_Restore(type, value, tback);
+ return;
+ }
errstr = PyObject_Str(value);
if (!errstr) {
goto error;
}
static int
-tok_underflow_file(struct tok_state *tok) {
- if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
- tok->cur = tok->inp = tok->buf;
- }
+tok_underflow_file(struct tok_state *tok)
+{
if (tok->decoding_state == STATE_INIT) {
/* We have not yet determined the encoding.
If an encoding is found, use the file-pointer
}
assert(tok->decoding_state != STATE_INIT);
}
+ int raw = tok->decoding_readline == NULL;
+ if (raw && tok->decoding_state != STATE_NORMAL) {
+ /* Keep the first line in the buffer to validate it later if
+ * the encoding has not yet been determined. */
+ }
+ else if (tok->start == NULL && !INSIDE_FSTRING(tok)) {
+ tok->cur = tok->inp = tok->buf;
+ }
/* Read until '\n' or EOF */
- if (tok->decoding_readline != NULL) {
+ if (!raw) {
/* We already have a codec associated with this input. */
if (!tok_readline_recode(tok)) {
return 0;
ADVANCE_LINENO();
if (tok->decoding_state != STATE_NORMAL) {
- if (tok->lineno > 2) {
- tok->decoding_state = STATE_NORMAL;
- }
- else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur),
+ if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur),
tok, fp_setreadl))
{
return 0;
}
+ if (tok->lineno >= 2) {
+ tok->decoding_state = STATE_NORMAL;
+ }
}
- /* The default encoding is UTF-8, so make sure we don't have any
- non-UTF-8 sequences in it. */
- if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) {
- _PyTokenizer_error_ret(tok);
- return 0;
+ if (raw && tok->decoding_state == STATE_NORMAL) {
+ const char *line = tok->lineno <= 2 ? tok->buf : tok->cur;
+ int lineno = tok->lineno <= 2 ? 1 : tok->lineno;
+ if (!tok->encoding) {
+ /* The default encoding is UTF-8, so make sure we don't have any
+ non-UTF-8 sequences in it. */
+ if (!_PyTokenizer_ensure_utf8(line, tok, lineno)) {
+ _PyTokenizer_error_ret(tok);
+ return 0;
+ }
+ }
+ else {
+ PyObject *tmp = PyUnicode_Decode(line, strlen(line),
+ tok->encoding, NULL);
+ if (tmp == NULL) {
+ _PyTokenizer_error_ret(tok);
+ return 0;
+ }
+ Py_DECREF(tmp);
+ }
}
assert(tok->done == E_OK);
return tok->done == E_OK;
goto error;
}
- args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno,
- col_offset, errtext, tok->lineno, end_col_offset);
+ args = Py_BuildValue("(O(OiiNii))", errmsg,
+ tok->filename ? tok->filename : Py_None,
+ tok->lineno, col_offset, errtext,
+ tok->lineno, end_col_offset);
if (args) {
PyErr_SetObject(PyExc_SyntaxError, args);
Py_DECREF(args);
tok->encoding = cs;
} else { /* then, compare cs with BOM */
if (strcmp(tok->encoding, cs) != 0) {
- _PyTokenizer_error_ret(tok);
- PyErr_Format(PyExc_SyntaxError,
- "encoding problem: %s with BOM", cs);
+ tok->line_start = line;
+ tok->cur = (char *)line;
+ assert(size <= INT_MAX);
+ _PyTokenizer_syntaxerror_known_range(tok, 0, (int)size,
+ "encoding problem: %s with BOM", cs);
PyMem_Free(cs);
+ _PyTokenizer_error_ret(tok);
return 0;
}
PyMem_Free(cs);
}
int
-_PyTokenizer_ensure_utf8(char *line, struct tok_state *tok)
+_PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno)
{
- int badchar = 0;
- unsigned char *c;
+ const char *badchar = NULL;
+ const char *c;
int length;
- for (c = (unsigned char *)line; *c; c += length) {
- if (!(length = valid_utf8(c))) {
- badchar = *c;
+ int col_offset = 0;
+ const char *line_start = line;
+ for (c = line; *c; c += length) {
+ if (!(length = valid_utf8((const unsigned char *)c))) {
+ badchar = c;
break;
}
+ col_offset++;
+ if (*c == '\n') {
+ lineno++;
+ col_offset = 0;
+ line_start = c + 1;
+ }
}
if (badchar) {
- PyErr_Format(PyExc_SyntaxError,
- "Non-UTF-8 code starting with '\\x%.2x' "
- "in file %U on line %i, "
- "but no encoding declared; "
- "see https://peps.python.org/pep-0263/ for details",
- badchar, tok->filename, tok->lineno);
+ tok->lineno = lineno;
+ tok->line_start = line_start;
+ tok->cur = (char *)badchar;
+ _PyTokenizer_syntaxerror_known_range(tok,
+ col_offset + 1, col_offset + 1,
+ "Non-UTF-8 code starting with '\\x%.2x'"
+ "%s%V on line %i, "
+ "but no encoding declared; "
+ "see https://peps.python.org/pep-0263/ for details",
+ (unsigned char)*badchar,
+ tok->filename ? " in file " : "", tok->filename, "",
+ lineno);
return 0;
}
return 1;
struct tok_state *tok);
int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
int set_readline(struct tok_state *, const char *));
-int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok);
+int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno);
#ifdef Py_DEBUG
void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size);
ADVANCE_LINENO();
/* The default encoding is UTF-8, so make sure we don't have any
non-UTF-8 sequences in it. */
- if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) {
+ if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {
_PyTokenizer_error_ret(tok);
return 0;
}
/* need to check line 1 and 2 separately since check_coding_spec
assumes a single line as input */
if (newl[0]) {
+ tok->lineno = 1;
if (!_PyTokenizer_check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) {
return NULL;
}
if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) {
+ tok->lineno = 2;
if (!_PyTokenizer_check_coding_spec(newl[0]+1, newl[1] - newl[0],
tok, buf_setreadl))
return NULL;
}
}
+ tok->lineno = 0;
if (tok->enc != NULL) {
assert(utf8 == NULL);
utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc);
return _PyTokenizer_error_ret(tok);
str = PyBytes_AS_STRING(utf8);
}
+ else if (!_PyTokenizer_ensure_utf8(str, tok, 1)) {
+ return _PyTokenizer_error_ret(tok);
+ }
assert(tok->decoding_buffer == NULL);
tok->decoding_buffer = utf8; /* CAUTION */
return str;
projects / thirdparty / Python / cpython.git / commitdiff
