]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
Issue #28701: Replace PyUnicode_CompareWithASCIIString with _PyUnicode_EqualToASCIISt...
authorSerhiy Storchaka <storchaka@gmail.com>
Wed, 16 Nov 2016 08:19:20 +0000 (10:19 +0200)
committerSerhiy Storchaka <storchaka@gmail.com>
Wed, 16 Nov 2016 08:19:20 +0000 (10:19 +0200)
The latter function is more readable, faster and doesn't raise exceptions.

22 files changed:
1  2 
Doc/c-api/unicode.rst
Include/unicodeobject.h
Modules/_decimal/_decimal.c
Modules/_elementtree.c
Modules/_io/textio.c
Modules/_io/winconsoleio.c
Modules/_lsprof.c
Modules/_pickle.c
Modules/_sqlite/connection.c
Modules/_testcapimodule.c
Modules/pyexpat.c
Objects/longobject.c
Objects/moduleobject.c
Objects/typeobject.c
Objects/unicodeobject.c
Python/_warnings.c
Python/ast.c
Python/compile.c
Python/future.c
Python/getargs.c
Python/import.c
Python/symtable.c

Simple merge
Simple merge
Simple merge
Simple merge
index dca4ea9e2deb330978fd3b4e046c3676199d9e0d,46c99b5a1d4b19694a981b92a212d01a8da37131..2f55eb0595b6e2c33eb459f9ea3830564ff5ac21
@@@ -1021,9 -1021,9 +1021,9 @@@ _io_TextIOWrapper___init___impl(textio 
                  goto error;
          }
          else if (PyUnicode_Check(res)) {
 -            encodefuncentry *e = encodefuncs;
 +            const encodefuncentry *e = encodefuncs;
              while (e->name != NULL) {
-                 if (!PyUnicode_CompareWithASCIIString(res, e->name)) {
+                 if (_PyUnicode_EqualToASCIIString(res, e->name)) {
                      self->encodefunc = e->encodefunc;
                      break;
                  }
index 4666a3867adb9d646d22834b16ff155486e9f43b,0000000000000000000000000000000000000000..7b00a9eb6d6d924d3f1d317f172b1098b01befd5
mode 100644,000000..100644
--- /dev/null
@@@ -1,1154 -1,0 +1,1154 @@@
-     if (PyUnicode_CompareWithASCIIString(decoded_upper, "CONIN$") == 0) {
 +/*
 +    An implementation of Windows console I/O
 +
 +    Classes defined here: _WindowsConsoleIO
 +
 +    Written by Steve Dower
 +*/
 +
 +#define PY_SSIZE_T_CLEAN
 +#include "Python.h"
 +
 +#ifdef MS_WINDOWS
 +
 +#include "structmember.h"
 +#ifdef HAVE_SYS_TYPES_H
 +#include <sys/types.h>
 +#endif
 +#ifdef HAVE_SYS_STAT_H
 +#include <sys/stat.h>
 +#endif
 +#include <stddef.h> /* For offsetof */
 +
 +#define WIN32_LEAN_AND_MEAN
 +#include <windows.h>
 +#include <fcntl.h>
 +
 +#include "_iomodule.h"
 +
 +/* BUFSIZ determines how many characters can be typed at the console
 +   before it starts blocking. */
 +#if BUFSIZ < (16*1024)
 +#define SMALLCHUNK (2*1024)
 +#elif (BUFSIZ >= (2 << 25))
 +#error "unreasonable BUFSIZ > 64MB defined"
 +#else
 +#define SMALLCHUNK BUFSIZ
 +#endif
 +
 +/* BUFMAX determines how many bytes can be read in one go. */
 +#define BUFMAX (32*1024*1024)
 +
 +/* SMALLBUF determines how many utf-8 characters will be
 +   buffered within the stream, in order to support reads
 +   of less than one character */
 +#define SMALLBUF 4
 +
 +char _get_console_type(HANDLE handle) {
 +    DWORD mode, peek_count;
 +
 +    if (handle == INVALID_HANDLE_VALUE)
 +        return '\0';
 +
 +    if (!GetConsoleMode(handle, &mode))
 +        return '\0';
 +
 +    /* Peek at the handle to see whether it is an input or output handle */
 +    if (GetNumberOfConsoleInputEvents(handle, &peek_count))
 +        return 'r';
 +    return 'w';
 +}
 +
 +char _PyIO_get_console_type(PyObject *path_or_fd) {
 +    int fd;
 +
 +    fd = PyLong_AsLong(path_or_fd);
 +    PyErr_Clear();
 +    if (fd >= 0) {
 +        HANDLE handle;
 +        _Py_BEGIN_SUPPRESS_IPH
 +        handle = (HANDLE)_get_osfhandle(fd);
 +        _Py_END_SUPPRESS_IPH
 +        if (!handle)
 +            return '\0';
 +        return _get_console_type(handle);
 +    }
 +
 +    PyObject *decoded, *decoded_upper;
 +
 +    int d = PyUnicode_FSDecoder(path_or_fd, &decoded);
 +    if (!d) {
 +        PyErr_Clear();
 +        return '\0';
 +    }
 +    if (!PyUnicode_Check(decoded)) {
 +        Py_CLEAR(decoded);
 +        return '\0';
 +    }
 +    decoded_upper = PyObject_CallMethod(decoded, "upper", "");
 +    Py_CLEAR(decoded);
 +    if (!decoded_upper) {
 +        PyErr_Clear();
 +        return '\0';
 +    }
 +
 +    char m = '\0';
-     } else if (PyUnicode_CompareWithASCIIString(decoded_upper, "CONOUT$") == 0) {
++    if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONIN$")) {
 +        m = 'r';
-     } else if (PyUnicode_CompareWithASCIIString(decoded_upper, "CON") == 0) {
++    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CONOUT$")) {
 +        m = 'w';
++    } else if (_PyUnicode_EqualToASCIIString(decoded_upper, "CON")) {
 +        m = 'x';
 +    }
 +
 +    Py_CLEAR(decoded_upper);
 +    return m;
 +}
 +
 +/*[clinic input]
 +module _io
 +class _io._WindowsConsoleIO "winconsoleio *" "&PyWindowsConsoleIO_Type"
 +[clinic start generated code]*/
 +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e897fdc1fba4e131]*/
 +
 +/*[python input]
 +class io_ssize_t_converter(CConverter):
 +    type = 'Py_ssize_t'
 +    converter = '_PyIO_ConvertSsize_t'
 +[python start generated code]*/
 +/*[python end generated code: output=da39a3ee5e6b4b0d input=d0a811d3cbfd1b33]*/
 +
 +typedef struct {
 +    PyObject_HEAD
 +    HANDLE handle;
 +    int fd;
 +    unsigned int created : 1;
 +    unsigned int readable : 1;
 +    unsigned int writable : 1;
 +    unsigned int closehandle : 1;
 +    char finalizing;
 +    unsigned int blksize;
 +    PyObject *weakreflist;
 +    PyObject *dict;
 +    char buf[SMALLBUF];
 +    wchar_t wbuf;
 +} winconsoleio;
 +
 +PyTypeObject PyWindowsConsoleIO_Type;
 +
 +_Py_IDENTIFIER(name);
 +
 +int
 +_PyWindowsConsoleIO_closed(PyObject *self)
 +{
 +    return ((winconsoleio *)self)->handle == INVALID_HANDLE_VALUE;
 +}
 +
 +
 +/* Returns 0 on success, -1 with exception set on failure. */
 +static int
 +internal_close(winconsoleio *self)
 +{
 +    if (self->handle != INVALID_HANDLE_VALUE) {
 +        if (self->closehandle) {
 +            if (self->fd >= 0) {
 +                _Py_BEGIN_SUPPRESS_IPH
 +                close(self->fd);
 +                _Py_END_SUPPRESS_IPH
 +            }
 +            CloseHandle(self->handle);
 +        }
 +        self->handle = INVALID_HANDLE_VALUE;
 +        self->fd = -1;
 +    }
 +    return 0;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.close
 +
 +Close the handle.
 +
 +A closed handle cannot be used for further I/O operations.  close() may be
 +called more than once without error.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_close_impl(winconsoleio *self)
 +/*[clinic end generated code: output=27ef95b66c29057b input=185617e349ae4c7b]*/
 +{
 +    PyObject *res;
 +    PyObject *exc, *val, *tb;
 +    int rc;
 +    _Py_IDENTIFIER(close);
 +    res = _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
 +                                 &PyId_close, "O", self);
 +    if (!self->closehandle) {
 +        self->handle = INVALID_HANDLE_VALUE;
 +        return res;
 +    }
 +    if (res == NULL)
 +        PyErr_Fetch(&exc, &val, &tb);
 +    rc = internal_close(self);
 +    if (res == NULL)
 +        _PyErr_ChainExceptions(exc, val, tb);
 +    if (rc < 0)
 +        Py_CLEAR(res);
 +    return res;
 +}
 +
 +static PyObject *
 +winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 +{
 +    winconsoleio *self;
 +
 +    assert(type != NULL && type->tp_alloc != NULL);
 +
 +    self = (winconsoleio *) type->tp_alloc(type, 0);
 +    if (self != NULL) {
 +        self->handle = INVALID_HANDLE_VALUE;
 +        self->fd = -1;
 +        self->created = 0;
 +        self->readable = 0;
 +        self->writable = 0;
 +        self->closehandle = 0;
 +        self->blksize = 0;
 +        self->weakreflist = NULL;
 +    }
 +
 +    return (PyObject *) self;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.__init__
 +    file as nameobj: object
 +    mode: str = "r"
 +    closefd: int(c_default="1") = True
 +    opener: object = None
 +
 +Open a console buffer by file descriptor.
 +
 +The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All
 +other mode characters will be ignored. Mode 'b' will be assumed if it is
 +omitted. The *opener* parameter is always ignored.
 +[clinic start generated code]*/
 +
 +static int
 +_io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
 +                                    const char *mode, int closefd,
 +                                    PyObject *opener)
 +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=61be39633a86f5d7]*/
 +{
 +    const char *s;
 +    wchar_t *name = NULL;
 +    char console_type = '\0';
 +    int ret = 0;
 +    int rwa = 0;
 +    int fd = -1;
 +    int fd_is_own = 0;
 +
 +    assert(PyWindowsConsoleIO_Check(self));
 +    if (self->handle >= 0) {
 +        if (self->closehandle) {
 +            /* Have to close the existing file first. */
 +            if (internal_close(self) < 0)
 +                return -1;
 +        }
 +        else
 +            self->handle = INVALID_HANDLE_VALUE;
 +    }
 +
 +    if (PyFloat_Check(nameobj)) {
 +        PyErr_SetString(PyExc_TypeError,
 +                        "integer argument expected, got float");
 +        return -1;
 +    }
 +
 +    fd = _PyLong_AsInt(nameobj);
 +    if (fd < 0) {
 +        if (!PyErr_Occurred()) {
 +            PyErr_SetString(PyExc_ValueError,
 +                            "negative file descriptor");
 +            return -1;
 +        }
 +        PyErr_Clear();
 +    }
 +    self->fd = fd;
 +
 +    if (fd < 0) {
 +        PyObject *decodedname = Py_None;
 +        Py_INCREF(decodedname);
 +
 +        int d = PyUnicode_FSDecoder(nameobj, (void*)&decodedname);
 +        if (!d)
 +            return -1;
 +
 +        Py_ssize_t length;
 +        name = PyUnicode_AsWideCharString(decodedname, &length);
 +        console_type = _PyIO_get_console_type(decodedname);
 +        Py_CLEAR(decodedname);
 +        if (name == NULL)
 +            return -1;
 +
 +        if (wcslen(name) != length) {
 +            PyMem_Free(name);
 +            PyErr_SetString(PyExc_ValueError, "embedded null character");
 +            return -1;
 +        }
 +    }
 +
 +    s = mode;
 +    while (*s) {
 +        switch (*s++) {
 +        case '+':
 +        case 'a':
 +        case 'b':
 +        case 'x':
 +            break;
 +        case 'r':
 +            if (rwa)
 +                goto bad_mode;
 +            rwa = 1;
 +            self->readable = 1;
 +            if (console_type == 'x')
 +                console_type = 'r';
 +            break;
 +        case 'w':
 +            if (rwa)
 +                goto bad_mode;
 +            rwa = 1;
 +            self->writable = 1;
 +            if (console_type == 'x')
 +                console_type = 'w';
 +            break;
 +        default:
 +            PyErr_Format(PyExc_ValueError,
 +                         "invalid mode: %.200s", mode);
 +            goto error;
 +        }
 +    }
 +
 +    if (!rwa)
 +        goto bad_mode;
 +
 +    if (fd >= 0) {
 +        _Py_BEGIN_SUPPRESS_IPH
 +        self->handle = (HANDLE)_get_osfhandle(fd);
 +        _Py_END_SUPPRESS_IPH
 +        self->closehandle = 0;
 +    } else {
 +        DWORD access = GENERIC_READ;
 +
 +        self->closehandle = 1;
 +        if (!closefd) {
 +            PyErr_SetString(PyExc_ValueError,
 +                "Cannot use closefd=False with file name");
 +            goto error;
 +        }
 +
 +        if (self->writable)
 +            access = GENERIC_WRITE;
 +
 +        Py_BEGIN_ALLOW_THREADS
 +        /* Attempt to open for read/write initially, then fall back
 +           on the specific access. This is required for modern names
 +           CONIN$ and CONOUT$, which allow reading/writing state as
 +           well as reading/writing content. */
 +        self->handle = CreateFileW(name, GENERIC_READ | GENERIC_WRITE,
 +            FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
 +        if (self->handle == INVALID_HANDLE_VALUE)
 +            self->handle = CreateFileW(name, access,
 +                FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL);
 +        Py_END_ALLOW_THREADS
 +
 +        if (self->handle == INVALID_HANDLE_VALUE) {
 +            PyErr_SetExcFromWindowsErrWithFilenameObject(PyExc_OSError, GetLastError(), nameobj);
 +            goto error;
 +        }
 +    }
 +
 +    if (console_type == '\0')
 +        console_type = _get_console_type(self->handle);
 +
 +    if (self->writable && console_type != 'w') {
 +        PyErr_SetString(PyExc_ValueError,
 +            "Cannot open console input buffer for writing");
 +        goto error;
 +    }
 +    if (self->readable && console_type != 'r') {
 +        PyErr_SetString(PyExc_ValueError,
 +            "Cannot open console output buffer for reading");
 +        goto error;
 +    }
 +
 +    self->blksize = DEFAULT_BUFFER_SIZE;
 +    memset(self->buf, 0, 4);
 +
 +    if (_PyObject_SetAttrId((PyObject *)self, &PyId_name, nameobj) < 0)
 +        goto error;
 +
 +    goto done;
 +
 +bad_mode:
 +    PyErr_SetString(PyExc_ValueError,
 +                    "Must have exactly one of read or write mode");
 +error:
 +    ret = -1;
 +    internal_close(self);
 +
 +done:
 +    if (name)
 +        PyMem_Free(name);
 +    return ret;
 +}
 +
 +static int
 +winconsoleio_traverse(winconsoleio *self, visitproc visit, void *arg)
 +{
 +    Py_VISIT(self->dict);
 +    return 0;
 +}
 +
 +static int
 +winconsoleio_clear(winconsoleio *self)
 +{
 +    Py_CLEAR(self->dict);
 +    return 0;
 +}
 +
 +static void
 +winconsoleio_dealloc(winconsoleio *self)
 +{
 +    self->finalizing = 1;
 +    if (_PyIOBase_finalize((PyObject *) self) < 0)
 +        return;
 +    _PyObject_GC_UNTRACK(self);
 +    if (self->weakreflist != NULL)
 +        PyObject_ClearWeakRefs((PyObject *) self);
 +    Py_CLEAR(self->dict);
 +    Py_TYPE(self)->tp_free((PyObject *)self);
 +}
 +
 +static PyObject *
 +err_closed(void)
 +{
 +    PyErr_SetString(PyExc_ValueError, "I/O operation on closed file");
 +    return NULL;
 +}
 +
 +static PyObject *
 +err_mode(const char *action)
 +{
 +    _PyIO_State *state = IO_STATE();
 +    if (state != NULL)
 +        PyErr_Format(state->unsupported_operation,
 +                     "Console buffer does not support %s", action);
 +    return NULL;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.fileno
 +
 +Return the underlying file descriptor (an integer).
 +
 +fileno is only set when a file descriptor is used to open
 +one of the standard streams.
 +
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_fileno_impl(winconsoleio *self)
 +/*[clinic end generated code: output=006fa74ce3b5cfbf input=079adc330ddaabe6]*/
 +{
 +    if (self->fd < 0 && self->handle != INVALID_HANDLE_VALUE) {
 +        _Py_BEGIN_SUPPRESS_IPH
 +        if (self->writable)
 +            self->fd = _open_osfhandle((intptr_t)self->handle, _O_WRONLY | _O_BINARY);
 +        else
 +            self->fd = _open_osfhandle((intptr_t)self->handle, _O_RDONLY | _O_BINARY);
 +        _Py_END_SUPPRESS_IPH
 +    }
 +    if (self->fd < 0)
 +        return err_mode("fileno");
 +    return PyLong_FromLong(self->fd);
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.readable
 +
 +True if console is an input buffer.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_readable_impl(winconsoleio *self)
 +/*[clinic end generated code: output=daf9cef2743becf0 input=6be9defb5302daae]*/
 +{
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +    return PyBool_FromLong((long) self->readable);
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.writable
 +
 +True if console is an output buffer.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_writable_impl(winconsoleio *self)
 +/*[clinic end generated code: output=e0a2ad7eae5abf67 input=cefbd8abc24df6a0]*/
 +{
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +    return PyBool_FromLong((long) self->writable);
 +}
 +
 +static DWORD
 +_buflen(winconsoleio *self)
 +{
 +    for (DWORD i = 0; i < SMALLBUF; ++i) {
 +        if (!self->buf[i])
 +            return i;
 +    }
 +    return SMALLBUF;
 +}
 +
 +static DWORD
 +_copyfrombuf(winconsoleio *self, char *buf, DWORD len)
 +{
 +    DWORD n = 0;
 +
 +    while (self->buf[0] && len--) {
 +        buf[n++] = self->buf[0];
 +        for (int i = 1; i < SMALLBUF; ++i)
 +            self->buf[i - 1] = self->buf[i];
 +        self->buf[SMALLBUF - 1] = 0;
 +    }
 +
 +    return n;
 +}
 +
 +static wchar_t *
 +read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) {
 +    int err = 0, sig = 0;
 +
 +    wchar_t *buf = (wchar_t*)PyMem_Malloc(maxlen * sizeof(wchar_t));
 +    if (!buf)
 +        goto error;
 +
 +    *readlen = 0;
 +
 +    //DebugBreak();
 +    Py_BEGIN_ALLOW_THREADS
 +    DWORD off = 0;
 +    while (off < maxlen) {
 +        DWORD n, len = min(maxlen - off, BUFSIZ);
 +        SetLastError(0);
 +        BOOL res = ReadConsoleW(handle, &buf[off], len, &n, NULL);
 +
 +        if (!res) {
 +            err = GetLastError();
 +            break;
 +        }
 +        if (n == 0) {
 +            err = GetLastError();
 +            if (err != ERROR_OPERATION_ABORTED)
 +                break;
 +            err = 0;
 +            HANDLE hInterruptEvent = _PyOS_SigintEvent();
 +            if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE)
 +                    == WAIT_OBJECT_0) {
 +                ResetEvent(hInterruptEvent);
 +                Py_BLOCK_THREADS
 +                sig = PyErr_CheckSignals();
 +                Py_UNBLOCK_THREADS
 +                if (sig < 0)
 +                    break;
 +            }
 +        }
 +        *readlen += n;
 +
 +        /* If we didn't read a full buffer that time, don't try
 +           again or we will block a second time. */
 +        if (n < len)
 +            break;
 +        /* If the buffer ended with a newline, break out */
 +        if (buf[*readlen - 1] == '\n')
 +            break;
 +        /* If the buffer ends with a high surrogate, expand the
 +           buffer and read an extra character. */
 +        WORD char_type;
 +        if (off + BUFSIZ >= maxlen &&
 +            GetStringTypeW(CT_CTYPE3, &buf[*readlen - 1], 1, &char_type) &&
 +            char_type == C3_HIGHSURROGATE) {
 +            wchar_t *newbuf;
 +            maxlen += 1;
 +            Py_BLOCK_THREADS
 +            newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t));
 +            Py_UNBLOCK_THREADS
 +            if (!newbuf) {
 +                sig = -1;
 +                break;
 +            }
 +            buf = newbuf;
 +            /* Only advance by n and not BUFSIZ in this case */
 +            off += n;
 +            continue;
 +        }
 +
 +        off += BUFSIZ;
 +    }
 +
 +    Py_END_ALLOW_THREADS
 +
 +    if (sig)
 +        goto error;
 +    if (err) {
 +        PyErr_SetFromWindowsErr(err);
 +        goto error;
 +    }
 +
 +    if (*readlen > 0 && buf[0] == L'\x1a') {
 +        PyMem_Free(buf);
 +        buf = (wchar_t *)PyMem_Malloc(sizeof(wchar_t));
 +        if (!buf)
 +            goto error;
 +        buf[0] = L'\0';
 +        *readlen = 0;
 +    }
 +
 +    return buf;
 +
 +error:
 +    if (buf)
 +        PyMem_Free(buf);
 +    return NULL;
 +}
 +
 +
 +static Py_ssize_t
 +readinto(winconsoleio *self, char *buf, Py_ssize_t len)
 +{
 +    if (self->handle == INVALID_HANDLE_VALUE) {
 +        err_closed();
 +        return -1;
 +    }
 +    if (!self->readable) {
 +        err_mode("reading");
 +        return -1;
 +    }
 +    if (len == 0)
 +        return 0;
 +    if (len > BUFMAX) {
 +        PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX);
 +        return -1;
 +    }
 +
 +    /* Each character may take up to 4 bytes in the final buffer.
 +       This is highly conservative, but necessary to avoid
 +       failure for any given Unicode input (e.g. \U0010ffff).
 +       If the caller requests fewer than 4 bytes, we buffer one
 +       character.
 +    */
 +    DWORD wlen = (DWORD)(len / 4);
 +    if (wlen == 0) {
 +        wlen = 1;
 +    }
 +
 +    DWORD read_len = _copyfrombuf(self, buf, (DWORD)len);
 +    if (read_len) {
 +        buf = &buf[read_len];
 +        len -= read_len;
 +        wlen -= 1;
 +    }
 +    if (len == read_len || wlen == 0)
 +        return read_len;
 +
 +    DWORD n;
 +    wchar_t *wbuf = read_console_w(self->handle, wlen, &n);
 +    if (wbuf == NULL)
 +        return -1;
 +    if (n == 0) {
 +        PyMem_Free(wbuf);
 +        return read_len;
 +    }
 +
 +    int err = 0;
 +    DWORD u8n = 0;
 +
 +    Py_BEGIN_ALLOW_THREADS
 +    if (len < 4) {
 +        if (WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
 +                self->buf, sizeof(self->buf) / sizeof(self->buf[0]),
 +                NULL, NULL))
 +            u8n = _copyfrombuf(self, buf, (DWORD)len);
 +    } else {
 +        u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
 +            buf, (DWORD)len, NULL, NULL);
 +    }
 +
 +    if (u8n) {
 +        read_len += u8n;
 +        u8n = 0;
 +    } else {
 +        err = GetLastError();
 +        if (err == ERROR_INSUFFICIENT_BUFFER) {
 +            /* Calculate the needed buffer for a more useful error, as this
 +                means our "/ 4" logic above is insufficient for some input.
 +            */
 +            u8n = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
 +                NULL, 0, NULL, NULL);
 +        }
 +    }
 +    Py_END_ALLOW_THREADS
 +
 +    PyMem_Free(wbuf);
 +
 +    if (u8n) {
 +        PyErr_Format(PyExc_SystemError,
 +            "Buffer had room for %d bytes but %d bytes required",
 +            len, u8n);
 +        return -1;
 +    }
 +    if (err) {
 +        PyErr_SetFromWindowsErr(err);
 +        return -1;
 +    }
 +
 +    return read_len;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.readinto
 +    buffer: Py_buffer(accept={rwbuffer})
 +    /
 +
 +Same as RawIOBase.readinto().
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_readinto_impl(winconsoleio *self, Py_buffer *buffer)
 +/*[clinic end generated code: output=66d1bdfa3f20af39 input=4ed68da48a6baffe]*/
 +{
 +    Py_ssize_t len = readinto(self, buffer->buf, buffer->len);
 +    if (len < 0)
 +        return NULL;
 +
 +    return PyLong_FromSsize_t(len);
 +}
 +
 +static DWORD
 +new_buffersize(winconsoleio *self, DWORD currentsize)
 +{
 +    DWORD addend;
 +
 +    /* Expand the buffer by an amount proportional to the current size,
 +       giving us amortized linear-time behavior.  For bigger sizes, use a
 +       less-than-double growth factor to avoid excessive allocation. */
 +    if (currentsize > 65536)
 +        addend = currentsize >> 3;
 +    else
 +        addend = 256 + currentsize;
 +    if (addend < SMALLCHUNK)
 +        /* Avoid tiny read() calls. */
 +        addend = SMALLCHUNK;
 +    return addend + currentsize;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.readall
 +
 +Read all data from the console, returned as bytes.
 +
 +Return an empty bytes object at EOF.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_readall_impl(winconsoleio *self)
 +/*[clinic end generated code: output=e6d312c684f6e23b input=4024d649a1006e69]*/
 +{
 +    wchar_t *buf;
 +    DWORD bufsize, n, len = 0;
 +    PyObject *bytes;
 +    DWORD bytes_size, rn;
 +
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +
 +    bufsize = BUFSIZ;
 +
 +    buf = (wchar_t*)PyMem_Malloc((bufsize + 1) * sizeof(wchar_t));
 +    if (buf == NULL)
 +        return NULL;
 +
 +    while (1) {
 +        wchar_t *subbuf;
 +
 +        if (len >= (Py_ssize_t)bufsize) {
 +            DWORD newsize = new_buffersize(self, len);
 +            if (newsize > BUFMAX)
 +                break;
 +            if (newsize < bufsize) {
 +                PyErr_SetString(PyExc_OverflowError,
 +                                "unbounded read returned more bytes "
 +                                "than a Python bytes object can hold");
 +                PyMem_Free(buf);
 +                return NULL;
 +            }
 +            bufsize = newsize;
 +
 +            buf = PyMem_Realloc(buf, (bufsize + 1) * sizeof(wchar_t));
 +            if (!buf) {
 +                PyMem_Free(buf);
 +                return NULL;
 +            }
 +        }
 +
 +        subbuf = read_console_w(self->handle, bufsize - len, &n);
 +
 +        if (subbuf == NULL) {
 +            PyMem_Free(buf);
 +            return NULL;
 +        }
 +
 +        if (n > 0)
 +            wcsncpy_s(&buf[len], bufsize - len + 1, subbuf, n);
 +
 +        PyMem_Free(subbuf);
 +
 +        /* when the read is empty we break */
 +        if (n == 0)
 +            break;
 +
 +        len += n;
 +    }
 +
 +    if (len == 0 && _buflen(self) == 0) {
 +        /* when the result starts with ^Z we return an empty buffer */
 +        PyMem_Free(buf);
 +        return PyBytes_FromStringAndSize(NULL, 0);
 +    }
 +
 +    if (len) {
 +        Py_BEGIN_ALLOW_THREADS
 +        bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len,
 +            NULL, 0, NULL, NULL);
 +        Py_END_ALLOW_THREADS
 +
 +        if (!bytes_size) {
 +            DWORD err = GetLastError();
 +            PyMem_Free(buf);
 +            return PyErr_SetFromWindowsErr(err);
 +        }
 +    } else {
 +        bytes_size = 0;
 +    }
 +
 +    bytes_size += _buflen(self);
 +    bytes = PyBytes_FromStringAndSize(NULL, bytes_size);
 +    rn = _copyfrombuf(self, PyBytes_AS_STRING(bytes), bytes_size);
 +
 +    if (len) {
 +        Py_BEGIN_ALLOW_THREADS
 +        bytes_size = WideCharToMultiByte(CP_UTF8, 0, buf, len,
 +            &PyBytes_AS_STRING(bytes)[rn], bytes_size - rn, NULL, NULL);
 +        Py_END_ALLOW_THREADS
 +
 +        if (!bytes_size) {
 +            DWORD err = GetLastError();
 +            PyMem_Free(buf);
 +            Py_CLEAR(bytes);
 +            return PyErr_SetFromWindowsErr(err);
 +        }
 +
 +        /* add back the number of preserved bytes */
 +        bytes_size += rn;
 +    }
 +
 +    PyMem_Free(buf);
 +    if (bytes_size < (size_t)PyBytes_GET_SIZE(bytes)) {
 +        if (_PyBytes_Resize(&bytes, n * sizeof(wchar_t)) < 0) {
 +            Py_CLEAR(bytes);
 +            return NULL;
 +        }
 +    }
 +    return bytes;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.read
 +    size: io_ssize_t = -1
 +    /
 +
 +Read at most size bytes, returned as bytes.
 +
 +Only makes one system call when size is a positive integer,
 +so less data may be returned than requested.
 +Return an empty bytes object at EOF.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_read_impl(winconsoleio *self, Py_ssize_t size)
 +/*[clinic end generated code: output=57df68af9f4b22d0 input=6c56fceec460f1dd]*/
 +{
 +    PyObject *bytes;
 +    Py_ssize_t bytes_size;
 +
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +    if (!self->readable)
 +        return err_mode("reading");
 +
 +    if (size < 0)
 +        return _io__WindowsConsoleIO_readall_impl(self);
 +    if (size > BUFMAX) {
 +        PyErr_Format(PyExc_ValueError, "cannot read more than %d bytes", BUFMAX);
 +        return NULL;
 +    }
 +
 +    bytes = PyBytes_FromStringAndSize(NULL, size);
 +    if (bytes == NULL)
 +        return NULL;
 +
 +    bytes_size = readinto(self, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes));
 +    if (bytes_size < 0) {
 +        Py_CLEAR(bytes);
 +        return NULL;
 +    }
 +
 +    if (bytes_size < PyBytes_GET_SIZE(bytes)) {
 +        if (_PyBytes_Resize(&bytes, bytes_size) < 0) {
 +            Py_CLEAR(bytes);
 +            return NULL;
 +        }
 +    }
 +
 +    return bytes;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.write
 +    b: Py_buffer
 +    /
 +
 +Write buffer b to file, return number of bytes written.
 +
 +Only makes one system call, so not all of the data may be written.
 +The number of bytes actually written is returned.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_write_impl(winconsoleio *self, Py_buffer *b)
 +/*[clinic end generated code: output=775bdb16fbf9137b input=be35fb624f97c941]*/
 +{
 +    BOOL res = TRUE;
 +    wchar_t *wbuf;
 +    DWORD len, wlen, n = 0;
 +
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +    if (!self->writable)
 +        return err_mode("writing");
 +
 +    if (b->len > BUFMAX)
 +        len = BUFMAX;
 +    else
 +        len = (DWORD)b->len;
 +
 +    Py_BEGIN_ALLOW_THREADS
 +    wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0);
 +
 +    /* issue11395 there is an unspecified upper bound on how many bytes
 +       can be written at once. We cap at 32k - the caller will have to
 +       handle partial writes.
 +       Since we don't know how many input bytes are being ignored, we
 +       have to reduce and recalculate. */
 +    while (wlen > 32766 / sizeof(wchar_t)) {
 +        len /= 2;
 +        wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0);
 +    }
 +    Py_END_ALLOW_THREADS
 +
 +    if (!wlen)
 +        return PyErr_SetFromWindowsErr(0);
 +
 +    wbuf = (wchar_t*)PyMem_Malloc(wlen * sizeof(wchar_t));
 +
 +    Py_BEGIN_ALLOW_THREADS
 +    wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, wbuf, wlen);
 +    if (wlen) {
 +        res = WriteConsoleW(self->handle, wbuf, wlen, &n, NULL);
 +        if (n < wlen) {
 +            /* Wrote fewer characters than expected, which means our
 +             * len value may be wrong. So recalculate it from the
 +             * characters that were written. As this could potentially
 +             * result in a different value, we also validate that value.
 +             */
 +            len = WideCharToMultiByte(CP_UTF8, 0, wbuf, n,
 +                NULL, 0, NULL, NULL);
 +            if (len) {
 +                wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len,
 +                    NULL, 0);
 +                assert(wlen == len);
 +            }
 +        }
 +    } else
 +        res = 0;
 +    Py_END_ALLOW_THREADS
 +
 +    if (!res) {
 +        DWORD err = GetLastError();
 +        PyMem_Free(wbuf);
 +        return PyErr_SetFromWindowsErr(err);
 +    }
 +
 +    PyMem_Free(wbuf);
 +    return PyLong_FromSsize_t(len);
 +}
 +
 +static PyObject *
 +winconsoleio_repr(winconsoleio *self)
 +{
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return PyUnicode_FromFormat("<_io._WindowsConsoleIO [closed]>");
 +
 +    if (self->readable)
 +        return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='rb' closefd=%s>",
 +            self->closehandle ? "True" : "False");
 +    if (self->writable)
 +        return PyUnicode_FromFormat("<_io._WindowsConsoleIO mode='wb' closefd=%s>",
 +            self->closehandle ? "True" : "False");
 +
 +    PyErr_SetString(PyExc_SystemError, "_WindowsConsoleIO has invalid mode");
 +    return NULL;
 +}
 +
 +/*[clinic input]
 +_io._WindowsConsoleIO.isatty
 +
 +Always True.
 +[clinic start generated code]*/
 +
 +static PyObject *
 +_io__WindowsConsoleIO_isatty_impl(winconsoleio *self)
 +/*[clinic end generated code: output=9eac09d287c11bd7 input=9b91591dbe356f86]*/
 +{
 +    if (self->handle == INVALID_HANDLE_VALUE)
 +        return err_closed();
 +
 +    Py_RETURN_TRUE;
 +}
 +
 +static PyObject *
 +winconsoleio_getstate(winconsoleio *self)
 +{
 +    PyErr_Format(PyExc_TypeError,
 +                 "cannot serialize '%s' object", Py_TYPE(self)->tp_name);
 +    return NULL;
 +}
 +
 +#include "clinic/winconsoleio.c.h"
 +
 +static PyMethodDef winconsoleio_methods[] = {
 +    _IO__WINDOWSCONSOLEIO_READ_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_READALL_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_READINTO_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_WRITE_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_READABLE_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_WRITABLE_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_FILENO_METHODDEF
 +    _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF
 +    {"__getstate__", (PyCFunction)winconsoleio_getstate, METH_NOARGS, NULL},
 +    {NULL,           NULL}             /* sentinel */
 +};
 +
 +/* 'closed' and 'mode' are attributes for compatibility with FileIO. */
 +
 +static PyObject *
 +get_closed(winconsoleio *self, void *closure)
 +{
 +    return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE));
 +}
 +
 +static PyObject *
 +get_closefd(winconsoleio *self, void *closure)
 +{
 +    return PyBool_FromLong((long)(self->closehandle));
 +}
 +
 +static PyObject *
 +get_mode(winconsoleio *self, void *closure)
 +{
 +    return PyUnicode_FromString(self->readable ? "rb" : "wb");
 +}
 +
 +static PyGetSetDef winconsoleio_getsetlist[] = {
 +    {"closed", (getter)get_closed, NULL, "True if the file is closed"},
 +    {"closefd", (getter)get_closefd, NULL,
 +        "True if the file descriptor will be closed by close()."},
 +    {"mode", (getter)get_mode, NULL, "String giving the file mode"},
 +    {NULL},
 +};
 +
 +static PyMemberDef winconsoleio_members[] = {
 +    {"_blksize", T_UINT, offsetof(winconsoleio, blksize), 0},
 +    {"_finalizing", T_BOOL, offsetof(winconsoleio, finalizing), 0},
 +    {NULL}
 +};
 +
 +PyTypeObject PyWindowsConsoleIO_Type = {
 +    PyVarObject_HEAD_INIT(NULL, 0)
 +    "_io._WindowsConsoleIO",
 +    sizeof(winconsoleio),
 +    0,
 +    (destructor)winconsoleio_dealloc,           /* tp_dealloc */
 +    0,                                          /* tp_print */
 +    0,                                          /* tp_getattr */
 +    0,                                          /* tp_setattr */
 +    0,                                          /* tp_reserved */
 +    (reprfunc)winconsoleio_repr,                /* tp_repr */
 +    0,                                          /* tp_as_number */
 +    0,                                          /* tp_as_sequence */
 +    0,                                          /* tp_as_mapping */
 +    0,                                          /* tp_hash */
 +    0,                                          /* tp_call */
 +    0,                                          /* tp_str */
 +    PyObject_GenericGetAttr,                    /* tp_getattro */
 +    0,                                          /* tp_setattro */
 +    0,                                          /* tp_as_buffer */
 +    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE
 +        | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,       /* tp_flags */
 +    _io__WindowsConsoleIO___init____doc__,      /* tp_doc */
 +    (traverseproc)winconsoleio_traverse,        /* tp_traverse */
 +    (inquiry)winconsoleio_clear,                /* tp_clear */
 +    0,                                          /* tp_richcompare */
 +    offsetof(winconsoleio, weakreflist),        /* tp_weaklistoffset */
 +    0,                                          /* tp_iter */
 +    0,                                          /* tp_iternext */
 +    winconsoleio_methods,                       /* tp_methods */
 +    winconsoleio_members,                       /* tp_members */
 +    winconsoleio_getsetlist,                    /* tp_getset */
 +    0,                                          /* tp_base */
 +    0,                                          /* tp_dict */
 +    0,                                          /* tp_descr_get */
 +    0,                                          /* tp_descr_set */
 +    offsetof(winconsoleio, dict),               /* tp_dictoffset */
 +    _io__WindowsConsoleIO___init__,             /* tp_init */
 +    PyType_GenericAlloc,                        /* tp_alloc */
 +    winconsoleio_new,                           /* tp_new */
 +    PyObject_GC_Del,                            /* tp_free */
 +    0,                                          /* tp_is_gc */
 +    0,                                          /* tp_bases */
 +    0,                                          /* tp_mro */
 +    0,                                          /* tp_cache */
 +    0,                                          /* tp_subclasses */
 +    0,                                          /* tp_weaklist */
 +    0,                                          /* tp_del */
 +    0,                                          /* tp_version_tag */
 +    0,                                          /* tp_finalize */
 +};
 +
 +PyAPI_DATA(PyObject *) _PyWindowsConsoleIO_Type = (PyObject*)&PyWindowsConsoleIO_Type;
 +
 +#endif /* MS_WINDOWS */
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
Simple merge
diff --cc Python/ast.c
index 14bcdb1b0a11a5b46c96f478a64a62b94163345e,5b7ed7c035eb7bc74c352d0dae595b220bfe1075..33b7df64ec0c68e3d30d658ccb24c01cd662bd6e
@@@ -938,32 -880,10 +938,32 @@@ forbidden_name(struct compiling *c, ide
          ast_error(c, n, "assignment to keyword");
          return 1;
      }
-     if (PyUnicode_CompareWithASCIIString(name, "async") == 0 ||
-         PyUnicode_CompareWithASCIIString(name, "await") == 0)
++    if (_PyUnicode_EqualToASCIIString(name, "async") ||
++        _PyUnicode_EqualToASCIIString(name, "await"))
 +    {
 +        PyObject *message = PyUnicode_FromString(
 +            "'async' and 'await' will become reserved keywords"
 +            " in Python 3.7");
 +        int ret;
 +        if (message == NULL) {
 +            return 1;
 +        }
 +        ret = PyErr_WarnExplicitObject(
 +                PyExc_DeprecationWarning,
 +                message,
 +                c->c_filename,
 +                LINENO(n),
 +                NULL,
 +                NULL);
 +        Py_DECREF(message);
 +        if (ret < 0) {
 +            return 1;
 +        }
 +    }
      if (full_checks) {
 -        const char **p;
 +        const char * const *p;
          for (p = FORBIDDEN; *p; p++) {
-             if (PyUnicode_CompareWithASCIIString(name, *p) == 0) {
+             if (_PyUnicode_EqualToASCIIString(name, *p)) {
                  ast_error(c, n, "assignment to keyword");
                  return 1;
              }
Simple merge
diff --cc Python/future.c
Simple merge
index 43656eb2aaef352c23e1f9900b5a8aed799c2463,b10e7769e87795d92c1cc2c5d977415162245e88..616c6eb107377fc2dfbc9289e09b9671fd25cf04
@@@ -1808,7 -1618,7 +1808,7 @@@ vgetargskeywords(PyObject *args, PyObje
                  return cleanreturn(0, &freelist);
              }
              for (i = 0; i < len; i++) {
-                 if (*kwlist[i] && !PyUnicode_CompareWithASCIIString(key, kwlist[i])) {
 -                if (_PyUnicode_EqualToASCIIString(key, kwlist[i])) {
++                if (*kwlist[i] && _PyUnicode_EqualToASCIIString(key, kwlist[i])) {
                      match = 1;
                      break;
                  }
diff --cc Python/import.c
index dfdd9409e5e119fe19fadfc09fb01d4a0a523733,3579273ae0fefe1e8283c9e6a005a9ea9f856401..cd865a5423666ed32959c62534bf38167f697314
@@@ -1324,10 -1325,11 +1319,9 @@@ remove_importlib_frames(void
  
          if (in_importlib &&
              (always_trim ||
-              PyUnicode_CompareWithASCIIString(code->co_name,
-                                               remove_frames) == 0)) {
+              _PyUnicode_EqualToASCIIString(code->co_name, remove_frames))) {
 -            PyObject *tmp = *outer_link;
 -            *outer_link = next;
              Py_XINCREF(next);
 -            Py_DECREF(tmp);
 +            Py_XSETREF(*outer_link, next);
              prev_link = outer_link;
          }
          else {
Simple merge