Issue #21932: os.read() now uses a :c:func:`Py_ssize_t` type instead of

:c:type:`int` for the size to support reading more than 2 GB at once. On
Windows, the size is truncted to INT_MAX. As any call to os.read(), the OS
may read less bytes than the number of requested bytes.

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index f559841ccf6541e308036e295bae9d1213756546..588df1696fd6af8dd21ed5369aefa477b15ae714 100644 (file)
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -43,6 +43,10 @@ try:
     import _winapi
 except ImportError:
     _winapi = None
+try:
+    from _testcapi import INT_MAX
+except ImportError:
+    INT_MAX = 2 ** 31 - 1
 
 from test.script_helper import assert_python_ok
 
@@ -119,6 +123,21 @@ class FileTests(unittest.TestCase):
             self.assertEqual(type(s), bytes)
             self.assertEqual(s, b"spam")
 
+    def test_large_read(self):
+        with open(support.TESTFN, "wb") as fp:
+            fp.write(b'test')
+        self.addCleanup(support.unlink, support.TESTFN)
+
+        # Issue #21932: Make sure that os.read() does not raise an
+        # OverflowError for size larger than INT_MAX
+        size = INT_MAX + 10
+        with open(support.TESTFN, "rb") as fp:
+            data = os.read(fp.fileno(), size)
+
+        # The test does not try to read more than 2 GB at once because the
+        # operating system is free to return less bytes than requested.
+        self.assertEqual(data, b'test')
+
     def test_write(self):
         # os.write() accepts bytes- and buffer-like objects but not strings
         fd = os.open(support.TESTFN, os.O_CREAT | os.O_WRONLY)

diff --git a/Misc/NEWS b/Misc/NEWS
index d651ffd4e4fbd05424f9a44f82dc0c5a9c64684b..16f0e6d43444d2c9456d8a9e9493655d4740723c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -108,6 +108,11 @@ Core and Builtins
 Library
 -------
 
+- Issue #21932: os.read() now uses a :c:func:`Py_ssize_t` type instead of
+  :c:type:`int` for the size to support reading more than 2 GB at once. On
+  Windows, the size is truncted to INT_MAX. As any call to os.read(), the OS
+  may read less bytes than the number of requested bytes.
+
 - Issue #21942: Fixed source file viewing in pydoc's server mode on Windows.
 
 - Issue #11259: asynchat.async_chat().set_terminator() now raises a ValueError

diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 899618f2665b9bd8974417a5408a37006b7df76a..b7acbc330c7c8e2f247dbb6bc848464489d6d72c 100644 (file)
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -7989,11 +7989,18 @@ Read a file descriptor.");
 static PyObject *
 posix_read(PyObject *self, PyObject *args)
 {
-    int fd, size;
+    int fd;
+    Py_ssize_t size;
     Py_ssize_t n;
     PyObject *buffer;
-    if (!PyArg_ParseTuple(args, "ii:read", &fd, &size))
+    if (!PyArg_ParseTuple(args, "in:read", &fd, &size))
         return NULL;
+    if (!_PyVerify_fd(fd))
+        return posix_error();
+#ifdef MS_WINDOWS
+    if (size > INT_MAX)
+        size = INT_MAX;
+#endif
     if (size < 0) {
         errno = EINVAL;
         return posix_error();
@@ -8001,12 +8008,12 @@ posix_read(PyObject *self, PyObject *args)
     buffer = PyBytes_FromStringAndSize((char *)NULL, size);
     if (buffer == NULL)
         return NULL;
-    if (!_PyVerify_fd(fd)) {
-        Py_DECREF(buffer);
-        return posix_error();
-    }
     Py_BEGIN_ALLOW_THREADS
+#ifdef MS_WINDOWS
+    n = read(fd, PyBytes_AS_STRING(buffer), (int)size);
+#else
     n = read(fd, PyBytes_AS_STRING(buffer), size);
+#endif
     Py_END_ALLOW_THREADS
     if (n < 0) {
         Py_DECREF(buffer);