From 58f3fe0d9b9882656e629e8caab687c7fcb21b36 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 18 Nov 2025 01:10:32 -0800 Subject: [PATCH] gh-129005: Remove copies from _pyio using take_bytes (#141539) Memory usage now matches that of _io for large files. --- Lib/_pyio.py | 8 ++++---- Lib/test/test_io/test_bufferedio.py | 3 ++- Lib/test/test_io/test_largefile.py | 6 ++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 423178e87a86..69a088df8fc9 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -546,7 +546,7 @@ class IOBase(metaclass=abc.ABCMeta): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -620,7 +620,7 @@ class RawIOBase(IOBase): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -628,7 +628,7 @@ class RawIOBase(IOBase): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -1738,7 +1738,7 @@ class FileIO(RawIOBase): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 30c34e818b15..3278665bdc9d 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1277,7 +1277,8 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest): def _readinto(bufio, n=-1): b = bytearray(n if n >= 0 else 9999) n = bufio.readinto(b) - return bytes(b[:n]) + b.resize(n) + return b.take_bytes() self.check_flush_and_read(_readinto) def test_flush_and_peek(self): diff --git a/Lib/test/test_io/test_largefile.py b/Lib/test/test_io/test_largefile.py index 41f7b70e5cfe..438a90a92ed5 100644 --- a/Lib/test/test_io/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ class TestFileMethods(LargeFileTest): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: -- 2.47.3