gh-121313: multiprocessing: simplify by increasing the connection buffer size to...

Increases the multiprocessing connection buffer size from 8k to 64k for efficiency, without overallocating.

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
Co-authored-by: Victor Stinner <vstinner@python.org>

diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index d84b52fe6d4f888cd9d25a12cd6d698d4e63f7ec..7e901cf2fb9852b4fa7a60f22d47957eb459aa0e 100644 (file)
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -13,7 +13,6 @@ import errno
 import io
 import itertools
 import os
-import stat
 import sys
 import socket
 import struct
@@ -40,7 +39,9 @@ except ImportError:
 #
 #
 
-BUFSIZE = 8192
+# 64 KiB is the default PIPE buffer size of most POSIX platforms.
+BUFSIZE = 64 * 1024
+
 # A very generous timeout when it comes to local connections...
 CONNECTION_TIMEOUT = 20.
 
@@ -361,11 +362,6 @@ if _winapi:
             f.write(ov.getbuffer())
             return f
 
-"""
-The default size of a pipe on Linux systems is 16 times the base page size:
-https://man7.org/linux/man-pages/man7/pipe.7.html
-"""
-PAGES_PER_PIPE = 16
 
 class Connection(_ConnectionBase):
     """
@@ -378,14 +374,11 @@ class Connection(_ConnectionBase):
             _close(self._handle)
         _write = _multiprocessing.send
         _read = _multiprocessing.recv
-        _default_pipe_size = 0
     else:
         def _close(self, _close=os.close):
             _close(self._handle)
         _write = os.write
         _read = os.read
-        _base_page_size = os.sysconf(os.sysconf_names['SC_PAGESIZE'])
-        _default_pipe_size = _base_page_size * PAGES_PER_PIPE
 
     def _send(self, buf, write=_write):
         remaining = len(buf)
@@ -400,13 +393,8 @@ class Connection(_ConnectionBase):
         buf = io.BytesIO()
         handle = self._handle
         remaining = size
-        is_pipe = False
-        if size > self._default_pipe_size > 0:
-            mode = os.fstat(handle).st_mode
-            is_pipe = stat.S_ISFIFO(mode)
-        limit = self._default_pipe_size if is_pipe else remaining
         while remaining > 0:
-            to_read = min(limit, remaining)
+            to_read = min(BUFSIZE, remaining)
             chunk = read(handle, to_read)
             n = len(chunk)
             if n == 0:

diff --git a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
deleted file mode 100644 (file)
index 06abce9..0000000
--- a/Misc/NEWS.d/next/C API/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
+++ /dev/null
@@ -1 +0,0 @@
-Limit reading size in multiprocessing connection._recv for pipes to default pipe size of 16 times base page size, in order to avoid memory overallocation and unnecessary memory management system calls.

diff --git a/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
new file mode 100644 (file)
index 0000000..bb41063
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-07-03-10-11-53.gh-issue-121313.D7gARW.rst
@@ -0,0 +1 @@
+Limit the reading size in the :class:`multiprocessing.connection.Connection` class to 64 KiB to prevent memory overallocation and unnecessary memory management system calls.