]> git.ipfire.org Git - thirdparty/Python/cpython.git/commitdiff
gh-89240: Enable multiprocessing on Windows to use large process pools (GH-107873)
authorSteve Dower <steve.dower@python.org>
Tue, 13 Feb 2024 00:28:35 +0000 (00:28 +0000)
committerGitHub <noreply@github.com>
Tue, 13 Feb 2024 00:28:35 +0000 (00:28 +0000)
We add _winapi.BatchedWaitForMultipleObjects to wait for larger numbers of handles.
This is an internal module, hence undocumented, and should be used with caution.
Check the docstring for info before using BatchedWaitForMultipleObjects.

12 files changed:
Include/internal/pycore_global_objects_fini_generated.h
Include/internal/pycore_global_strings.h
Include/internal/pycore_runtime_init_generated.h
Include/internal/pycore_unicodeobject_generated.h
Lib/multiprocessing/connection.py
Lib/test/_test_multiprocessing.py
Lib/test/test_winapi.py [new file with mode: 0644]
Misc/NEWS.d/next/Windows/2023-08-11-18-21-38.gh-issue-89240.dtSOLG.rst [new file with mode: 0644]
Modules/_winapi.c
Modules/clinic/_winapi.c.h
Objects/exceptions.c
PC/errmap.h

index 932738c3049882b9f209a2721da4013b33fbbd2f..11755210d65432dbc9fa666af93678823751550c 100644 (file)
@@ -883,6 +883,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(defaultaction));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(delete));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(depth));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(desired_access));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(detect_types));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device));
@@ -973,6 +974,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(groups));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(h));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(handle_seq));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hash_name));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(header));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers));
@@ -990,9 +992,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(indexgroup));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inf));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(infer_variance));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inherit_handle));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inheritable));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_bytes));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_owner));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_state));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initial_value));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(initval));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(inner_size));
@@ -1048,6 +1053,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(locals));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(logoption));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(loop));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(manual_reset));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mapping));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length));
@@ -1064,6 +1070,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(metadata));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(method));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(microsecond));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode));
@@ -1073,6 +1080,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(month));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mro));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(msg));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mutex));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mycmp));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(n_arg));
@@ -1176,6 +1184,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(second));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(security_attributes));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seek));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(seekable));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(selectors));
@@ -1263,6 +1272,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) {
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume));
+    _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wait_all));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions));
     _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits));
index da62b4f0a951ff877e171d8e85cd38d6eb1e3ef3..576ac703ca15081e90eaf8981c4e01f4b86efed9 100644 (file)
@@ -372,6 +372,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(defaultaction)
         STRUCT_FOR_ID(delete)
         STRUCT_FOR_ID(depth)
+        STRUCT_FOR_ID(desired_access)
         STRUCT_FOR_ID(detect_types)
         STRUCT_FOR_ID(deterministic)
         STRUCT_FOR_ID(device)
@@ -462,6 +463,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(groups)
         STRUCT_FOR_ID(h)
         STRUCT_FOR_ID(handle)
+        STRUCT_FOR_ID(handle_seq)
         STRUCT_FOR_ID(hash_name)
         STRUCT_FOR_ID(header)
         STRUCT_FOR_ID(headers)
@@ -479,9 +481,12 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(indexgroup)
         STRUCT_FOR_ID(inf)
         STRUCT_FOR_ID(infer_variance)
+        STRUCT_FOR_ID(inherit_handle)
         STRUCT_FOR_ID(inheritable)
         STRUCT_FOR_ID(initial)
         STRUCT_FOR_ID(initial_bytes)
+        STRUCT_FOR_ID(initial_owner)
+        STRUCT_FOR_ID(initial_state)
         STRUCT_FOR_ID(initial_value)
         STRUCT_FOR_ID(initval)
         STRUCT_FOR_ID(inner_size)
@@ -537,6 +542,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(locals)
         STRUCT_FOR_ID(logoption)
         STRUCT_FOR_ID(loop)
+        STRUCT_FOR_ID(manual_reset)
         STRUCT_FOR_ID(mapping)
         STRUCT_FOR_ID(match)
         STRUCT_FOR_ID(max_length)
@@ -553,6 +559,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(metadata)
         STRUCT_FOR_ID(method)
         STRUCT_FOR_ID(microsecond)
+        STRUCT_FOR_ID(milliseconds)
         STRUCT_FOR_ID(minute)
         STRUCT_FOR_ID(mod)
         STRUCT_FOR_ID(mode)
@@ -562,6 +569,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(month)
         STRUCT_FOR_ID(mro)
         STRUCT_FOR_ID(msg)
+        STRUCT_FOR_ID(mutex)
         STRUCT_FOR_ID(mycmp)
         STRUCT_FOR_ID(n)
         STRUCT_FOR_ID(n_arg)
@@ -665,6 +673,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(sched_priority)
         STRUCT_FOR_ID(scheduler)
         STRUCT_FOR_ID(second)
+        STRUCT_FOR_ID(security_attributes)
         STRUCT_FOR_ID(seek)
         STRUCT_FOR_ID(seekable)
         STRUCT_FOR_ID(selectors)
@@ -752,6 +761,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(values)
         STRUCT_FOR_ID(version)
         STRUCT_FOR_ID(volume)
+        STRUCT_FOR_ID(wait_all)
         STRUCT_FOR_ID(warnings)
         STRUCT_FOR_ID(warnoptions)
         STRUCT_FOR_ID(wbits)
index 68fbbcb4378e17fde06d890725a0715502aa20e5..e682c97e7c0248d5cb8b61a837f0e64eff06e2ad 100644 (file)
@@ -881,6 +881,7 @@ extern "C" {
     INIT_ID(defaultaction), \
     INIT_ID(delete), \
     INIT_ID(depth), \
+    INIT_ID(desired_access), \
     INIT_ID(detect_types), \
     INIT_ID(deterministic), \
     INIT_ID(device), \
@@ -971,6 +972,7 @@ extern "C" {
     INIT_ID(groups), \
     INIT_ID(h), \
     INIT_ID(handle), \
+    INIT_ID(handle_seq), \
     INIT_ID(hash_name), \
     INIT_ID(header), \
     INIT_ID(headers), \
@@ -988,9 +990,12 @@ extern "C" {
     INIT_ID(indexgroup), \
     INIT_ID(inf), \
     INIT_ID(infer_variance), \
+    INIT_ID(inherit_handle), \
     INIT_ID(inheritable), \
     INIT_ID(initial), \
     INIT_ID(initial_bytes), \
+    INIT_ID(initial_owner), \
+    INIT_ID(initial_state), \
     INIT_ID(initial_value), \
     INIT_ID(initval), \
     INIT_ID(inner_size), \
@@ -1046,6 +1051,7 @@ extern "C" {
     INIT_ID(locals), \
     INIT_ID(logoption), \
     INIT_ID(loop), \
+    INIT_ID(manual_reset), \
     INIT_ID(mapping), \
     INIT_ID(match), \
     INIT_ID(max_length), \
@@ -1062,6 +1068,7 @@ extern "C" {
     INIT_ID(metadata), \
     INIT_ID(method), \
     INIT_ID(microsecond), \
+    INIT_ID(milliseconds), \
     INIT_ID(minute), \
     INIT_ID(mod), \
     INIT_ID(mode), \
@@ -1071,6 +1078,7 @@ extern "C" {
     INIT_ID(month), \
     INIT_ID(mro), \
     INIT_ID(msg), \
+    INIT_ID(mutex), \
     INIT_ID(mycmp), \
     INIT_ID(n), \
     INIT_ID(n_arg), \
@@ -1174,6 +1182,7 @@ extern "C" {
     INIT_ID(sched_priority), \
     INIT_ID(scheduler), \
     INIT_ID(second), \
+    INIT_ID(security_attributes), \
     INIT_ID(seek), \
     INIT_ID(seekable), \
     INIT_ID(selectors), \
@@ -1261,6 +1270,7 @@ extern "C" {
     INIT_ID(values), \
     INIT_ID(version), \
     INIT_ID(volume), \
+    INIT_ID(wait_all), \
     INIT_ID(warnings), \
     INIT_ID(warnoptions), \
     INIT_ID(wbits), \
index c8458b4e36ccc93ad2c1cc38cc36be01530af1f9..739af0e73c23ff86fdd9902524c72398be6c1f2f 100644 (file)
@@ -957,6 +957,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(depth);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(desired_access);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(detect_types);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1227,6 +1230,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(handle);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(handle_seq);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(hash_name);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1278,6 +1284,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(infer_variance);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(inherit_handle);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(inheritable);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1287,6 +1296,12 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(initial_bytes);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(initial_owner);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(initial_state);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(initial_value);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1452,6 +1467,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(loop);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(manual_reset);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(mapping);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1500,6 +1518,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(microsecond);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(milliseconds);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(minute);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1527,6 +1548,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(msg);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(mutex);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(mycmp);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -1836,6 +1860,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(second);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(security_attributes);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(seek);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
@@ -2097,6 +2124,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) {
     string = &_Py_ID(volume);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
+    string = &_Py_ID(wait_all);
+    assert(_PyUnicode_CheckConsistency(string, 1));
+    _PyUnicode_InternInPlace(interp, &string);
     string = &_Py_ID(warnings);
     assert(_PyUnicode_CheckConsistency(string, 1));
     _PyUnicode_InternInPlace(interp, &string);
index c6a66a1bc963c3acae906682be4dca41d9efcbb9..58d697fdecacc05750535e65b660299f9c6bfcab 100644 (file)
@@ -1011,8 +1011,20 @@ if sys.platform == 'win32':
         # returning the first signalled might create starvation issues.)
         L = list(handles)
         ready = []
+        # Windows limits WaitForMultipleObjects at 64 handles, and we use a
+        # few for synchronisation, so we switch to batched waits at 60.
+        if len(L) > 60:
+            try:
+                res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
+            except TimeoutError:
+                return []
+            ready.extend(L[i] for i in res)
+            if res:
+                L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
+            timeout = 0
         while L:
-            res = _winapi.WaitForMultipleObjects(L, False, timeout)
+            short_L = L[:60] if len(L) > 60 else L
+            res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
             if res == WAIT_TIMEOUT:
                 break
             elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
index c0d3ca50f17d69d3da4d914f1af44bbaeeb31479..94ce85cac754ae5632b5f8876f3207e66876e865 100644 (file)
@@ -6113,6 +6113,24 @@ class MiscTestCase(unittest.TestCase):
         self.assertEqual(rc, 0)
         self.assertFalse(err, msg=err.decode('utf-8'))
 
+    def test_large_pool(self):
+        #
+        # gh-89240: Check that large pools are always okay
+        #
+        testfn = os_helper.TESTFN
+        self.addCleanup(os_helper.unlink, testfn)
+        with open(testfn, 'w', encoding='utf-8') as f:
+            f.write(textwrap.dedent('''\
+                import multiprocessing
+                def f(x): return x*x
+                if __name__ == '__main__':
+                    with multiprocessing.Pool(200) as p:
+                        print(sum(p.map(f, range(1000))))
+            '''))
+        rc, out, err = script_helper.assert_python_ok(testfn)
+        self.assertEqual("332833500", out.decode('utf-8').strip())
+        self.assertFalse(err, msg=err.decode('utf-8'))
+
 
 #
 # Mixins
diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py
new file mode 100644 (file)
index 0000000..014aeea
--- /dev/null
@@ -0,0 +1,94 @@
+# Test the Windows-only _winapi module
+
+import random
+import threading
+import time
+import unittest
+from test.support import import_helper
+
+_winapi = import_helper.import_module('_winapi', required_on=['win'])
+
+MAXIMUM_WAIT_OBJECTS = 64
+MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2
+
+class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase):
+    def _events_waitall_test(self, n):
+        evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)]
+
+        with self.assertRaises(TimeoutError):
+            _winapi.BatchedWaitForMultipleObjects(evts, True, 100)
+
+        # Ensure no errors raised when all are triggered
+        for e in evts:
+            _winapi.SetEvent(e)
+        try:
+            _winapi.BatchedWaitForMultipleObjects(evts, True, 100)
+        except TimeoutError:
+            self.fail("expected wait to complete immediately")
+
+        # Choose 8 events to set, distributed throughout the list, to make sure
+        # we don't always have them in the first chunk
+        chosen = [i * (len(evts) // 8) for i in range(8)]
+
+        # Replace events with invalid handles to make sure we fail
+        for i in chosen:
+            old_evt = evts[i]
+            evts[i] = -1
+            with self.assertRaises(OSError):
+                _winapi.BatchedWaitForMultipleObjects(evts, True, 100)
+            evts[i] = old_evt
+
+
+    def _events_waitany_test(self, n):
+        evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)]
+
+        with self.assertRaises(TimeoutError):
+            _winapi.BatchedWaitForMultipleObjects(evts, False, 100)
+
+        # Choose 8 events to set, distributed throughout the list, to make sure
+        # we don't always have them in the first chunk
+        chosen = [i * (len(evts) // 8) for i in range(8)]
+
+        # Trigger one by one. They are auto-reset events, so will only trigger once
+        for i in chosen:
+            with self.subTest(f"trigger event {i} of {len(evts)}"):
+                _winapi.SetEvent(evts[i])
+                triggered = _winapi.BatchedWaitForMultipleObjects(evts, False, 10000)
+                self.assertSetEqual(set(triggered), {i})
+
+        # Trigger all at once. This may require multiple calls
+        for i in chosen:
+            _winapi.SetEvent(evts[i])
+        triggered = set()
+        while len(triggered) < len(chosen):
+            triggered.update(_winapi.BatchedWaitForMultipleObjects(evts, False, 10000))
+        self.assertSetEqual(triggered, set(chosen))
+
+        # Replace events with invalid handles to make sure we fail
+        for i in chosen:
+            with self.subTest(f"corrupt event {i} of {len(evts)}"):
+                old_evt = evts[i]
+                evts[i] = -1
+                with self.assertRaises(OSError):
+                    _winapi.BatchedWaitForMultipleObjects(evts, False, 100)
+                evts[i] = old_evt
+
+
+    def test_few_events_waitall(self):
+        self._events_waitall_test(16)
+
+    def test_many_events_waitall(self):
+        self._events_waitall_test(256)
+
+    def test_max_events_waitall(self):
+        self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS)
+
+
+    def test_few_events_waitany(self):
+        self._events_waitany_test(16)
+
+    def test_many_events_waitany(self):
+        self._events_waitany_test(256)
+
+    def test_max_events_waitany(self):
+        self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS)
diff --git a/Misc/NEWS.d/next/Windows/2023-08-11-18-21-38.gh-issue-89240.dtSOLG.rst b/Misc/NEWS.d/next/Windows/2023-08-11-18-21-38.gh-issue-89240.dtSOLG.rst
new file mode 100644 (file)
index 0000000..8ffe328
--- /dev/null
@@ -0,0 +1 @@
+Allows :mod:`multiprocessing` to create pools of greater than 62 processes.
index 5e5eb123c4ccfffc44492cfac2845f545073c068..83a4ccd4802ae01eb718bdc8c4c228a6e226fa99 100644 (file)
@@ -438,6 +438,39 @@ _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle,
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+_winapi.CreateEventW -> HANDLE
+
+    security_attributes: LPSECURITY_ATTRIBUTES
+    manual_reset: BOOL
+    initial_state: BOOL
+    name: LPCWSTR(accept={str, NoneType})
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_CreateEventW_impl(PyObject *module,
+                          LPSECURITY_ATTRIBUTES security_attributes,
+                          BOOL manual_reset, BOOL initial_state,
+                          LPCWSTR name)
+/*[clinic end generated code: output=2d4c7d5852ecb298 input=4187cee28ac763f8]*/
+{
+    HANDLE handle;
+
+    if (PySys_Audit("_winapi.CreateEventW", "bbu", manual_reset, initial_state, name) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = CreateEventW(security_attributes, manual_reset, initial_state, name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == INVALID_HANDLE_VALUE) {
+        PyErr_SetFromWindowsErr(0);
+    }
+
+    return handle;
+}
+
 /*[clinic input]
 _winapi.CreateFile -> HANDLE
 
@@ -674,6 +707,37 @@ cleanup:
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+_winapi.CreateMutexW -> HANDLE
+
+    security_attributes: LPSECURITY_ATTRIBUTES
+    initial_owner: BOOL
+    name: LPCWSTR(accept={str, NoneType})
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_CreateMutexW_impl(PyObject *module,
+                          LPSECURITY_ATTRIBUTES security_attributes,
+                          BOOL initial_owner, LPCWSTR name)
+/*[clinic end generated code: output=31b9ee8fc37e49a5 input=7d54b921e723254a]*/
+{
+    HANDLE handle;
+
+    if (PySys_Audit("_winapi.CreateMutexW", "bu", initial_owner, name) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = CreateMutexW(security_attributes, initial_owner, name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == INVALID_HANDLE_VALUE) {
+        PyErr_SetFromWindowsErr(0);
+    }
+
+    return handle;
+}
+
 /*[clinic input]
 _winapi.CreateNamedPipe -> HANDLE
 
@@ -1590,6 +1654,67 @@ _winapi_UnmapViewOfFile_impl(PyObject *module, LPCVOID address)
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+_winapi.OpenEventW -> HANDLE
+
+    desired_access: DWORD
+    inherit_handle: BOOL
+    name: LPCWSTR
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access,
+                        BOOL inherit_handle, LPCWSTR name)
+/*[clinic end generated code: output=c4a45e95545a4bd2 input=dec26598748d35aa]*/
+{
+    HANDLE handle;
+
+    if (PySys_Audit("_winapi.OpenEventW", "Iu", desired_access, name) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = OpenEventW(desired_access, inherit_handle, name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == INVALID_HANDLE_VALUE) {
+        PyErr_SetFromWindowsErr(0);
+    }
+
+    return handle;
+}
+
+
+/*[clinic input]
+_winapi.OpenMutexW -> HANDLE
+
+    desired_access: DWORD
+    inherit_handle: BOOL
+    name: LPCWSTR
+[clinic start generated code]*/
+
+static HANDLE
+_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access,
+                        BOOL inherit_handle, LPCWSTR name)
+/*[clinic end generated code: output=dda39d7844397bf0 input=f3a7b466c5307712]*/
+{
+    HANDLE handle;
+
+    if (PySys_Audit("_winapi.OpenMutexW", "Iu", desired_access, name) < 0) {
+        return INVALID_HANDLE_VALUE;
+    }
+
+    Py_BEGIN_ALLOW_THREADS
+    handle = OpenMutexW(desired_access, inherit_handle, name);
+    Py_END_ALLOW_THREADS
+
+    if (handle == INVALID_HANDLE_VALUE) {
+        PyErr_SetFromWindowsErr(0);
+    }
+
+    return handle;
+}
+
 /*[clinic input]
 _winapi.OpenFileMapping -> HANDLE
 
@@ -1820,6 +1945,75 @@ _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size,
     return Py_BuildValue("NI", buf, err);
 }
 
+/*[clinic input]
+_winapi.ReleaseMutex
+
+    mutex: HANDLE
+[clinic start generated code]*/
+
+static PyObject *
+_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex)
+/*[clinic end generated code: output=5b9001a72dd8af37 input=49e9d20de3559d84]*/
+{
+    int err = 0;
+
+    Py_BEGIN_ALLOW_THREADS
+    if (!ReleaseMutex(mutex)) {
+        err = GetLastError();
+    }
+    Py_END_ALLOW_THREADS
+    if (err) {
+        return PyErr_SetFromWindowsErr(err);
+    }
+    Py_RETURN_NONE;
+}
+
+/*[clinic input]
+_winapi.ResetEvent
+
+    event: HANDLE
+[clinic start generated code]*/
+
+static PyObject *
+_winapi_ResetEvent_impl(PyObject *module, HANDLE event)
+/*[clinic end generated code: output=81c8501d57c0530d input=e2d42d990322e87a]*/
+{
+    int err = 0;
+
+    Py_BEGIN_ALLOW_THREADS
+    if (!ResetEvent(event)) {
+        err = GetLastError();
+    }
+    Py_END_ALLOW_THREADS
+    if (err) {
+        return PyErr_SetFromWindowsErr(err);
+    }
+    Py_RETURN_NONE;
+}
+
+/*[clinic input]
+_winapi.SetEvent
+
+    event: HANDLE
+[clinic start generated code]*/
+
+static PyObject *
+_winapi_SetEvent_impl(PyObject *module, HANDLE event)
+/*[clinic end generated code: output=c18ba09eb9aa774d input=e660e830a37c09f8]*/
+{
+    int err = 0;
+
+    Py_BEGIN_ALLOW_THREADS
+    if (!SetEvent(event)) {
+        err = GetLastError();
+    }
+    Py_END_ALLOW_THREADS
+    if (err) {
+        return PyErr_SetFromWindowsErr(err);
+    }
+    Py_RETURN_NONE;
+}
+
 /*[clinic input]
 _winapi.SetNamedPipeHandleState
 
@@ -1942,6 +2136,310 @@ _winapi_WaitNamedPipe_impl(PyObject *module, LPCTSTR name, DWORD timeout)
     Py_RETURN_NONE;
 }
 
+
+typedef struct {
+    HANDLE handles[MAXIMUM_WAIT_OBJECTS];
+    HANDLE cancel_event;
+    DWORD handle_base;
+    DWORD handle_count;
+    HANDLE thread;
+    volatile DWORD result;
+} BatchedWaitData;
+
+static DWORD WINAPI
+_batched_WaitForMultipleObjects_thread(LPVOID param)
+{
+    BatchedWaitData *data = (BatchedWaitData *)param;
+    data->result = WaitForMultipleObjects(
+        data->handle_count,
+        data->handles,
+        FALSE,
+        INFINITE
+    );
+    if (data->result == WAIT_FAILED) {
+        DWORD err = GetLastError();
+        SetEvent(data->cancel_event);
+        return err;
+    } else if (data->result >= WAIT_ABANDONED_0 && data->result < WAIT_ABANDONED_0 + MAXIMUM_WAIT_OBJECTS) {
+        data->result = WAIT_FAILED;
+        SetEvent(data->cancel_event);
+        return ERROR_ABANDONED_WAIT_0;
+    }
+    return 0;
+}
+
+/*[clinic input]
+_winapi.BatchedWaitForMultipleObjects
+
+    handle_seq: object
+    wait_all: BOOL
+    milliseconds: DWORD(c_default='INFINITE') = _winapi.INFINITE
+
+Supports a larger number of handles than WaitForMultipleObjects
+
+Note that the handles may be waited on other threads, which could cause
+issues for objects like mutexes that become associated with the thread
+that was waiting for them. Objects may also be left signalled, even if
+the wait fails.
+
+It is recommended to use WaitForMultipleObjects whenever possible, and
+only switch to BatchedWaitForMultipleObjects for scenarios where you
+control all the handles involved, such as your own thread pool or
+files, and all wait objects are left unmodified by a wait (for example,
+manual reset events, threads, and files/pipes).
+
+Overlapped handles returned from this module use manual reset events.
+[clinic start generated code]*/
+
+static PyObject *
+_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module,
+                                           PyObject *handle_seq,
+                                           BOOL wait_all, DWORD milliseconds)
+/*[clinic end generated code: output=d21c1a4ad0a252fd input=7e196f29005dc77b]*/
+{
+    Py_ssize_t thread_count = 0, handle_count = 0, i, j;
+    Py_ssize_t nhandles;
+    BatchedWaitData *thread_data[MAXIMUM_WAIT_OBJECTS];
+    HANDLE handles[MAXIMUM_WAIT_OBJECTS];
+    HANDLE sigint_event = NULL;
+    HANDLE cancel_event = NULL;
+    DWORD result;
+
+    const Py_ssize_t _MAXIMUM_TOTAL_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) * (MAXIMUM_WAIT_OBJECTS - 1);
+
+    if (!PySequence_Check(handle_seq)) {
+        PyErr_Format(PyExc_TypeError,
+                     "sequence type expected, got '%s'",
+                     Py_TYPE(handle_seq)->tp_name);
+        return NULL;
+    }
+    nhandles = PySequence_Length(handle_seq);
+    if (nhandles == -1) {
+        return NULL;
+    }
+    if (nhandles == 0) {
+        return wait_all ? Py_NewRef(Py_None) : PyList_New(0);
+    }
+
+    /* If this is the main thread then make the wait interruptible
+       by Ctrl-C. When waiting for *all* handles, it is only checked
+       in between batches. */
+    if (_PyOS_IsMainThread()) {
+        sigint_event = _PyOS_SigintEvent();
+        assert(sigint_event != NULL);
+    }
+
+    if (nhandles < 0 || nhandles > _MAXIMUM_TOTAL_OBJECTS) {
+        PyErr_Format(PyExc_ValueError,
+                     "need at most %zd handles, got a sequence of length %zd",
+                     _MAXIMUM_TOTAL_OBJECTS, nhandles);
+        return NULL;
+    }
+
+    if (!wait_all) {
+        cancel_event = CreateEventW(NULL, TRUE, FALSE, NULL);
+        if (!cancel_event) {
+            PyErr_SetExcFromWindowsErr(PyExc_OSError, 0);
+            return NULL;
+        }
+    }
+
+    i = 0;
+    while (i < nhandles) {
+        BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData));
+        if (!data) {
+            goto error;
+        }
+        thread_data[thread_count++] = data;
+        data->thread = NULL;
+        data->cancel_event = cancel_event;
+        data->handle_base = Py_SAFE_DOWNCAST(i, Py_ssize_t, DWORD);
+        data->handle_count = Py_SAFE_DOWNCAST(nhandles - i, Py_ssize_t, DWORD);
+        if (data->handle_count > MAXIMUM_WAIT_OBJECTS - 1) {
+            data->handle_count = MAXIMUM_WAIT_OBJECTS - 1;
+        }
+        for (j = 0; j < data->handle_count; ++i, ++j) {
+            PyObject *v = PySequence_GetItem(handle_seq, i);
+            if (!v || !PyArg_Parse(v, F_HANDLE, &data->handles[j])) {
+                Py_XDECREF(v);
+                goto error;
+            }
+            Py_DECREF(v);
+        }
+        if (!wait_all) {
+            data->handles[data->handle_count++] = cancel_event;
+        }
+    }
+
+    DWORD err = 0;
+
+    /* We need to use different strategies when waiting for ALL handles
+       as opposed to ANY handle. This is because there is no way to
+       (safely) interrupt a thread that is waiting for all handles in a
+       group. So for ALL handles, we loop over each set and wait. For
+       ANY handle, we use threads and wait on them. */
+    if (wait_all) {
+        Py_BEGIN_ALLOW_THREADS
+        long long deadline = 0;
+        if (milliseconds != INFINITE) {
+            deadline = (long long)GetTickCount64() + milliseconds;
+        }
+
+        for (i = 0; !err && i < thread_count; ++i) {
+            DWORD timeout = milliseconds;
+            if (deadline) {
+                long long time_to_deadline = deadline - GetTickCount64();
+                if (time_to_deadline <= 0) {
+                    err = WAIT_TIMEOUT;
+                    break;
+                } else if (time_to_deadline < UINT_MAX) {
+                    timeout = (DWORD)time_to_deadline;
+                }
+            }
+            result = WaitForMultipleObjects(thread_data[i]->handle_count,
+                                            thread_data[i]->handles, TRUE, timeout);
+            // ABANDONED is not possible here because we own all the handles
+            if (result == WAIT_FAILED) {
+                err = GetLastError();
+            } else if (result == WAIT_TIMEOUT) {
+                err = WAIT_TIMEOUT;
+            }
+
+            if (!err && sigint_event) {
+                result = WaitForSingleObject(sigint_event, 0);
+                if (result == WAIT_OBJECT_0) {
+                    err = ERROR_CONTROL_C_EXIT;
+                } else if (result == WAIT_FAILED) {
+                    err = GetLastError();
+                }
+            }
+        }
+
+        CloseHandle(cancel_event);
+
+        Py_END_ALLOW_THREADS
+    } else {
+        Py_BEGIN_ALLOW_THREADS
+
+        for (i = 0; i < thread_count; ++i) {
+            BatchedWaitData *data = thread_data[i];
+            data->thread = CreateThread(
+                NULL,
+                1,  // smallest possible initial stack
+                _batched_WaitForMultipleObjects_thread,
+                (LPVOID)data,
+                CREATE_SUSPENDED,
+                NULL
+            );
+            if (!data->thread) {
+                err = GetLastError();
+                break;
+            }
+            handles[handle_count++] = data->thread;
+        }
+        Py_END_ALLOW_THREADS
+
+        if (err) {
+            PyErr_SetExcFromWindowsErr(PyExc_OSError, err);
+            goto error;
+        }
+        if (handle_count > MAXIMUM_WAIT_OBJECTS - 1) {
+            // basically an assert, but stronger
+            PyErr_SetString(PyExc_SystemError, "allocated too many wait objects");
+            goto error;
+        }
+
+        Py_BEGIN_ALLOW_THREADS
+
+        // Once we start resuming threads, can no longer "goto error"
+        for (i = 0; i < thread_count; ++i) {
+            ResumeThread(thread_data[i]->thread);
+        }
+        if (sigint_event) {
+            handles[handle_count++] = sigint_event;
+        }
+        result = WaitForMultipleObjects((DWORD)handle_count, handles, wait_all, milliseconds);
+        // ABANDONED is not possible here because we own all the handles
+        if (result == WAIT_FAILED) {
+            err = GetLastError();
+        } else if (result == WAIT_TIMEOUT) {
+            err = WAIT_TIMEOUT;
+        } else if (sigint_event && result == WAIT_OBJECT_0 + handle_count) {
+            err = ERROR_CONTROL_C_EXIT;
+        }
+
+        SetEvent(cancel_event);
+
+        // Wait for all threads to finish before we start freeing their memory
+        if (sigint_event) {
+            handle_count -= 1;
+        }
+        WaitForMultipleObjects((DWORD)handle_count, handles, TRUE, INFINITE);
+
+        for (i = 0; i < thread_count; ++i) {
+            if (!err && thread_data[i]->result == WAIT_FAILED) {
+                if (!GetExitCodeThread(thread_data[i]->thread, &err)) {
+                    err = GetLastError();
+                }
+            }
+            CloseHandle(thread_data[i]->thread);
+        }
+
+        CloseHandle(cancel_event);
+
+        Py_END_ALLOW_THREADS
+
+    }
+
+    PyObject *triggered_indices;
+    if (sigint_event != NULL && err == ERROR_CONTROL_C_EXIT) {
+        errno = EINTR;
+        PyErr_SetFromErrno(PyExc_OSError);
+        triggered_indices = NULL;
+    } else if (err) {
+        PyErr_SetExcFromWindowsErr(PyExc_OSError, err);
+        triggered_indices = NULL;
+    } else if (wait_all) {
+        triggered_indices = Py_NewRef(Py_None);
+    } else {
+        triggered_indices = PyList_New(0);
+        if (triggered_indices) {
+            for (i = 0; i < thread_count; ++i) {
+                Py_ssize_t triggered = (Py_ssize_t)thread_data[i]->result - WAIT_OBJECT_0;
+                if (triggered >= 0 && triggered < thread_data[i]->handle_count - 1) {
+                    PyObject *v = PyLong_FromSsize_t(thread_data[i]->handle_base + triggered);
+                    if (!v || PyList_Append(triggered_indices, v) < 0) {
+                        Py_XDECREF(v);
+                        Py_CLEAR(triggered_indices);
+                        break;
+                    }
+                    Py_DECREF(v);
+                }
+            }
+        }
+    }
+
+    for (i = 0; i < thread_count; ++i) {
+        PyMem_Free((void *)thread_data[i]);
+    }
+
+    return triggered_indices;
+
+error:
+    // We should only enter here before any threads start running.
+    // Once we start resuming threads, different cleanup is required
+    CloseHandle(cancel_event);
+    while (--thread_count >= 0) {
+        HANDLE t = thread_data[thread_count]->thread;
+        if (t) {
+            TerminateThread(t, WAIT_ABANDONED_0);
+            CloseHandle(t);
+        }
+        PyMem_Free((void *)thread_data[thread_count]);
+    }
+    return NULL;
+}
+
 /*[clinic input]
 _winapi.WaitForMultipleObjects
 
@@ -2335,8 +2833,10 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name,
 static PyMethodDef winapi_functions[] = {
     _WINAPI_CLOSEHANDLE_METHODDEF
     _WINAPI_CONNECTNAMEDPIPE_METHODDEF
+    _WINAPI_CREATEEVENTW_METHODDEF
     _WINAPI_CREATEFILE_METHODDEF
     _WINAPI_CREATEFILEMAPPING_METHODDEF
+    _WINAPI_CREATEMUTEXW_METHODDEF
     _WINAPI_CREATENAMEDPIPE_METHODDEF
     _WINAPI_CREATEPIPE_METHODDEF
     _WINAPI_CREATEPROCESS_METHODDEF
@@ -2350,17 +2850,23 @@ static PyMethodDef winapi_functions[] = {
     _WINAPI_GETSTDHANDLE_METHODDEF
     _WINAPI_GETVERSION_METHODDEF
     _WINAPI_MAPVIEWOFFILE_METHODDEF
+    _WINAPI_OPENEVENTW_METHODDEF
     _WINAPI_OPENFILEMAPPING_METHODDEF
+    _WINAPI_OPENMUTEXW_METHODDEF
     _WINAPI_OPENPROCESS_METHODDEF
     _WINAPI_PEEKNAMEDPIPE_METHODDEF
     _WINAPI_LCMAPSTRINGEX_METHODDEF
     _WINAPI_READFILE_METHODDEF
+    _WINAPI_RELEASEMUTEX_METHODDEF
+    _WINAPI_RESETEVENT_METHODDEF
+    _WINAPI_SETEVENT_METHODDEF
     _WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF
     _WINAPI_TERMINATEPROCESS_METHODDEF
     _WINAPI_UNMAPVIEWOFFILE_METHODDEF
     _WINAPI_VIRTUALQUERYSIZE_METHODDEF
     _WINAPI_WAITNAMEDPIPE_METHODDEF
     _WINAPI_WAITFORMULTIPLEOBJECTS_METHODDEF
+    _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF
     _WINAPI_WAITFORSINGLEOBJECT_METHODDEF
     _WINAPI_WRITEFILE_METHODDEF
     _WINAPI_GETACP_METHODDEF
index d1052f38919ddefcec2a072c9ecf0b730a400d77..468457e624c6916e1008ae65182df50e92f77fdd 100644 (file)
@@ -151,6 +151,76 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_CreateEventW__doc__,
+"CreateEventW($module, /, security_attributes, manual_reset,\n"
+"             initial_state, name)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_CREATEEVENTW_METHODDEF    \
+    {"CreateEventW", _PyCFunction_CAST(_winapi_CreateEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateEventW__doc__},
+
+static HANDLE
+_winapi_CreateEventW_impl(PyObject *module,
+                          LPSECURITY_ATTRIBUTES security_attributes,
+                          BOOL manual_reset, BOOL initial_state,
+                          LPCWSTR name);
+
+static PyObject *
+_winapi_CreateEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 4
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(security_attributes), &_Py_ID(manual_reset), &_Py_ID(initial_state), &_Py_ID(name), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"security_attributes", "manual_reset", "initial_state", "name", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "" F_POINTER "iiO&:CreateEventW",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    LPSECURITY_ATTRIBUTES security_attributes;
+    BOOL manual_reset;
+    BOOL initial_state;
+    LPCWSTR name = NULL;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &security_attributes, &manual_reset, &initial_state, _PyUnicode_WideCharString_Opt_Converter, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_CreateEventW_impl(module, security_attributes, manual_reset, initial_state, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    /* Cleanup for name */
+    PyMem_Free((void *)name);
+
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_CreateFile__doc__,
 "CreateFile($module, file_name, desired_access, share_mode,\n"
 "           security_attributes, creation_disposition,\n"
@@ -297,6 +367,73 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_CreateMutexW__doc__,
+"CreateMutexW($module, /, security_attributes, initial_owner, name)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_CREATEMUTEXW_METHODDEF    \
+    {"CreateMutexW", _PyCFunction_CAST(_winapi_CreateMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_CreateMutexW__doc__},
+
+static HANDLE
+_winapi_CreateMutexW_impl(PyObject *module,
+                          LPSECURITY_ATTRIBUTES security_attributes,
+                          BOOL initial_owner, LPCWSTR name);
+
+static PyObject *
+_winapi_CreateMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 3
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(security_attributes), &_Py_ID(initial_owner), &_Py_ID(name), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"security_attributes", "initial_owner", "name", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "" F_POINTER "iO&:CreateMutexW",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    LPSECURITY_ATTRIBUTES security_attributes;
+    BOOL initial_owner;
+    LPCWSTR name = NULL;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &security_attributes, &initial_owner, _PyUnicode_WideCharString_Opt_Converter, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_CreateMutexW_impl(module, security_attributes, initial_owner, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    /* Cleanup for name */
+    PyMem_Free((void *)name);
+
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_CreateNamedPipe__doc__,
 "CreateNamedPipe($module, name, open_mode, pipe_mode, max_instances,\n"
 "                out_buffer_size, in_buffer_size, default_timeout,\n"
@@ -771,6 +908,138 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_OpenEventW__doc__,
+"OpenEventW($module, /, desired_access, inherit_handle, name)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_OPENEVENTW_METHODDEF    \
+    {"OpenEventW", _PyCFunction_CAST(_winapi_OpenEventW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenEventW__doc__},
+
+static HANDLE
+_winapi_OpenEventW_impl(PyObject *module, DWORD desired_access,
+                        BOOL inherit_handle, LPCWSTR name);
+
+static PyObject *
+_winapi_OpenEventW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 3
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "kiO&:OpenEventW",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    DWORD desired_access;
+    BOOL inherit_handle;
+    LPCWSTR name = NULL;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_OpenEventW_impl(module, desired_access, inherit_handle, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    /* Cleanup for name */
+    PyMem_Free((void *)name);
+
+    return return_value;
+}
+
+PyDoc_STRVAR(_winapi_OpenMutexW__doc__,
+"OpenMutexW($module, /, desired_access, inherit_handle, name)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_OPENMUTEXW_METHODDEF    \
+    {"OpenMutexW", _PyCFunction_CAST(_winapi_OpenMutexW), METH_FASTCALL|METH_KEYWORDS, _winapi_OpenMutexW__doc__},
+
+static HANDLE
+_winapi_OpenMutexW_impl(PyObject *module, DWORD desired_access,
+                        BOOL inherit_handle, LPCWSTR name);
+
+static PyObject *
+_winapi_OpenMutexW(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 3
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(desired_access), &_Py_ID(inherit_handle), &_Py_ID(name), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"desired_access", "inherit_handle", "name", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "kiO&:OpenMutexW",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    DWORD desired_access;
+    BOOL inherit_handle;
+    LPCWSTR name = NULL;
+    HANDLE _return_value;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &desired_access, &inherit_handle, _PyUnicode_WideCharString_Converter, &name)) {
+        goto exit;
+    }
+    _return_value = _winapi_OpenMutexW_impl(module, desired_access, inherit_handle, name);
+    if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) {
+        goto exit;
+    }
+    if (_return_value == NULL) {
+        Py_RETURN_NONE;
+    }
+    return_value = HANDLE_TO_PYNUM(_return_value);
+
+exit:
+    /* Cleanup for name */
+    PyMem_Free((void *)name);
+
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_OpenFileMapping__doc__,
 "OpenFileMapping($module, desired_access, inherit_handle, name, /)\n"
 "--\n"
@@ -991,6 +1260,162 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_ReleaseMutex__doc__,
+"ReleaseMutex($module, /, mutex)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_RELEASEMUTEX_METHODDEF    \
+    {"ReleaseMutex", _PyCFunction_CAST(_winapi_ReleaseMutex), METH_FASTCALL|METH_KEYWORDS, _winapi_ReleaseMutex__doc__},
+
+static PyObject *
+_winapi_ReleaseMutex_impl(PyObject *module, HANDLE mutex);
+
+static PyObject *
+_winapi_ReleaseMutex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 1
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(mutex), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"mutex", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "" F_HANDLE ":ReleaseMutex",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    HANDLE mutex;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &mutex)) {
+        goto exit;
+    }
+    return_value = _winapi_ReleaseMutex_impl(module, mutex);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_winapi_ResetEvent__doc__,
+"ResetEvent($module, /, event)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_RESETEVENT_METHODDEF    \
+    {"ResetEvent", _PyCFunction_CAST(_winapi_ResetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_ResetEvent__doc__},
+
+static PyObject *
+_winapi_ResetEvent_impl(PyObject *module, HANDLE event);
+
+static PyObject *
+_winapi_ResetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 1
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(event), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"event", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "" F_HANDLE ":ResetEvent",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    HANDLE event;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &event)) {
+        goto exit;
+    }
+    return_value = _winapi_ResetEvent_impl(module, event);
+
+exit:
+    return return_value;
+}
+
+PyDoc_STRVAR(_winapi_SetEvent__doc__,
+"SetEvent($module, /, event)\n"
+"--\n"
+"\n");
+
+#define _WINAPI_SETEVENT_METHODDEF    \
+    {"SetEvent", _PyCFunction_CAST(_winapi_SetEvent), METH_FASTCALL|METH_KEYWORDS, _winapi_SetEvent__doc__},
+
+static PyObject *
+_winapi_SetEvent_impl(PyObject *module, HANDLE event);
+
+static PyObject *
+_winapi_SetEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 1
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(event), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"event", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "" F_HANDLE ":SetEvent",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    HANDLE event;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &event)) {
+        goto exit;
+    }
+    return_value = _winapi_SetEvent_impl(module, event);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_SetNamedPipeHandleState__doc__,
 "SetNamedPipeHandleState($module, named_pipe, mode,\n"
 "                        max_collection_count, collect_data_timeout, /)\n"
@@ -1114,6 +1539,77 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_winapi_BatchedWaitForMultipleObjects__doc__,
+"BatchedWaitForMultipleObjects($module, /, handle_seq, wait_all,\n"
+"                              milliseconds=_winapi.INFINITE)\n"
+"--\n"
+"\n"
+"Supports a larger number of handles than WaitForMultipleObjects\n"
+"\n"
+"Note that the handles may be waited on other threads, which could cause\n"
+"issues for objects like mutexes that become associated with the thread\n"
+"that was waiting for them. Objects may also be left signalled, even if\n"
+"the wait fails.\n"
+"\n"
+"It is recommended to use WaitForMultipleObjects whenever possible, and\n"
+"only switch to BatchedWaitForMultipleObjects for scenarios where you\n"
+"control all the handles involved, such as your own thread pool or\n"
+"files, and all wait objects are left unmodified by a wait (for example,\n"
+"manual reset events, threads, and files/pipes).\n"
+"\n"
+"Overlapped handles returned from this module use manual reset events.");
+
+#define _WINAPI_BATCHEDWAITFORMULTIPLEOBJECTS_METHODDEF    \
+    {"BatchedWaitForMultipleObjects", _PyCFunction_CAST(_winapi_BatchedWaitForMultipleObjects), METH_FASTCALL|METH_KEYWORDS, _winapi_BatchedWaitForMultipleObjects__doc__},
+
+static PyObject *
+_winapi_BatchedWaitForMultipleObjects_impl(PyObject *module,
+                                           PyObject *handle_seq,
+                                           BOOL wait_all, DWORD milliseconds);
+
+static PyObject *
+_winapi_BatchedWaitForMultipleObjects(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+    PyObject *return_value = NULL;
+    #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
+
+    #define NUM_KEYWORDS 3
+    static struct {
+        PyGC_Head _this_is_not_used;
+        PyObject_VAR_HEAD
+        PyObject *ob_item[NUM_KEYWORDS];
+    } _kwtuple = {
+        .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
+        .ob_item = { &_Py_ID(handle_seq), &_Py_ID(wait_all), &_Py_ID(milliseconds), },
+    };
+    #undef NUM_KEYWORDS
+    #define KWTUPLE (&_kwtuple.ob_base.ob_base)
+
+    #else  // !Py_BUILD_CORE
+    #  define KWTUPLE NULL
+    #endif  // !Py_BUILD_CORE
+
+    static const char * const _keywords[] = {"handle_seq", "wait_all", "milliseconds", NULL};
+    static _PyArg_Parser _parser = {
+        .keywords = _keywords,
+        .format = "Oi|k:BatchedWaitForMultipleObjects",
+        .kwtuple = KWTUPLE,
+    };
+    #undef KWTUPLE
+    PyObject *handle_seq;
+    BOOL wait_all;
+    DWORD milliseconds = INFINITE;
+
+    if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser,
+        &handle_seq, &wait_all, &milliseconds)) {
+        goto exit;
+    }
+    return_value = _winapi_BatchedWaitForMultipleObjects_impl(module, handle_seq, wait_all, milliseconds);
+
+exit:
+    return return_value;
+}
+
 PyDoc_STRVAR(_winapi_WaitForMultipleObjects__doc__,
 "WaitForMultipleObjects($module, handle_seq, wait_flag,\n"
 "                       milliseconds=_winapi.INFINITE, /)\n"
@@ -1482,4 +1978,4 @@ exit:
 
     return return_value;
 }
-/*[clinic end generated code: output=2350d4f2275d3a6f input=a9049054013a1b77]*/
+/*[clinic end generated code: output=1f5bbcfa8d1847c5 input=a9049054013a1b77]*/
index cff55d05163b6ba811899b9392f05436937a805e..3df3a9b3b1a2530a90599dd0decbbd0a4f569739 100644 (file)
@@ -3539,7 +3539,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,
 #undef EOPNOTSUPP
 #undef EPROTONOSUPPORT
 #undef EPROTOTYPE
-#undef ETIMEDOUT
 #undef EWOULDBLOCK
 
 #if defined(WSAEALREADY) && !defined(EALREADY)
@@ -3560,9 +3559,6 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,
 #if defined(WSAESHUTDOWN) && !defined(ESHUTDOWN)
 #define ESHUTDOWN WSAESHUTDOWN
 #endif
-#if defined(WSAETIMEDOUT) && !defined(ETIMEDOUT)
-#define ETIMEDOUT WSAETIMEDOUT
-#endif
 #if defined(WSAEWOULDBLOCK) && !defined(EWOULDBLOCK)
 #define EWOULDBLOCK WSAEWOULDBLOCK
 #endif
@@ -3747,6 +3743,9 @@ _PyExc_InitState(PyInterpreterState *interp)
 #endif
     ADD_ERRNO(ProcessLookupError, ESRCH);
     ADD_ERRNO(TimeoutError, ETIMEDOUT);
+#ifdef WSAETIMEDOUT
+    ADD_ERRNO(TimeoutError, WSAETIMEDOUT);
+#endif
 
     return _PyStatus_OK();
 
index a7489ab75c65618c5486a9fb0b8c6f87fc88489c..a064ecb80b1ed93112ad9c275b6ac4037e7de423 100644 (file)
@@ -129,6 +129,9 @@ winerror_to_errno(int winerror)
     case ERROR_NO_UNICODE_TRANSLATION:    // 1113
         return EILSEQ;
 
+    case WAIT_TIMEOUT:                    //  258
+        return ETIMEDOUT;
+
     case ERROR_INVALID_FUNCTION:          //    1
     case ERROR_INVALID_ACCESS:            //   12
     case ERROR_INVALID_DATA:              //   13