From: Denis Laxalde <denis.laxalde@dalibo.com>
Date: Fri, 1 Mar 2024 13:21:35 +0000 (+0100)
Subject: feat(tools): run async_to_sync.py concurrently
X-Git-Tag: pool-3.2.2~5^2~2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=16f37d21652c8b5b9363cc201ab235e16fcee593;p=thirdparty%2Fpsycopg.git

feat(tools): run async_to_sync.py concurrently

Multi-processing is used by default (using all available processors),
but can be disabled or limited with the -j/--jobs option.
---

diff --git a/tools/async_to_sync.py b/tools/async_to_sync.py
index c4c053a1a..3ff046e76 100755
--- a/tools/async_to_sync.py
+++ b/tools/async_to_sync.py
@@ -12,6 +12,7 @@ Hint: in order to explore the AST of a module you can run:
 from __future__ import annotations
 
 import os
+from concurrent.futures import ProcessPoolExecutor
 import sys
 import logging
 import subprocess as sp
@@ -83,27 +84,36 @@ def main() -> int:
             PYVER,
         )
 
-    outputs = []
-    for fpin in opt.inputs:
-        fpout = fpin.parent / fpin.name.replace("_async", "")
-        outputs.append(str(fpout))
-        logger.info("converting %s", fpin)
-        with fpin.open() as f:
-            source = f.read()
+    if opt.jobs == 1:
+        logger.debug("multi-processing disabled")
+        for fpin in opt.inputs:
+            convert(fpin)
+    else:
+        with ProcessPoolExecutor(max_workers=opt.jobs) as executor:
+            outputs = executor.map(convert, opt.inputs)
 
-        tree = ast.parse(source, filename=str(fpin))
-        tree = async_to_sync(tree, filepath=fpin)
-        output = tree_to_str(tree, fpin)
+    if opt.check:
+        return check([str(o) for o in outputs])
 
-        with fpout.open("w") as f:
-            print(output, file=f)
+    return 0
 
-        sp.check_call(["black", "-q", str(fpout)])
 
-    if opt.check:
-        return check(outputs)
+def convert(fpin: Path) -> Path:
+    fpout = fpin.parent / fpin.name.replace("_async", "")
+    logger.info("converting %s", fpin)
+    with fpin.open() as f:
+        source = f.read()
 
-    return 0
+    tree = ast.parse(source, filename=str(fpin))
+    tree = async_to_sync(tree, filepath=fpin)
+    output = tree_to_str(tree, fpin)
+
+    with fpout.open("w") as f:
+        print(output, file=f)
+
+    sp.check_call(["black", "-q", str(fpout)])
+
+    return fpout
 
 
 def check(outputs: list[str]) -> int:
@@ -567,6 +577,16 @@ def parse_cmdline() -> Namespace:
     parser.add_argument(
         "--all", action="store_true", help="process all the files of the project"
     )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        metavar="N",
+        help=(
+            "process files concurrently using at most N workers; "
+            "if unspecified, the number of processors on the machine will be used"
+        ),
+    )
     container = parser.add_mutually_exclusive_group()
     container.add_argument(
         "--docker",