From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Fri, 13 Oct 2023 00:19:03 +0000 (+0200)
Subject: refactor: drop 'convert_async_to_sync.sh' script
X-Git-Tag: pool-3.2.0~12^2~4
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a74f89391610cd261b85391b07c85bc9fe508d10;p=thirdparty%2Fpsycopg.git

refactor: drop 'convert_async_to_sync.sh' script

Do all in 'async_to_sync.py:' added --all and --check options.
---

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index ad8379d03..3fea1c28f 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -37,7 +37,7 @@ jobs:
         run: mypy
 
       - name: Check for sync/async inconsistencies
-        run: ./tools/convert_async_to_sync.sh --check
+        run: ./tools/async_to_sync.py --all --check
 
       - name: Check spelling
         run: codespell
diff --git a/tools/async_to_sync.py b/tools/async_to_sync.py
index 7b949d137..14ea4d36e 100755
--- a/tools/async_to_sync.py
+++ b/tools/async_to_sync.py
@@ -1,15 +1,20 @@
 #!/usr/bin/env python
-"""Convert an async module to a sync module.
+"""Convert async code in the project to sync code.
+
+Note: the version of Python used to run this script affects the output. Please
+use the `async-to-sync.sh` wrapper to use a version consistent with CI checks.
 """
 
 from __future__ import annotations
 
 import os
 import sys
+import logging
+import subprocess as sp
 from copy import deepcopy
 from typing import Any
 from pathlib import Path
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
 
 import ast_comments as ast
 
@@ -26,21 +31,89 @@ ast.Dict = ast_orig.Dict
 ast.List = ast_orig.List
 ast.Tuple = ast_orig.Tuple
 
+ALL_INPUTS = """
+    psycopg/psycopg/_copy_async.py
+    psycopg/psycopg/connection_async.py
+    psycopg/psycopg/cursor_async.py
+    psycopg_pool/psycopg_pool/null_pool_async.py
+    psycopg_pool/psycopg_pool/pool_async.py
+    psycopg_pool/psycopg_pool/sched_async.py
+    tests/pool/test_pool_async.py
+    tests/pool/test_pool_common_async.py
+    tests/pool/test_pool_null_async.py
+    tests/pool/test_sched_async.py
+    tests/test_connection_async.py
+    tests/test_copy_async.py
+    tests/test_cursor_async.py
+    tests/test_cursor_client_async.py
+    tests/test_cursor_common_async.py
+    tests/test_cursor_raw_async.py
+    tests/test_cursor_server_async.py
+    tests/test_pipeline_async.py
+    tests/test_prepared_async.py
+    tests/test_tpc_async.py
+    tests/test_transaction_async.py
+""".split()
+
+PROJECT_DIR = Path(__file__).parent.parent
+SCRIPT_NAME = os.path.basename(sys.argv[0])
+
+logger = logging.getLogger()
+logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+
 
 def main() -> int:
     opt = parse_cmdline()
-    with opt.filepath.open() as f:
-        source = f.read()
+    outputs = []
+    for fpin in opt.inputs:
+        fpout = fpin.parent / fpin.name.replace("_async", "")
+        outputs.append(str(fpout))
+        logger.info("converting %s", fpin)
+        with fpin.open() as f:
+            source = f.read()
+
+        tree = ast.parse(source, filename=str(fpin))
+        tree = async_to_sync(tree, filepath=fpin)
+        output = tree_to_str(tree, fpin)
+
+        with fpout.open("w") as f:
+            print(output, file=f)
 
-    tree = ast.parse(source, filename=str(opt.filepath))
-    tree = async_to_sync(tree, filepath=opt.filepath)
-    output = tree_to_str(tree, opt.filepath)
+        sp.check_call(["black", "-q", str(fpout)])
 
-    if opt.output:
-        with open(opt.output, "w") as f:
-            print(output, file=f)
-    else:
-        print(output)
+    if opt.check:
+        return check(outputs)
+
+    return 0
+
+
+def check(outputs: list[str]) -> int:
+    try:
+        sp.check_call(["git", "diff", "--exit-code"] + outputs)
+    except sp.CalledProcessError:
+        logger.error("sync and async files... out of sync!")
+        return 1
+
+    # Check that all the files to convert are included in the --all list
+    cmdline = ["git", "ls-files", "**.py"]
+    git_pys = sp.check_output(cmdline, cwd=str(PROJECT_DIR)).decode().split()
+
+    cmdline = ["grep", "-l", f"auto-generated by '{SCRIPT_NAME}'"]
+    cmdline += git_pys
+    maybe_conv = sp.check_output(cmdline, cwd=str(PROJECT_DIR)).decode().split()
+    if not maybe_conv:
+        logger.error("no file to check? Maybe this script bitrot?")
+        return 1
+    unk_conv = sorted(
+        set(maybe_conv) - set(fn.replace("_async", "") for fn in ALL_INPUTS)
+    )
+    if unk_conv:
+        logger.error(
+            "files converted by %s but not included in --all list: %s",
+            SCRIPT_NAME,
+            ", ".join(unk_conv),
+        )
+        return 1
 
     return 0
 
@@ -54,7 +127,7 @@ def async_to_sync(tree: ast.AST, filepath: Path | None = None) -> ast.AST:
 
 def tree_to_str(tree: ast.AST, filepath: Path) -> str:
     rv = f"""\
-# WARNING: this file is auto-generated by '{os.path.basename(sys.argv[0])}'
+# WARNING: this file is auto-generated by '{SCRIPT_NAME}'
 # from the original file '{filepath.name}'
 # DO NOT CHANGE! Change the original file instead.
 """
@@ -439,14 +512,40 @@ class Unparser(ast._Unparser):
 
 
 def parse_cmdline() -> Namespace:
-    parser = ArgumentParser(description=__doc__)
+    parser = ArgumentParser(
+        description=__doc__, formatter_class=RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument(
+        "--check", action="store_true", help="return with error in case of differences"
+    )
     parser.add_argument(
-        "filepath", metavar="FILE", type=Path, help="the file to process"
+        "--all", action="store_true", help="process all the files of the project"
     )
     parser.add_argument(
-        "output", metavar="OUTPUT", nargs="?", help="file where to write (or stdout)"
+        "inputs",
+        metavar="FILE",
+        nargs="*",
+        type=Path,
+        help="the files to process (if --all is not specified)",
     )
+
     opt = parser.parse_args()
+    if opt.all and opt.inputs:
+        parser.error("can't specify input files and --all together")
+
+    if opt.all:
+        opt.inputs = [PROJECT_DIR / Path(fn) for fn in ALL_INPUTS]
+
+    if not opt.inputs:
+        parser.error("no input file provided")
+
+    fp: Path
+    for fp in opt.inputs:
+        if not fp.is_file():
+            parser.error("not a file: %s" % fp)
+        if "_async" not in fp.name:
+            parser.error("file should have '_async' in the name: %s" % fp)
 
     return opt
 
diff --git a/tools/convert_async_to_sync.sh b/tools/convert_async_to_sync.sh
deleted file mode 100755
index 76a6e4b06..000000000
--- a/tools/convert_async_to_sync.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/bin/bash
-
-# Convert all the auto-generated sync files from their async counterparts.
-
-set -euo pipefail
-
-dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "${dir}/.."
-
-function log {
-    echo "$@" >&2
-}
-function error {
-    # Print an error message and exit.
-    log "
-ERROR: $@"
-    exit 1
-}
-
-check=
-
-# If --check is used, give an error if files are changed
-# (note: it's not a --dry-run)
-if [[ ${1:-} == '--check' ]]; then
-    check=1
-    shift
-fi
-
-all_inputs="
-    psycopg/psycopg/_copy_async.py
-    psycopg/psycopg/connection_async.py
-    psycopg/psycopg/cursor_async.py
-    psycopg_pool/psycopg_pool/null_pool_async.py
-    psycopg_pool/psycopg_pool/pool_async.py
-    psycopg_pool/psycopg_pool/sched_async.py
-    tests/pool/test_pool_async.py
-    tests/pool/test_pool_common_async.py
-    tests/pool/test_pool_null_async.py
-    tests/pool/test_sched_async.py
-    tests/test_connection_async.py
-    tests/test_copy_async.py
-    tests/test_cursor_async.py
-    tests/test_cursor_client_async.py
-    tests/test_cursor_common_async.py
-    tests/test_cursor_raw_async.py
-    tests/test_cursor_server_async.py
-    tests/test_pipeline_async.py
-    tests/test_prepared_async.py
-    tests/test_tpc_async.py
-    tests/test_transaction_async.py
-"
-
-# Take other arguments as file names if specified
-if [[ ${1:-} ]]; then
-    inputs="$@"
-else
-    inputs="$all_inputs"
-fi
-
-
-outputs=""
-
-for async in $inputs; do
-    test -f "${async}" || error "file not found: '${async}'"
-    sync=${async/_async/}
-    log "converting '${async}' -> '${sync}'"
-    python "${dir}/async_to_sync.py" ${async} > ${sync}
-    black -q ${sync}
-    outputs="$outputs ${sync}"
-done
-
-if [[ $check ]]; then
-    if ! git diff --exit-code $outputs; then
-        error "sync and async files... out of sync!"
-    fi
-
-    # Verify that all the transformed files are included in this script
-    # Note: the 'cd' early in the script ensures that cwd is the project root.
-    checked=0
-    errors=0
-    for fn in $(find psycopg psycopg_pool tests -type f -name \*.py); do
-        # Skip files ignored by git (build artifacts)
-        ! git status --porcelain --ignored "${fn}" | grep -q '!!' || continue
-        # Skip non-auto-generated files
-        grep -q "auto-generated by 'async_to_sync.py'" "${fn}" || continue
-
-        checked=$(( $checked + 1 ))
-        afn=${fn/.py/_async.py}
-        if ! grep -q $afn tools/$(basename $0); then
-            errors=$(( $errors + 1 ))
-            log "file '${fn}' seems converted but not included in '$(basename $0)'"
-        fi
-    done
-    [[ $checked -gt 0 ]] || error "No file found to check. Script bitrot?"
-    [[ $errors -eq 0 ]] || error "Some files are not included in async-to-sync conversion."
-fi