From: Daniele Varrazzo <daniele.varrazzo@gmail.com>
Date: Thu, 13 Jan 2022 21:56:44 +0000 (+0100)
Subject: Retry automatically flakey tests in tox runs
X-Git-Tag: pool-3.1~17
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6cf9ea8c333d70640ab26ebf5e47359ec20e9a79;p=thirdparty%2Fpsycopg.git

Retry automatically flakey tests in tox runs
---

diff --git a/psycopg/tox.ini b/psycopg/tox.ini
index 201a90eb4..88412f399 100644
--- a/psycopg/tox.ini
+++ b/psycopg/tox.ini
@@ -2,10 +2,23 @@
 envlist = {3.7,3.8,3.9,3.10}
 isolated_build = True
 
+# Retry flakey tests by re-running the failed tests up to 3 times.
+#
+# - `--lfnf=none` makes pytest running no test (instead of all) if the previous
+#   run had no failure
+# - `--no-collect-ok` changes the exit value from 5 to 0 if not test was
+#   collected (because the previous run was successful)
+# - the `-` in front of the first two commands in toxmakes it ignore failures,
+#   so that only the exit stastus of the last command is considered.
+#
+# This is *slightly* more complicated than what I'd hoped, but, ok.
+
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -14,7 +27,9 @@ deps =
 [testenv:dns]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -23,7 +38,9 @@ deps =
 [testenv:postgis]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
diff --git a/psycopg_c/tox.ini b/psycopg_c/tox.ini
index a1acf5820..d2e2839d7 100644
--- a/psycopg_c/tox.ini
+++ b/psycopg_c/tox.ini
@@ -5,7 +5,9 @@ isolated_build = True
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 deps =
     -e {toxinidir}/../psycopg[test]
diff --git a/tests/conftest.py b/tests/conftest.py
index 77c08ce1b..184a9cbb1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,6 +38,13 @@ def pytest_addoption(parser):
         help="The asyncio loop to use for async tests.",
     )
 
+    parser.addoption(
+        "--no-collect-ok",
+        action="store_true",
+        help="If no test is collected, exit with 0 instead of 5"
+        " (useful with --lfnf=none).",
+    )
+
 
 def pytest_report_header(config):
     loop = config.getoption("--loop")
@@ -74,3 +81,9 @@ def pytest_sessionstart(session):
 
     if sys.platform == "win32":
         asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+
+def pytest_sessionfinish(session, exitstatus):
+    no_collect_ok = session.config.getoption("--no-collect-ok")
+    if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED and no_collect_ok:
+        session.exitstatus = pytest.ExitCode.OK