Retry automatically flakey tests in tox runs

diff --git a/psycopg/tox.ini b/psycopg/tox.ini
index 201a90eb4c5592989abc80147ff2446968164522..88412f3995217fead76c2dc8c4aa1202861dabd0 100644 (file)
--- a/psycopg/tox.ini
+++ b/psycopg/tox.ini
@@ -2,10 +2,23 @@
 envlist = {3.7,3.8,3.9,3.10}
 isolated_build = True
 
+# Retry flakey tests by re-running the failed tests up to 3 times.
+#
+# - `--lfnf=none` makes pytest running no test (instead of all) if the previous
+#   run had no failure
+# - `--no-collect-ok` changes the exit value from 5 to 0 if not test was
+#   collected (because the previous run was successful)
+# - the `-` in front of the first two commands in toxmakes it ignore failures,
+#   so that only the exit stastus of the last command is considered.
+#
+# This is *slightly* more complicated than what I'd hoped, but, ok.
+
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -14,7 +27,9 @@ deps =
 [testenv:dns]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -23,7 +38,9 @@ deps =
 [testenv:postgis]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =

diff --git a/psycopg_c/tox.ini b/psycopg_c/tox.ini
index a1acf58206b678444b910f9ce97d2247e2151c16..d2e2839d7ee9af93c90dcb552d069b370f82bfa3 100644 (file)
--- a/psycopg_c/tox.ini
+++ b/psycopg_c/tox.ini
@@ -5,7 +5,9 @@ isolated_build = True
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 deps =
     -e {toxinidir}/../psycopg[test]

diff --git a/tests/conftest.py b/tests/conftest.py
index 77c08ce1b2cc003d85d3da2131213605b229044c..184a9cbb1a3273263c340f4ac38341d1a30883ff 100644 (file)
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,6 +38,13 @@ def pytest_addoption(parser):
         help="The asyncio loop to use for async tests.",
     )
 
+    parser.addoption(
+        "--no-collect-ok",
+        action="store_true",
+        help="If no test is collected, exit with 0 instead of 5"
+        " (useful with --lfnf=none).",
+    )
+
 
 def pytest_report_header(config):
     loop = config.getoption("--loop")
@@ -74,3 +81,9 @@ def pytest_sessionstart(session):
 
     if sys.platform == "win32":
         asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+
+def pytest_sessionfinish(session, exitstatus):
+    no_collect_ok = session.config.getoption("--no-collect-ok")
+    if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED and no_collect_ok:
+        session.exitstatus = pytest.ExitCode.OK