Retry automatically flakey tests in tox runs

diff --git a/psycopg/tox.ini b/psycopg/tox.ini
index 3ffe8c36b2c0c7e909a1f5014a26b0a5da5b7dfb..6e96385fe01eb8e3c8f15afd7b860103d623ed28 100644 (file)
--- a/psycopg/tox.ini
+++ b/psycopg/tox.ini
@@ -2,10 +2,23 @@
 envlist = {3.6,3.7,3.8,3.9,3.10}
 isolated_build = True
 
+# Retry flakey tests by re-running the failed tests up to 3 times.
+#
+# - `--lfnf=none` makes pytest running no test (instead of all) if the previous
+#   run had no failure
+# - `--no-collect-ok` changes the exit value from 5 to 0 if not test was
+#   collected (because the previous run was successful)
+# - the `-` in front of the first two commands in toxmakes it ignore failures,
+#   so that only the exit stastus of the last command is considered.
+#
+# This is *slightly* more complicated than what I'd hoped, but, ok.
+
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -14,7 +27,9 @@ deps =
 [testenv:dns]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =
@@ -23,7 +38,9 @@ deps =
 [testenv:postgis]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 extras = test
 deps =

diff --git a/psycopg_c/tox.ini b/psycopg_c/tox.ini
index 421d3f88b081ab121ae047b07e2ab78c35e0e803..c58c705dab922648736e9117174d69c9797c3b97 100644 (file)
--- a/psycopg_c/tox.ini
+++ b/psycopg_c/tox.ini
@@ -5,7 +5,9 @@ isolated_build = True
 [testenv]
 changedir = ..
 commands =
-    python -bb -m pytest {posargs}
+    -python -bb -m pytest {posargs}
+    -python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
+    python -bb -m pytest --lf --lfnf=none --no-collect-ok {posargs}
 passenv = PG* PSYCOPG_TEST_DSN PYTEST_ADDOPTS PSYCOPG_IMPL
 deps =
     -e {toxinidir}/../psycopg[test]

diff --git a/tests/conftest.py b/tests/conftest.py
index 588fa8affc4beb374739f559f15f6d70ea19e2e2..409fbac42bfc05e0fecdc66065a769d38e1435a7 100644 (file)
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,6 +38,13 @@ def pytest_addoption(parser):
         help="The asyncio loop to use for async tests.",
     )
 
+    parser.addoption(
+        "--no-collect-ok",
+        action="store_true",
+        help="If no test is collected, exit with 0 instead of 5"
+        " (useful with --lfnf=none).",
+    )
+
 
 def pytest_report_header(config):
     loop = config.getoption("--loop")
@@ -89,3 +96,9 @@ def event_loop(request):
         loop = asyncio.get_event_loop_policy().new_event_loop()
     yield loop
     loop.close()
+
+
+def pytest_sessionfinish(session, exitstatus):
+    no_collect_ok = session.config.getoption("--no-collect-ok")
+    if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED and no_collect_ok:
+        session.exitstatus = pytest.ExitCode.OK