From: Otto Moerbeek <otto.moerbeek@open-xchange.com>
Date: Thu, 5 Dec 2024 13:54:13 +0000 (+0100)
Subject: rec: mark test that depend on external servers and or are unreliable
X-Git-Tag: dnsdist-2.0.0-alpha0~1^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F14935%2Fhead;p=thirdparty%2Fpdns.git

rec: mark test that depend on external servers and or are unreliable

While there get rid of the special status of test_WellKnown.py and
correct the way args are passed to pytest
---

diff --git a/regression-tests.recursor-dnssec/pytest.ini b/regression-tests.recursor-dnssec/pytest.ini
new file mode 100644
index 0000000000..261099a05f
--- /dev/null
+++ b/regression-tests.recursor-dnssec/pytest.ini
@@ -0,0 +1,4 @@
+[pytest]
+markers = 
+    external: uses external web servers
+    unreliable_on_gh: test is fine when run locally, but shows issues on GitHub
diff --git a/regression-tests.recursor-dnssec/runtests b/regression-tests.recursor-dnssec/runtests
index adee05c461..3543dd2b79 100755
--- a/regression-tests.recursor-dnssec/runtests
+++ b/regression-tests.recursor-dnssec/runtests
@@ -61,9 +61,13 @@ if ! "$PDNSRECURSOR" --version 2>&1 | grep Features | grep -q dnstap-framestream
   export NODNSTAPTESTS=1
 fi
 
+# Run with -m 'not external' to skip test that require external connectivity
+# Run with -m 'not unreliable_on_gh' to skip tests that are unreliable on GitHUb
+# Run with -m 'not (external or unreliable_on_gh)' to skip both categories
+
 # LIBFAKETIME is only added to LD_PRELOAD by the pyton code when needed
 if [ "${LIBASAN}" != "" -o "${LIBAUTHBIND}" != "" ]; then
-LD_PRELOAD="${LIBASAN} ${LIBAUTHBIND}" pytest --ignore=test_WellKnown.py --junitxml=pytest.xml $@
+LD_PRELOAD="${LIBASAN} ${LIBAUTHBIND}" pytest --junitxml=pytest.xml "$@"
 else
-pytest --ignore=test_WellKnown.py  --junitxml=pytest.xml $@
+pytest --junitxml=pytest.xml "$@"
 fi
diff --git a/regression-tests.recursor-dnssec/test_Chain.py b/regression-tests.recursor-dnssec/test_Chain.py
index 6c20937b31..3c20da0fc7 100644
--- a/regression-tests.recursor-dnssec/test_Chain.py
+++ b/regression-tests.recursor-dnssec/test_Chain.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import time
@@ -23,6 +24,7 @@ class ChainTest(RecursorTest):
     api-key=%s
 """ % (_wsPort, _wsPassword, _apiKey)
 
+    @pytest.mark.unreliable_on_gh
     def testBasic(self):
         """
         Tests the case of #14624. Sending many equal requests could lead to ServFail because of clashing
diff --git a/regression-tests.recursor-dnssec/test_ExtendedErrors.py b/regression-tests.recursor-dnssec/test_ExtendedErrors.py
index 635c2c579c..a684665a79 100644
--- a/regression-tests.recursor-dnssec/test_ExtendedErrors.py
+++ b/regression-tests.recursor-dnssec/test_ExtendedErrors.py
@@ -83,6 +83,7 @@ extended-resolution-errors=yes
 
         super(ExtendedErrorsTest, cls).generateRecursorConfig(confdir)
 
+    @pytest.mark.external
     def testNotIncepted(self):
         qname = 'signotincepted.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -96,6 +97,7 @@ extended-resolution-errors=yes
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(8, b''))
 
+    @pytest.mark.external
     def testExpired(self):
         qname = 'sigexpired.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -109,6 +111,7 @@ extended-resolution-errors=yes
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(7, b''))
 
+    @pytest.mark.external
     def testAllExpired(self):
         qname = 'servfail.nl.'
         query = dns.message.make_query(qname, 'AAAA', want_dnssec=True)
@@ -122,6 +125,7 @@ extended-resolution-errors=yes
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(6, b''))
 
+    @pytest.mark.external
     def testBogus(self):
         qname = 'bogussig.ok.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -135,6 +139,7 @@ extended-resolution-errors=yes
             self.assertEqual(res.options[0].otype, 15)
             self.assertEqual(res.options[0], extendederrors.ExtendedErrorOption(6, b''))
 
+    @pytest.mark.external
     def testMissingRRSIG(self):
         qname = 'brokendnssec.net.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
@@ -236,6 +241,7 @@ extended-resolution-errors=no
     def generateRecursorConfig(cls, confdir):
         super(NoExtendedErrorsTest, cls).generateRecursorConfig(confdir)
 
+    @pytest.mark.external
     def testNotIncepted(self):
         qname = 'signotincepted.bad-dnssec.wb.sidnlabs.nl.'
         query = dns.message.make_query(qname, 'A', want_dnssec=True)
diff --git a/regression-tests.recursor-dnssec/test_SimpleDoT.py b/regression-tests.recursor-dnssec/test_SimpleDoT.py
index 4802e4ef4c..20eaea132d 100644
--- a/regression-tests.recursor-dnssec/test_SimpleDoT.py
+++ b/regression-tests.recursor-dnssec/test_SimpleDoT.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import subprocess
@@ -30,6 +31,7 @@ devonly-regression-test-mode
         cls.generateRecursorConfig(confdir)
         cls.startRecursor(confdir, cls._recursorPort)
 
+    @pytest.mark.external
     def testTXT(self):
         expected = dns.rrset.from_text('dot-test-target.powerdns.org.', 0, dns.rdataclass.IN, 'TXT', 'https://github.com/PowerDNS/pdns/pull/12825')
         query = dns.message.make_query('dot-test-target.powerdns.org', 'TXT', want_dnssec=True)
diff --git a/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py b/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py
index d7b85e648c..8de66060b5 100644
--- a/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py
+++ b/regression-tests.recursor-dnssec/test_SimpleForwardOverDoT.py
@@ -1,3 +1,4 @@
+import pytest
 import dns
 import os
 import subprocess
@@ -27,6 +28,7 @@ devonly-regression-test-mode
         cls.generateRecursorConfig(confdir)
         cls.startRecursor(confdir, cls._recursorPort)
 
+    @pytest.mark.external
     def testA(self):
         expected = dns.rrset.from_text('dns.google.', 0, dns.rdataclass.IN, 'A', '8.8.8.8', '8.8.4.4')
         query = dns.message.make_query('dns.google', 'A', want_dnssec=True)
diff --git a/regression-tests.recursor-dnssec/test_WellKnown.py b/regression-tests.recursor-dnssec/test_WellKnown.py
index feaaa6f29d..fa04a9692c 100644
--- a/regression-tests.recursor-dnssec/test_WellKnown.py
+++ b/regression-tests.recursor-dnssec/test_WellKnown.py
@@ -1,7 +1,9 @@
+import pytest
 import dns
 from recursortests import RecursorTest
 
-class TestWellKnown(RecursorTest):
+@pytest.mark.external
+class WellKnownTest(RecursorTest):
     _auths_zones = None
     _confdir = 'WellKnown'
     _roothints = None