tests: updated tlsfuzzer tests to latest version

This adds new tests, reduces running time, and removes test-tls13-obsolete-curves.py.
The latter introduced too pendantic tests on curves we don't implement,
and requires significant changes to passing with limited benefit. For example
it requires the server to error on mismatching entries (and we simply ignore
them). As its value is low (we do not target to be a reference implementation
for testing broken clients), it was removed.

Signed-off-by: Nikos Mavrogiannopoulos <nmav@gnutls.org>

diff --git a/tests/suite/tls-fuzzer/gnutls-cert.json b/tests/suite/tls-fuzzer/gnutls-cert.json
index f0443d8a7de3586e9835fb73fe06012ddcf057d6..7a5af26e53aaa29f5f6929b37fb521fbe895b4db 100644 (file)
--- a/tests/suite/tls-fuzzer/gnutls-cert.json
+++ b/tests/suite/tls-fuzzer/gnutls-cert.json
@@ -2,7 +2,7 @@
     {"server_command": ["@SERVER@", "--http",
                  "--x509keyfile", "tests/serverX509Key.pem",
                  "--x509certfile", "tests/serverX509Cert.pem",
-                 "--debug=4",
+                 "--debug=6",
                  "--priority=@PRIORITY@",
                  "--port=@PORT@"],
      "environment": {"PYTHONPATH" : "."},
@@ -10,16 +10,20 @@
      "server_port": @PORT@,
      "tests" : [
         {"name" : "test-tls13-certificate-verify.py",
+          "comment" : "tlsfuzzer doesn't like our status request (see #633)",
+          "exp_pass" : false,
           "comment" : "tlsfuzzer doesn't like our set of algorithms (e.g., ed25519)",
           "arguments" : ["-k", "tests/clientX509Key.pem",
                          "-c", "tests/clientX509Cert.pem",
                          "-n", "10",
                          "-e", "check sigalgs in cert request",
                         "-p", "@PORT@"]},
-        {"name" : "test-tls13-certificate-verify.py",
+        {"name" : "test-tls13-ecdsa-in-certificate-verify.py",
+          "comment" : "tlsfuzzer doesn't like our status request (see #633)",
+          "exp_pass" : false,
           "comment" : "tlsfuzzer doesn't like our set of algorithms (e.g., ed25519)",
-          "arguments" : ["-k", "tests/clientRSAPSSKey.pem",
-                         "-c", "tests/clientRSAPSSCert.pem",
+          "arguments" : ["-k", "tests/serverECKey.pem",
+                         "-c", "tests/serverECCert.pem",
                          "-n", "10",
                          "-e", "check sigalgs in cert request",
                         "-p", "@PORT@"]},
@@ -33,6 +37,18 @@
                          "-c", "tests/clientX509Cert.pem",
                          "-p", "@PORT@"]
           },
+         {"name" : "test-ecdsa-in-certificate-verify.py",
+          "comment" : "we don't support sha224; we send illegal_parameter instead of handshake_failure in md5+ecdsa",
+          "arguments" : ["-k", "tests/serverECKey.pem",
+                         "-c", "tests/serverECCert.pem",
+                         "-e", "make sha224+ecdsa signature in CertificateVerify",
+                         "-e", "make sha224+ecdsa signature, advertise it as sha1+ecdsa in CertificateVerify",
+                         "-e", "make sha224+ecdsa signature, advertise it as sha256+ecdsa in CertificateVerify",
+                         "-e", "make sha224+ecdsa signature, advertise it as sha384+ecdsa in CertificateVerify",
+                         "-e", "make sha224+ecdsa signature, advertise it as sha512+ecdsa in CertificateVerify",
+                         "-e", "md5+ecdsa forced",
+                         "-p", "@PORT@"]
+          },
          {"name" : "test-certificate-verify-malformed.py",
           "arguments" : ["-k", "tests/clientX509Key.pem",
                          "-c", "tests/clientX509Cert.pem",
@@ -44,10 +60,11 @@
                          "-p", "@PORT@"]
           },
          {"name" : "test-certificate-request.py",
-          "comment" : "tlsfuzzer doesn't like our set of algorithms",
+          "comment" : "tlsfuzzer doesn't like our set of algorithms or supported cert types",
           "arguments" : ["-k", "tests/clientX509Key.pem",
                          "-c", "tests/clientX509Cert.pem",
                          "-e", "check sigalgs in cert request",
+                         "-e", "check cert types in cert request",
                          "-p", "@PORT@"]
           },
          {"name" : "test-rsa-pss-sigs-on-certificate-verify.py",

diff --git a/tests/suite/tls-fuzzer/gnutls-nocert-tls13.json b/tests/suite/tls-fuzzer/gnutls-nocert-tls13.json
index e293b1ce7822e3a69df39dd4a44d623dcdf964e7..0bf5be4bea5fac132ef02e146aadb660cd016e84 100644 (file)
--- a/tests/suite/tls-fuzzer/gnutls-nocert-tls13.json
+++ b/tests/suite/tls-fuzzer/gnutls-nocert-tls13.json
@@ -4,9 +4,13 @@
                  "--x509certfile", "tests/serverX509Cert.pem",
                  "--x509keyfile", "tests/serverRSAPSSKey.pem",
                  "--x509certfile", "tests/serverRSAPSSCert.pem",
-                 "--x509keyfile", "../../../certs/ecc256.pem",
-                 "--x509certfile", "../../../certs/cert-ecc256.pem",
-                 "--debug=3",
+                 "--x509keyfile", "tests/serverECKey.pem",
+                 "--x509certfile", "tests/serverECCert.pem",
+                 "--x509keyfile", "tests/serverP384ECKey.pem",
+                 "--x509certfile", "tests/serverP384ECCert.pem",
+                 "--x509keyfile", "tests/serverP521ECKey.pem",
+                 "--x509certfile", "tests/serverP521ECCert.pem",
+                 "--debug=6",
                  "--httpdata=../http.dat",
                  "--priority=@PRIORITY@",
                  "--disable-client-cert", "--port=@PORT@"],
@@ -52,9 +56,21 @@
          "comment": "We do not support x448",
          "arguments": ["-p", "@PORT@",
                        "-e", "TLS 1.3 with x448",
-                       "-n", "5"]},
+                       "-n", "4"]},
+        {"name" : "test-tls13-ecdhe-curves.py",
+         "arguments": ["-p", "@PORT@"]},
+        {"name" : "test-tls13-ecdsa-support.py",
+         "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-empty-alert.py",
          "arguments": ["-p", "@PORT@"]},
+        {"name" : "test-tls13-ffdhe-groups.py",
+         "comment" : "We do not check for duplicate entries",
+         "arguments": ["-p", "@PORT@",
+                       "-e", "ffdhe2048 - duplicated key share entry",
+                       "-e", "ffdhe3072 - duplicated key share entry",
+                       "-e", "ffdhe4096 - duplicated key share entry",
+                       "-e", "ffdhe6144 - duplicated key share entry",
+                       "-e", "ffdhe8192 - duplicated key share entry"]},
         {"name" : "test-tls13-ffdhe-sanity.py",
          "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-finished.py",
@@ -68,6 +84,16 @@
          "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-keyshare-omitted.py",
          "arguments": ["-p", "@PORT@"]},
+        {"name" : "test-tls13-keyupdate.py",
+         "comment" : "we have limits that prohibit the running multiple messages test; app data split timeouts waiting for new session ticket",
+         "arguments": ["-p", "@PORT@",
+                       "-e", "app data split, conversation with KeyUpdate msg",
+                       "-e", "multiple KeyUpdate messages"]},
+        {"name" : "test-tls13-large-number-of-extensions.py",
+         "comment" : "This test assumes that 22 (EtM) is unassigned which is incorrect - see #632",
+         "arguments": ["-p", "@PORT@",
+                       "-e", "empty unassigned extensions, ids in range from 2 to 4118",
+                       "-e", "unassigned extensions with random payload, ids in range from 2 to 1046"]},
         {"name" : "test-tls13-legacy-version.py",
          "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-nociphers.py",
@@ -75,8 +101,6 @@
         {"name" : "test-tls13-non-support.py",
          "arguments": ["-p", "@PORT@"],
          "exp_pass" : false},
-        {"name" : "test-tls13-obsolete-curves.py",
-         "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-pkcs-signature.py",
          "arguments": ["-p", "@PORT@"]},
         {"name" : "test-tls13-record-padding.py",

diff --git a/tests/suite/tls-fuzzer/gnutls-nocert.json b/tests/suite/tls-fuzzer/gnutls-nocert.json
index bef461789f09dd676a6939fdc2a94943624e8a4a..2a6f47d68dbeb908f5160dbad215e5fbc8ea3581 100644 (file)
--- a/tests/suite/tls-fuzzer/gnutls-nocert.json
+++ b/tests/suite/tls-fuzzer/gnutls-nocert.json
@@ -2,9 +2,13 @@
     {"server_command": ["@SERVER@", "--http",
                  "--x509keyfile", "tests/serverX509Key.pem",
                  "--x509certfile", "tests/serverX509Cert.pem",
-                 "--x509keyfile", "../../../certs/ecc256.pem",
-                 "--x509certfile", "../../../certs/cert-ecc256.pem",
-                 "--debug=3",
+                 "--x509keyfile", "tests/serverECKey.pem",
+                 "--x509certfile", "tests/serverECCert.pem",
+                 "--x509keyfile", "tests/serverP384ECKey.pem",
+                 "--x509certfile", "tests/serverP384ECCert.pem",
+                 "--x509keyfile", "tests/serverP521ECKey.pem",
+                 "--x509certfile", "tests/serverP521ECCert.pem",
+                 "--debug=6",
                  "--noticket",
                  "--httpdata=../http.dat",
                  "--priority=@PRIORITY@",
@@ -13,7 +17,8 @@
      "server_port": @PORT@,
      "tests" : [
          {"name" : "test-fuzzed-plaintext.py",
-          "arguments" : ["-p", "@PORT@"] },
+          "arguments" : ["-p", "@PORT@",
+                         "-n", "4"] },
          {"name" : "test-large-hello.py",
           "arguments" : [
           "-p", "@PORT@",
@@ -31,7 +36,9 @@
           "fragmented, padding ext 65354 bytes",
           "fragmented, padding ext 16213 bytes"]},
          {"name" : "test-ecdsa-sig-flexibility.py",
-          "arguments" : ["-p", "@PORT@"] },
+          "comment" : "we do not support SHA224",
+          "arguments" : ["-p", "@PORT@",
+                         "-e", "connect with sha224+ecdsa only"] },
          {"name" : "test-encrypt-then-mac.py",
           "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-ocsp-stapling.py",
@@ -64,11 +71,13 @@
           "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-aes-gcm-nonces.py",
           "arguments" : ["-p", "@PORT@"] },
+         {"name" : "test-aesccm.py",
+          "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-atypical-padding.py",
           "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-bleichenbacher-workaround.py",
           "arguments" : ["-p", "@PORT@",
-                         "-n", "20"]
+                         "-n", "5"]
          },
          {"name" : "test-clienthello-md5.py",
           "arguments" : ["-p", "@PORT@"] },
@@ -79,6 +88,8 @@
                          "-e", "100: IE 6 on XP"]},
          {"name" : "test-conversation.py",
           "arguments" : ["-p", "@PORT@"] },
+         {"name" : "test-no-heartbeat.py",
+          "arguments" : ["-p", "@PORT@"] },
         {"name" : "test-client-hello-max-size.py",
          "comment" : "FIXME: we fail with: Handshake buffer length is 131400 (max: 131072)",
          "arguments" : ["-p", "@PORT@",
@@ -101,7 +112,7 @@
           "arguments" : ["-p", "@PORT@",
                          "-e", "Protocol (3, 0)",
                          "-e", "Protocol (3, 0) in SSLv2 compatible ClientHello",
-                         "-n", "6",
+                         "-n", "5",
                          "-z"]},
          {"name" : "test-dhe-rsa-key-exchange.py",
           "arguments" : ["-p", "@PORT@"] },
@@ -124,7 +135,7 @@
                          "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello",
                          "-e", "Protocol (3, 0)",
                          "-z",
-                         "-n", "6"]},
+                         "-n", "5"]},
          {"name" : "test-ecdhe-rsa-key-exchange.py",
           "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-ecdhe-rsa-key-exchange-with-bad-messages.py",
@@ -135,7 +146,7 @@
                          "-e", "Protocol (3, 1) in SSLv2 compatible ClientHello",
                          "-e", "Protocol (3, 0)",
                          "-z",
-                         "-n", "6"]},
+                         "-n", "5"]},
          {"name" : "test-empty-extensions.py",
           "arguments" : ["-p", "@PORT@"] },
          {"name" : "test-export-ciphers-rejected.py",
@@ -264,7 +275,7 @@
                          "-e", "duplicated 2342 non-rsa schemes",
                          "-e", "duplicated 8119 non-rsa schemes",
                          "-e", "duplicated 23741 non-rsa schemes",
-                         "-e", "duplicated 32748 non-rsa schemes",
+                         "-e", "duplicated 32744 non-rsa schemes",
                          "-e", "tolerance max (32764) number of methods"]
          },
          {"name" : "test-sslv2-connection.py",

diff --git a/tests/suite/tls-fuzzer/tls-fuzzer-nocert-tls13.sh b/tests/suite/tls-fuzzer/tls-fuzzer-nocert-tls13.sh
index 1b9b0f1765e662bdb07905c94d36a37b0cf79271..f9268627a2dd09e8a80dcaa41278bf70377e0818 100755 (executable)
--- a/tests/suite/tls-fuzzer/tls-fuzzer-nocert-tls13.sh
+++ b/tests/suite/tls-fuzzer/tls-fuzzer-nocert-tls13.sh
@@ -21,7 +21,7 @@
 srcdir="${srcdir:-.}"
 
 tls_fuzzer_prepare() {
-PRIORITY="NORMAL:-VERS-ALL:+VERS-TLS1.3:+VERS-TLS1.2:+VERS-TLS1.1"
+PRIORITY="NORMAL:-VERS-ALL:+VERS-TLS1.3:+VERS-TLS1.2:+VERS-TLS1.1:+AES-128-CCM:+AES-256-CCM:+AES-128-CCM-8"
 
 sed -e "s|@SERVER@|$SERV|g" -e "s/@PORT@/$PORT/g" -e "s/@PRIORITY@/$PRIORITY/g" ../gnutls-nocert-tls13.json >${TMPFILE}
 }

diff --git a/tests/suite/tls-fuzzer/tls-fuzzer-nocert.sh b/tests/suite/tls-fuzzer/tls-fuzzer-nocert.sh
index 6e6b809c5707478d3bea5ccd133755d4f73d5fee..5405fc936c4f86dd3c8a43948ba39f6b4f8c213e 100755 (executable)
--- a/tests/suite/tls-fuzzer/tls-fuzzer-nocert.sh
+++ b/tests/suite/tls-fuzzer/tls-fuzzer-nocert.sh
@@ -22,10 +22,10 @@ srcdir="${srcdir:-.}"
 
 tls_fuzzer_prepare() {
 VERSIONS="-VERS-ALL:+VERS-TLS1.3:+VERS-TLS1.2:+VERS-TLS1.1:+VERS-TLS1.0:+VERS-SSL3.0"
-PRIORITY="NORMAL:%VERIFY_ALLOW_SIGN_WITH_SHA1:+ARCFOUR-128:+3DES-CBC:+DHE-DSS:+SIGN-DSA-SHA256:+SIGN-DSA-SHA1:-CURVE-SECP192R1:${VERSIONS}:+SHA256:+SHA384"
+PRIORITY="NORMAL:%VERIFY_ALLOW_SIGN_WITH_SHA1:+ARCFOUR-128:+3DES-CBC:+DHE-DSS:+SIGN-DSA-SHA256:+SIGN-DSA-SHA1:-CURVE-SECP192R1:${VERSIONS}:+SHA256:+SHA384:+AES-128-CCM:+AES-256-CCM:+AES-128-CCM-8:+AES-256-CCM-8"
 ${CLI} --list --priority "${PRIORITY}" >/dev/null 2>&1
 if test $? != 0;then
-       PRIORITY="NORMAL:%VERIFY_ALLOW_SIGN_WITH_SHA1:+ARCFOUR-128:+3DES-CBC:+DHE-DSS:+SIGN-DSA-SHA256:+SIGN-DSA-SHA1:${VERSIONS}:+SHA256:+SHA384"
+       PRIORITY="NORMAL:%VERIFY_ALLOW_SIGN_WITH_SHA1:+ARCFOUR-128:+3DES-CBC:+DHE-DSS:+SIGN-DSA-SHA256:+SIGN-DSA-SHA1:${VERSIONS}:+SHA256:+SHA384:+AES-128-CCM:+AES-256-CCM:+AES-128-CCM-8:+AES-256-CCM-8"
 fi
 
 sed -e "s|@SERVER@|$SERV|g" -e "s/@PORT@/$PORT/g" -e "s/@PRIORITY@/$PRIORITY/g" ../gnutls-nocert.json >${TMPFILE}

diff --git a/tests/suite/tls-fuzzer/tlsfuzzer b/tests/suite/tls-fuzzer/tlsfuzzer
index 3d57169c83e960597d7f90f4b837858d9530d7fb..ca536d11ac14da2deacbde95f3f0a70a5ce42112 160000 (submodule)
--- a/tests/suite/tls-fuzzer/tlsfuzzer
+++ b/tests/suite/tls-fuzzer/tlsfuzzer
@@ -1 +1 @@
-Subproject commit 3d57169c83e960597d7f90f4b837858d9530d7fb
+Subproject commit ca536d11ac14da2deacbde95f3f0a70a5ce42112

diff --git a/tests/suite/tls-fuzzer/tlslite-ng b/tests/suite/tls-fuzzer/tlslite-ng
index 029425144f6b26d5ab8acc3a79e3ead79253ad71..bff8773e4309cc43dd6acf0a6d4623949f911fd7 160000 (submodule)
--- a/tests/suite/tls-fuzzer/tlslite-ng
+++ b/tests/suite/tls-fuzzer/tlslite-ng
@@ -1 +1 @@
-Subproject commit 029425144f6b26d5ab8acc3a79e3ead79253ad71
+Subproject commit bff8773e4309cc43dd6acf0a6d4623949f911fd7