Fix sanity tests for ssl_version_cmp for dtls 1.3 branch

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24293)

diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index 88b252a32359252958ec0f02eec5cceac6ab74e3..2424c93484b8a588d2fe5535bd4ba409757fc76c 100644 (file)
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -2202,10 +2202,10 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello,
         return SSL_R_UNSUPPORTED_PROTOCOL;
 
     if (suppversions->present) {
-        unsigned int candidate_vers = 0;
-        const unsigned int best_vers_init = SSL_CONNECTION_IS_DTLS(s) ? UINT_MAX
-                                                                      : 0;
-        unsigned int best_vers = best_vers_init;
+        int candidate_vers = 0;
+        const int best_vers_init = SSL_CONNECTION_IS_DTLS(s) ? INT_MAX
+                                                             : 0;
+        int best_vers = best_vers_init;
         const SSL_METHOD *best_method = NULL;
         PACKET versionslist;
 
@@ -2228,9 +2228,9 @@ int ssl_choose_server_version(SSL_CONNECTION *s, CLIENTHELLO_MSG *hello,
         if (client_version <= SSL3_VERSION)
             return SSL_R_BAD_LEGACY_VERSION;
 
-        while (PACKET_get_net_2(&versionslist, &candidate_vers)) {
+        while (PACKET_get_net_2(&versionslist, (unsigned int*)&candidate_vers)) {
             if (candidate_vers <= 0
-                    || (best_vers != 0
+                    || (best_vers != best_vers_init
                         && ssl_version_cmp(s, candidate_vers, best_vers) <= 0))
                 continue;
             if (ssl_version_supported(s, candidate_vers, &best_method))

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 8e0795553b292e5ff7828e058bb99a386c90deeb..18a7f9c6e07a6783d8302641ba7574a6d5cb7d8d 100644 (file)
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2641,7 +2641,8 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op,
 {
     unsigned char sigalgstr[2];
     int secbits;
-    int dsa_version_limit;
+    const int version1_3 = SSL_CONNECTION_IS_DTLS(s) ? DTLS1_3_VERSION
+                                                     : TLS1_3_VERSION;
 
     if (lu == NULL || !lu->enabled)
         return 0;
@@ -2652,8 +2653,8 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op,
      * At some point we should fully axe DSA/etc. in ClientHello as per (D)TLSv1.3
      * spec
      */
-    dsa_version_limit = SSL_CONNECTION_IS_DTLS(s) ? DTLS1_3_VERSION : TLS1_3_VERSION;
-    if (!s->server && ssl_version_cmp(s, s->s3.tmp.min_ver, dsa_version_limit) >= 0
+    if (!s->server && s->s3.tmp.min_ver > 0
+        && ssl_version_cmp(s, s->s3.tmp.min_ver, version1_3) >= 0
         && (lu->sig == EVP_PKEY_DSA || lu->hash_idx == SSL_MD_SHA1_IDX
             || lu->hash_idx == SSL_MD_MD5_IDX
             || lu->hash_idx == SSL_MD_SHA224_IDX))
@@ -2667,14 +2668,14 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op,
             || lu->sig == NID_id_GostR3410_2012_512
             || lu->sig == NID_id_GostR3410_2001) {
         int any_version = SSL_CONNECTION_IS_DTLS(s) ? DTLS_ANY_VERSION : TLS_ANY_VERSION;
-        int gost_version_limit = SSL_CONNECTION_IS_DTLS(s) ? DTLS1_3_VERSION : TLS1_3_VERSION;
 
         /* We never allow GOST sig algs on the server with (D)TLSv1.3 */
         if (s->server && SSL_CONNECTION_IS_VERSION13(s))
             return 0;
         if (!s->server
                 && SSL_CONNECTION_GET_SSL(s)->method->version == any_version
-                && ssl_version_cmp(s, s->s3.tmp.max_ver, gost_version_limit) >= 0) {
+                && s->s3.tmp.max_ver > 0
+                && ssl_version_cmp(s, s->s3.tmp.max_ver, version1_3) >= 0) {
             int i, num;
             STACK_OF(SSL_CIPHER) *sk;
 
@@ -2684,7 +2685,8 @@ static int tls12_sigalg_allowed(const SSL_CONNECTION *s, int op,
              * ciphersuites enabled.
              */
 
-            if (ssl_version_cmp(s, s->s3.tmp.min_ver, gost_version_limit) >= 0)
+            if (s->s3.tmp.min_ver > 0
+                && ssl_version_cmp(s, s->s3.tmp.min_ver, version1_3) >= 0)
                 return 0;
 
             sk = SSL_get_ciphers(SSL_CONNECTION_GET_SSL(s));