gmp: Fix RSA signature verification for m >= n

By definition, m must be <= n-1, we didn't enforce that and because
mpz_export() returns NULL if the passed value is zero a crash could have
been triggered with m == n.

Fixes CVE-2017-11185.

diff --git a/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c b/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
index 32a72ac9600b871b47ab96e1d8ac7713f957d56b..065c889033440bc34969631a26982a198bcdddf2 100644 (file)
--- a/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
+++ b/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
@@ -78,11 +78,17 @@ static chunk_t rsaep(private_gmp_rsa_public_key_t *this, chunk_t data)
        mpz_t m, c;
        chunk_t encrypted;
 
-       mpz_init(c);
        mpz_init(m);
-
        mpz_import(m, data.len, 1, 1, 1, 0, data.ptr);
 
+       if (mpz_cmp_ui(m, 0) <= 0 || mpz_cmp(m, this->n) >= 0)
+       {       /* m must be <= n-1, and while 0 is technically a valid value, it
+                * doesn't really make sense here, so we filter that too */
+               mpz_clear(m);
+               return chunk_empty;
+       }
+
+       mpz_init(c);
        mpz_powm(c, m, this->e, this->n);
 
        encrypted.len = this->k;
@@ -150,7 +156,7 @@ static bool verify_emsa_pkcs1_signature(private_gmp_rsa_public_key_t *this,
         */
 
        /* check magic bytes */
-       if (*(em.ptr) != 0x00 || *(em.ptr+1) != 0x01)
+       if (em.len < 2 || *(em.ptr) != 0x00 || *(em.ptr+1) != 0x01)
        {
                goto end;
        }