OPENSSL_hexstr2buf_ex(): Handle zero-length input correctly

In case of zero-length input the code wrote one byte
before the start of the output buffer. The length
of the output was also reported incorrectly in this case.

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Neil Horman <nhorman@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/24770)

(cherry picked from commit 3f7b355733407cf777bfad5ce5b79610588bacc5)

diff --git a/crypto/o_str.c b/crypto/o_str.c
index ff6d3362250310aeb4c40ba7a8406af7ea855792..e3416b122e77f2e4d2ee0497290d32fa2256d9b3 100644 (file)
--- a/crypto/o_str.c
+++ b/crypto/o_str.c
@@ -229,12 +229,14 @@ static int buf2hexstr_sep(char *str, size_t str_n, size_t *strlength,
     int has_sep = (sep != CH_ZERO);
     size_t len = has_sep ? buflen * 3 : 1 + buflen * 2;
 
+    if (len == 0)
+        ++len;
     if (strlength != NULL)
         *strlength = len;
     if (str == NULL)
         return 1;
 
-    if (str_n < (unsigned long)len) {
+    if (str_n < len) {
         ERR_raise(ERR_LIB_CRYPTO, CRYPTO_R_TOO_SMALL_BUFFER);
         return 0;
     }
@@ -246,7 +248,7 @@ static int buf2hexstr_sep(char *str, size_t str_n, size_t *strlength,
         if (has_sep)
             *q++ = sep;
     }
-    if (has_sep)
+    if (has_sep && buflen > 0)
         --q;
     *q = CH_ZERO;
 

diff --git a/test/hexstr_test.c b/test/hexstr_test.c
index 5a9684e0e6977932d3448cedb2d4d0b7363d2bfd..89d6d9713978ceebf5f2d22bff7b0d481558b319 100644 (file)
--- a/test/hexstr_test.c
+++ b/test/hexstr_test.c
@@ -120,9 +120,14 @@ static int test_hexstr_ex_to_from(int test_index)
 
     return TEST_true(OPENSSL_hexstr2buf_ex(buf, sizeof(buf), &len, test->in, ':'))
            && TEST_mem_eq(buf, len, test->expected, test->expected_len)
+           && TEST_false(OPENSSL_buf2hexstr_ex(out, 3 * len - 1, NULL, buf, len,
+                                               ':'))
            && TEST_true(OPENSSL_buf2hexstr_ex(out, sizeof(out), NULL, buf, len,
-                        ':'))
-           && TEST_str_eq(out, test->in);
+                                              ':'))
+           && TEST_str_eq(out, test->in)
+           && TEST_true(OPENSSL_buf2hexstr_ex(out, sizeof(out), NULL, buf, 0,
+                                              ':'))
+           && TEST_size_t_eq(strlen(out), 0);
 }
 
 int setup_tests(void)