Handle invalid windowBits in init functions

Negative windowBits arguments are eventually turned positive in
deflateInit2_ and inflateInit2_ (more precisely in inflateReset2).
Such values are used to indicate that raw deflate/inflate should
be performed.

If a user supplies INT32_MIN for windowBits, the code will perform
-INT32_MIN which does not fit into int32_t. In fact, this is
undefined behavior in C and should be avoided.

Clearly this is a user error, but given the careful validation of
input arguments a few lines later in deflateInit2_ I think this
might be of interest.

Proof of Concept:

- Compile zlib-ng with gcc -ftrapv or -fsanitize=undefined
- Compile and run this program:

```
 #include <limits.h>
 #include <stdio.h>
 #include <zlib-ng.h>

 int main(void) {
  zng_stream de_stream = { 0 }, in_stream = { 0 };
  int result;

  result = zng_deflateInit2(&de_stream, 0, Z_DEFLATED, INT32_MIN,
      MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
  printf("zng_deflateInit2: %d\n", result);

  result = zng_inflateInit2(&in_stream, INT32_MIN);
  printf("zng_inflateInit2: %d\n", result);

  return 0;
 }
```

diff --git a/deflate.c b/deflate.c
index 71920982d48da02bab1c3bb12252d9e5ada6c19e..2970aa4e08855c74a0131521f561240958d73dbe 100644 (file)
--- a/deflate.c
+++ b/deflate.c
@@ -213,6 +213,8 @@ int32_t ZNG_CONDEXPORT PREFIX(deflateInit2)(PREFIX3(stream) *strm, int32_t level
 
     if (windowBits < 0) { /* suppress zlib wrapper */
         wrap = 0;
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         windowBits = -windowBits;
 #ifdef GZIP
     } else if (windowBits > 15) {

diff --git a/inflate.c b/inflate.c
index 5f52dd140ada0cab7c6d7354d21651b93078077c..4bd9b938a2f1cc4f3e9f81d20410fee72ed72608 100644 (file)
--- a/inflate.c
+++ b/inflate.c
@@ -111,6 +111,8 @@ int32_t Z_EXPORT PREFIX(inflateReset2)(PREFIX3(stream) *strm, int32_t windowBits
     /* extract wrap request from windowBits parameter */
     if (windowBits < 0) {
         wrap = 0;
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         windowBits = -windowBits;
     } else {
         wrap = (windowBits >> 4) + 5;