Handle invalid windowBits in init functions

Negative windowBits arguments are eventually turned positive in
deflateInit2_ and inflateInit2_ (more precisely in inflateReset2).
Such values are used to indicate that raw deflate/inflate should
be performed.

If a user supplies INT32_MIN for windowBits, the code will perform
-INT32_MIN which does not fit into int32_t. In fact, this is
undefined behavior in C and should be avoided.

Clearly this is a user error, but given the careful validation of
input arguments a few lines later in deflateInit2_ I think this
might be of interest.

Proof of Concept:

- Compile zlib-ng with gcc -ftrapv or -fsanitize=undefined
- Compile and run this program:

```
 #include <limits.h>
 #include <stdio.h>
 #include <zlib-ng.h>

 int main(void) {
  zng_stream de_stream = { 0 }, in_stream = { 0 };
  int result;

  result = zng_deflateInit2(&de_stream, 0, Z_DEFLATED, INT32_MIN,
      MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
  printf("zng_deflateInit2: %d\n", result);

  result = zng_inflateInit2(&in_stream, INT32_MIN);
  printf("zng_inflateInit2: %d\n", result);

  return 0;
 }
```

diff --git a/deflate.c b/deflate.c
index 4c7e179eae678081ceabafb2bcf0816a9766359b..f17fc2d31c15265b1df43942b9f143faf9d81062 100644 (file)
--- a/deflate.c
+++ b/deflate.c
@@ -284,6 +284,8 @@ int32_t Z_EXPORT PREFIX(deflateInit2_)(PREFIX3(stream) *strm, int32_t level, int
 
     if (windowBits < 0) { /* suppress zlib wrapper */
         wrap = 0;
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         windowBits = -windowBits;
 #ifdef GZIP
     } else if (windowBits > 15) {

diff --git a/inflate.c b/inflate.c
index a59cd44a92c6637be43071ea4bc0d2d014b5213f..5a774fa0847225a13eb44b52409fa0cd36e14d30 100644 (file)
--- a/inflate.c
+++ b/inflate.c
@@ -105,6 +105,8 @@ int32_t Z_EXPORT PREFIX(inflateReset2)(PREFIX3(stream) *strm, int32_t windowBits
     /* extract wrap request from windowBits parameter */
     if (windowBits < 0) {
         wrap = 0;
+        if (windowBits < -15)
+            return Z_STREAM_ERROR;
         windowBits = -windowBits;
     } else {
         wrap = (windowBits >> 4) + 5;