IBM zSystems: Fix calling deflateBound() before deflateInit()

Even though zlib officialy forbids calling deflateBound() before
deflateInit(), Firefox does this anyway, and it happens to work [1],
but unfortunately not with DFLTCC [2], because the DFLTCC code assumes
that the deflate state is allocated, and segfaults when it isn't.

Bow down before Hyrum's Law and add deflateStateCheck() to
DEFLATE_BOUND_ADJUST_COMPLEN().

Extend the deflate_bound test accordingly.

[1] https://searchfox.org/mozilla-esr102/source/dom/script/ScriptCompression.cpp#97
[2] https://bugzilla.suse.com/show_bug.cgi?id=1210593

diff --git a/arch/s390/dfltcc_deflate.h b/arch/s390/dfltcc_deflate.h
index d12523017df37c76e2e1760771f9cd263f366e6e..cb261b156c78b203ed04f55fbc80ba58b47abe69 100644 (file)
--- a/arch/s390/dfltcc_deflate.h
+++ b/arch/s390/dfltcc_deflate.h
@@ -45,7 +45,7 @@ int Z_INTERNAL PREFIX(dfltcc_deflate_get_dictionary)(PREFIX3(streamp) strm, unsi
 
 #define DEFLATE_BOUND_ADJUST_COMPLEN(strm, complen, source_len) \
     do { \
-        if (PREFIX(dfltcc_can_deflate)((strm))) \
+        if (deflateStateCheck((strm)) || PREFIX(dfltcc_can_deflate)((strm))) \
             (complen) = DEFLATE_BOUND_COMPLEN(source_len); \
     } while (0)
 

diff --git a/test/test_deflate_bound.cc b/test/test_deflate_bound.cc
index 8de65535f29439047d9b9f8082b5d61be05ffdc8..33bc452ebdc873d6d64a0b16d4762044bb752518 100644 (file)
--- a/test/test_deflate_bound.cc
+++ b/test/test_deflate_bound.cc
@@ -22,12 +22,15 @@ typedef struct {
     int32_t level;
     int32_t window_size;
     int32_t mem_level;
+    bool after_init;
 } deflate_bound_test;
 
 static const deflate_bound_test tests[] = {
-    {0, MAX_WBITS + 16, 1},
-    {Z_BEST_SPEED, MAX_WBITS, MAX_MEM_LEVEL},
-    {Z_BEST_COMPRESSION, MAX_WBITS, MAX_MEM_LEVEL}
+    {0, MAX_WBITS + 16, 1, true},
+    {Z_BEST_SPEED, MAX_WBITS, MAX_MEM_LEVEL, true},
+    {Z_BEST_COMPRESSION, MAX_WBITS, MAX_MEM_LEVEL, true},
+    {Z_BEST_SPEED, MAX_WBITS, MAX_MEM_LEVEL, false},
+    {Z_BEST_COMPRESSION, MAX_WBITS, MAX_MEM_LEVEL, false},
 };
 
 class deflate_bound_variant : public testing::TestWithParam<deflate_bound_test> {
@@ -51,12 +54,16 @@ public:
             c_stream.avail_out = 0;
             c_stream.next_out = out_buf;
 
+            if (!param.after_init)
+                estimate_len = PREFIX(deflateBound)(&c_stream, i);
+
             err = PREFIX(deflateInit2)(&c_stream, param.level, Z_DEFLATED,
                 param.window_size, param.mem_level, Z_DEFAULT_STRATEGY);
             EXPECT_EQ(err, Z_OK);
 
             /* calculate actual output length and update structure */
-            estimate_len = PREFIX(deflateBound)(&c_stream, i);
+            if (param.after_init)
+                estimate_len = PREFIX(deflateBound)(&c_stream, i);
             out_buf = (uint8_t *)malloc(estimate_len);
 
             if (out_buf != NULL) {
@@ -70,6 +77,7 @@ public:
                     "level: " << param.level << "\n" <<
                     "window_size: " << param.window_size << "\n" <<
                     "mem_level: " << param.mem_level << "\n" <<
+                    "after_init: " << param.after_init << "\n" <<
                     "length: " << i;
 
                 free(out_buf);