Add "reproducible" deflate parameter

IBM Z DEFLATE CONVERSION CALL may produce different (but valid)
compressed data for the same uncompressed data. This behavior might be
unacceptable for certain use cases (e.g. reproducible builds). This
patch introduces Z_DEFLATE_REPRODUCIBLE parameter, which can be used to
indicate that this is the case, and turn off IBM Z DEFLATE CONVERSION
CALL.

diff --git a/arch/s390/README.md b/arch/s390/README.md
index 6995b106ecb6bd6c8bb62857270f5ae18ea0e5ae..841eb896c70ce821ca6c3722d120ebc32948718d 100644 (file)
--- a/arch/s390/README.md
+++ b/arch/s390/README.md
@@ -20,7 +20,10 @@ Two DFLTCC compression calls produce the same results only when they
 both are made on machines of the same generation, and when the
 respective buffers have the same offset relative to the start of the
 page. Therefore care should be taken when using hardware compression
-when reproducible results are desired.
+when reproducible results are desired. In particular, zlib-ng-specific
+zng_deflateSetParams call allows setting Z_DEFLATE_REPRODUCIBLE
+parameter, which would disable DFLTCC if reproducible results are
+required.
 
 DFLTCC does not support every single zlib-ng feature, in particular:
 
@@ -67,3 +70,11 @@ In addition to compression, DFLTCC computes CRC-32 and Adler-32
 checksums, therefore, whenever it's used, software checksumming is
 suppressed using DEFLATE_NEED_CHECKSUM and INFLATE_NEED_CHECKSUM
 macros.
+
+While software always produces reproducible compression results, this
+is not the case for DFLTCC. Therefore, zlib-ng users are given the
+ability to specify whether or not reproducible compression results
+are required. While it is always possible to specify this setting
+before the compression begins, it is not always possible to do so in
+the middle of a deflate stream - the exact conditions for that are
+determined by DEFLATE_CAN_SET_REPRODUCIBLE macro.

diff --git a/arch/s390/dfltcc_deflate.c b/arch/s390/dfltcc_deflate.c
index adab3e000062c4c4e52045bc4a5093bdab790b96..e164d987cbbf8bc90fb1355e7d74e039b0de0aae 100644 (file)
--- a/arch/s390/dfltcc_deflate.c
+++ b/arch/s390/dfltcc_deflate.c
@@ -19,11 +19,13 @@
 #include "dfltcc_deflate.h"
 #include "dfltcc_detail.h"
 
-static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask)
+static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask,
+                                       int reproducible)
 {
     return (level_mask & ((uint16_t)1 << level)) != 0 &&
         (window_bits == HB_BITS) &&
-        (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY);
+        (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY) &&
+        !reproducible;
 }
 
 
@@ -33,7 +35,8 @@ int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm)
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
 
     /* Unsupported compression settings */
-    if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask))
+    if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask,
+                              state->reproducible))
         return 0;
 
     /* Unsupported hardware */
@@ -277,19 +280,27 @@ again:
    fly with deflateParams, we need to convert between hardware and software
    window formats.
 */
+static int dfltcc_was_deflate_used(PREFIX3(streamp) strm)
+{
+    deflate_state *state = (deflate_state *)strm->state;
+    struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param;
+
+    return strm->total_in > 0 || param->nt == 0 || param->hl > 0;
+}
+
 int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy)
 {
     deflate_state *state = (deflate_state *)strm->state;
     struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state);
-    struct dfltcc_param_v0 *param = &dfltcc_state->param;
     int could_deflate = dfltcc_can_deflate(strm);
-    int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask);
+    int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask,
+                                           state->reproducible);
 
     if (can_deflate == could_deflate)
         /* We continue to work in the same mode - no changes needed */
         return Z_OK;
 
-    if (strm->total_in == 0 && param->nt == 1 && param->hl == 0)
+    if (!dfltcc_was_deflate_used(strm))
         /* DFLTCC was not used yet - no changes needed */
         return Z_OK;
 
@@ -297,6 +308,13 @@ int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int st
     return Z_STREAM_ERROR;
 }
 
+int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible)
+{
+    deflate_state *state = (deflate_state *)strm->state;
+
+    return reproducible != state->reproducible && !dfltcc_was_deflate_used(strm);
+}
+
 /*
    Preloading history.
 */

diff --git a/arch/s390/dfltcc_deflate.h b/arch/s390/dfltcc_deflate.h
index 93945f0a0276eea37265bfbfa184a17ec7e930c0..8775321b3a3fad6164bd9c729cf47c778b36f49c 100644 (file)
--- a/arch/s390/dfltcc_deflate.h
+++ b/arch/s390/dfltcc_deflate.h
@@ -6,6 +6,7 @@
 int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm);
 int ZLIB_INTERNAL dfltcc_deflate(PREFIX3(streamp) strm, int flush, block_state *result);
 int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy);
+int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible);
 int ZLIB_INTERNAL dfltcc_deflate_set_dictionary(PREFIX3(streamp) strm,
                                                 const unsigned char *dictionary, uInt dict_length);
 int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned char *dictionary, uInt* dict_length);
@@ -47,4 +48,6 @@ int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned
 
 #define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm)))
 
+#define DEFLATE_CAN_SET_REPRODUCIBLE dfltcc_can_set_reproducible
+
 #endif

diff --git a/deflate.c b/deflate.c
index 41d1f72a9dabf171e65af41762641db5b1e3de21..05480f5f946d98cb5d0b5b4f40a8e4893a02a715 100644 (file)
--- a/deflate.c
+++ b/deflate.c
@@ -94,6 +94,8 @@ const char deflate_copyright[] = " deflate 1.2.11.f Copyright 1995-2016 Jean-lou
 #  define DEFLATE_HOOK(strm, flush, bstate) 0
 /* Returns whether zlib-ng should compute a checksum. Set to 0 if arch-specific deflation code already does that. */
 #  define DEFLATE_NEED_CHECKSUM(strm) 1
+/* Returns whether reproducibility parameter can be set to a given value. */
+#  define DEFLATE_CAN_SET_REPRODUCIBLE(strm, reproducible) 1
 #endif
 
 /* ===========================================================================
@@ -409,6 +411,7 @@ int ZEXPORT PREFIX(deflateInit2_)(PREFIX3(stream) *strm, int level, int method,
     s->strategy = strategy;
     s->method = (unsigned char)method;
     s->block_open = 0;
+    s->reproducible = 0;
 
     return PREFIX(deflateReset)(strm);
 }
@@ -1732,11 +1735,13 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
     deflate_state *s;
     zng_deflate_param_value *new_level = NULL;
     zng_deflate_param_value *new_strategy = NULL;
+    zng_deflate_param_value *new_reproducible = NULL;
     int param_buf_error;
     int version_error = 0;
     int buf_error = 0;
     int stream_error = 0;
     int ret;
+    int val;
 
     /* Initialize the statuses. */
     for (i = 0; i < count; i++)
@@ -1756,6 +1761,9 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
             case Z_DEFLATE_STRATEGY:
                 param_buf_error = deflateSetParamPre(&new_strategy, sizeof(int), &params[i]);
                 break;
+            case Z_DEFLATE_REPRODUCIBLE:
+                param_buf_error = deflateSetParamPre(&new_reproducible, sizeof(int), &params[i]);
+                break;
             default:
                 params[i].status = Z_VERSION_ERROR;
                 version_error = 1;
@@ -1783,6 +1791,15 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para
             stream_error = 1;
         }
     }
+    if (new_reproducible != NULL) {
+        val = *(int *)new_reproducible->buf;
+        if (DEFLATE_CAN_SET_REPRODUCIBLE(strm, val))
+            s->reproducible = val;
+        else {
+            new_reproducible->status = Z_STREAM_ERROR;
+            stream_error = 1;
+        }
+    }
 
     /* Report version errors only if there are no real errors. */
     return stream_error ? Z_STREAM_ERROR : (version_error ? Z_VERSION_ERROR : Z_OK);
@@ -1818,6 +1835,12 @@ int ZEXPORT zng_deflateGetParams(zng_stream *strm, zng_deflate_param_value *para
                 else
                     *(int *)params[i].buf = s->strategy;
                 break;
+            case Z_DEFLATE_REPRODUCIBLE:
+                if (params[i].size < sizeof(int))
+                    params[i].status = Z_BUF_ERROR;
+                else
+                    *(int *)params[i].buf = s->reproducible;
+                break;
             default:
                 params[i].status = Z_VERSION_ERROR;
                 version_error = 1;

diff --git a/deflate.h b/deflate.h
index bbe8f6b07ea937b1940330506abe1d0814c5cba2..c512a3f1abe1acee2ba174334bf7a8752f29ddb5 100644 (file)
--- a/deflate.h
+++ b/deflate.h
@@ -284,6 +284,9 @@ typedef struct internal_state {
      * This is set to 1 if there is an active block, or 0 if the block was just
      * closed.
      */
+    int reproducible;
+    /* Whether reproducible compression results are required.
+     */
 
 } deflate_state;
 

diff --git a/zlib-ng.h b/zlib-ng.h
index 5662b27ccb766e8abb690f377152b10dc8cb529a..a66b2af14ed3b299ef915888c569e9f9a7d681df 100644 (file)
--- a/zlib-ng.h
+++ b/zlib-ng.h
@@ -1793,6 +1793,13 @@ ZEXTERN int ZEXPORT zng_gzgetc_(gzFile file);  /* backward compatibility */
 typedef enum {
     Z_DEFLATE_LEVEL = 0,         /* compression level, represented as an int */
     Z_DEFLATE_STRATEGY = 1,      /* compression strategy, represented as an int */
+    Z_DEFLATE_REPRODUCIBLE = 2,
+    /*
+         Whether reproducible compression results are required. Represented as an int, where 0 means that it is allowed
+       to trade reproducibility for e.g. improved performance or compression ratio, and non-0 means that
+       reproducibility is strictly required. Reproducibility is guaranteed only when using an identical zlib-ng build.
+       Default is 0.
+    */
 } zng_deflate_param;
 
 typedef struct {