From: Ilya Leoshkevich Date: Fri, 24 May 2019 09:18:33 +0000 (+0200) Subject: Add "reproducible" deflate parameter X-Git-Tag: 1.9.9-b1~466 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d746d233c1dc70e0717877bccf8d9dc099d056d5;p=thirdparty%2Fzlib-ng.git Add "reproducible" deflate parameter IBM Z DEFLATE CONVERSION CALL may produce different (but valid) compressed data for the same uncompressed data. This behavior might be unacceptable for certain use cases (e.g. reproducible builds). This patch introduces Z_DEFLATE_REPRODUCIBLE parameter, which can be used to indicate that this is the case, and turn off IBM Z DEFLATE CONVERSION CALL. --- diff --git a/arch/s390/README.md b/arch/s390/README.md index 6995b106..841eb896 100644 --- a/arch/s390/README.md +++ b/arch/s390/README.md @@ -20,7 +20,10 @@ Two DFLTCC compression calls produce the same results only when they both are made on machines of the same generation, and when the respective buffers have the same offset relative to the start of the page. Therefore care should be taken when using hardware compression -when reproducible results are desired. +when reproducible results are desired. In particular, zlib-ng-specific +zng_deflateSetParams call allows setting Z_DEFLATE_REPRODUCIBLE +parameter, which would disable DFLTCC if reproducible results are +required. DFLTCC does not support every single zlib-ng feature, in particular: @@ -67,3 +70,11 @@ In addition to compression, DFLTCC computes CRC-32 and Adler-32 checksums, therefore, whenever it's used, software checksumming is suppressed using DEFLATE_NEED_CHECKSUM and INFLATE_NEED_CHECKSUM macros. + +While software always produces reproducible compression results, this +is not the case for DFLTCC. Therefore, zlib-ng users are given the +ability to specify whether or not reproducible compression results +are required. While it is always possible to specify this setting +before the compression begins, it is not always possible to do so in +the middle of a deflate stream - the exact conditions for that are +determined by DEFLATE_CAN_SET_REPRODUCIBLE macro. diff --git a/arch/s390/dfltcc_deflate.c b/arch/s390/dfltcc_deflate.c index adab3e00..e164d987 100644 --- a/arch/s390/dfltcc_deflate.c +++ b/arch/s390/dfltcc_deflate.c @@ -19,11 +19,13 @@ #include "dfltcc_deflate.h" #include "dfltcc_detail.h" -static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask) +static inline int dfltcc_are_params_ok(int level, uInt window_bits, int strategy, uint16_t level_mask, + int reproducible) { return (level_mask & ((uint16_t)1 << level)) != 0 && (window_bits == HB_BITS) && - (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY); + (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY) && + !reproducible; } @@ -33,7 +35,8 @@ int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm) struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); /* Unsupported compression settings */ - if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask)) + if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy, dfltcc_state->level_mask, + state->reproducible)) return 0; /* Unsupported hardware */ @@ -277,19 +280,27 @@ again: fly with deflateParams, we need to convert between hardware and software window formats. */ +static int dfltcc_was_deflate_used(PREFIX3(streamp) strm) +{ + deflate_state *state = (deflate_state *)strm->state; + struct dfltcc_param_v0 *param = &GET_DFLTCC_STATE(state)->param; + + return strm->total_in > 0 || param->nt == 0 || param->hl > 0; +} + int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy) { deflate_state *state = (deflate_state *)strm->state; struct dfltcc_state *dfltcc_state = GET_DFLTCC_STATE(state); - struct dfltcc_param_v0 *param = &dfltcc_state->param; int could_deflate = dfltcc_can_deflate(strm); - int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask); + int can_deflate = dfltcc_are_params_ok(level, state->w_bits, strategy, dfltcc_state->level_mask, + state->reproducible); if (can_deflate == could_deflate) /* We continue to work in the same mode - no changes needed */ return Z_OK; - if (strm->total_in == 0 && param->nt == 1 && param->hl == 0) + if (!dfltcc_was_deflate_used(strm)) /* DFLTCC was not used yet - no changes needed */ return Z_OK; @@ -297,6 +308,13 @@ int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int st return Z_STREAM_ERROR; } +int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible) +{ + deflate_state *state = (deflate_state *)strm->state; + + return reproducible != state->reproducible && !dfltcc_was_deflate_used(strm); +} + /* Preloading history. */ diff --git a/arch/s390/dfltcc_deflate.h b/arch/s390/dfltcc_deflate.h index 93945f0a..8775321b 100644 --- a/arch/s390/dfltcc_deflate.h +++ b/arch/s390/dfltcc_deflate.h @@ -6,6 +6,7 @@ int ZLIB_INTERNAL dfltcc_can_deflate(PREFIX3(streamp) strm); int ZLIB_INTERNAL dfltcc_deflate(PREFIX3(streamp) strm, int flush, block_state *result); int ZLIB_INTERNAL dfltcc_deflate_params(PREFIX3(streamp) strm, int level, int strategy); +int ZLIB_INTERNAL dfltcc_can_set_reproducible(PREFIX3(streamp) strm, int reproducible); int ZLIB_INTERNAL dfltcc_deflate_set_dictionary(PREFIX3(streamp) strm, const unsigned char *dictionary, uInt dict_length); int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned char *dictionary, uInt* dict_length); @@ -47,4 +48,6 @@ int ZLIB_INTERNAL dfltcc_deflate_get_dictionary(PREFIX3(streamp) strm, unsigned #define DEFLATE_NEED_CHECKSUM(strm) (!dfltcc_can_deflate((strm))) +#define DEFLATE_CAN_SET_REPRODUCIBLE dfltcc_can_set_reproducible + #endif diff --git a/deflate.c b/deflate.c index 41d1f72a..05480f5f 100644 --- a/deflate.c +++ b/deflate.c @@ -94,6 +94,8 @@ const char deflate_copyright[] = " deflate 1.2.11.f Copyright 1995-2016 Jean-lou # define DEFLATE_HOOK(strm, flush, bstate) 0 /* Returns whether zlib-ng should compute a checksum. Set to 0 if arch-specific deflation code already does that. */ # define DEFLATE_NEED_CHECKSUM(strm) 1 +/* Returns whether reproducibility parameter can be set to a given value. */ +# define DEFLATE_CAN_SET_REPRODUCIBLE(strm, reproducible) 1 #endif /* =========================================================================== @@ -409,6 +411,7 @@ int ZEXPORT PREFIX(deflateInit2_)(PREFIX3(stream) *strm, int level, int method, s->strategy = strategy; s->method = (unsigned char)method; s->block_open = 0; + s->reproducible = 0; return PREFIX(deflateReset)(strm); } @@ -1732,11 +1735,13 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para deflate_state *s; zng_deflate_param_value *new_level = NULL; zng_deflate_param_value *new_strategy = NULL; + zng_deflate_param_value *new_reproducible = NULL; int param_buf_error; int version_error = 0; int buf_error = 0; int stream_error = 0; int ret; + int val; /* Initialize the statuses. */ for (i = 0; i < count; i++) @@ -1756,6 +1761,9 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para case Z_DEFLATE_STRATEGY: param_buf_error = deflateSetParamPre(&new_strategy, sizeof(int), ¶ms[i]); break; + case Z_DEFLATE_REPRODUCIBLE: + param_buf_error = deflateSetParamPre(&new_reproducible, sizeof(int), ¶ms[i]); + break; default: params[i].status = Z_VERSION_ERROR; version_error = 1; @@ -1783,6 +1791,15 @@ int ZEXPORT zng_deflateSetParams(zng_stream *strm, zng_deflate_param_value *para stream_error = 1; } } + if (new_reproducible != NULL) { + val = *(int *)new_reproducible->buf; + if (DEFLATE_CAN_SET_REPRODUCIBLE(strm, val)) + s->reproducible = val; + else { + new_reproducible->status = Z_STREAM_ERROR; + stream_error = 1; + } + } /* Report version errors only if there are no real errors. */ return stream_error ? Z_STREAM_ERROR : (version_error ? Z_VERSION_ERROR : Z_OK); @@ -1818,6 +1835,12 @@ int ZEXPORT zng_deflateGetParams(zng_stream *strm, zng_deflate_param_value *para else *(int *)params[i].buf = s->strategy; break; + case Z_DEFLATE_REPRODUCIBLE: + if (params[i].size < sizeof(int)) + params[i].status = Z_BUF_ERROR; + else + *(int *)params[i].buf = s->reproducible; + break; default: params[i].status = Z_VERSION_ERROR; version_error = 1; diff --git a/deflate.h b/deflate.h index bbe8f6b0..c512a3f1 100644 --- a/deflate.h +++ b/deflate.h @@ -284,6 +284,9 @@ typedef struct internal_state { * This is set to 1 if there is an active block, or 0 if the block was just * closed. */ + int reproducible; + /* Whether reproducible compression results are required. + */ } deflate_state; diff --git a/zlib-ng.h b/zlib-ng.h index 5662b27c..a66b2af1 100644 --- a/zlib-ng.h +++ b/zlib-ng.h @@ -1793,6 +1793,13 @@ ZEXTERN int ZEXPORT zng_gzgetc_(gzFile file); /* backward compatibility */ typedef enum { Z_DEFLATE_LEVEL = 0, /* compression level, represented as an int */ Z_DEFLATE_STRATEGY = 1, /* compression strategy, represented as an int */ + Z_DEFLATE_REPRODUCIBLE = 2, + /* + Whether reproducible compression results are required. Represented as an int, where 0 means that it is allowed + to trade reproducibility for e.g. improved performance or compression ratio, and non-0 means that + reproducibility is strictly required. Reproducibility is guaranteed only when using an identical zlib-ng build. + Default is 0. + */ } zng_deflate_param; typedef struct {