From: inikep Date: Tue, 20 Sep 2016 14:22:28 +0000 (+0200) Subject: added ZWRAP_DCtx.decompState X-Git-Tag: v1.1.0~3^2~36 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=86fc8e000332fe27f3a60208442eaca00459db98;p=thirdparty%2Fzstd.git added ZWRAP_DCtx.decompState --- diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile index f966b4523..cfdafb6d8 100644 --- a/zlibWrapper/Makefile +++ b/zlibWrapper/Makefile @@ -37,7 +37,7 @@ testzstd: example_zstd testfitblk: fitblk ./fitblk 10240 <../zstd_compression_format.md - #./fitblk 40960 <../zstd_compression_format.md + ./fitblk 40960 <../zstd_compression_format.md .c.o: $(CC) $(CFLAGS) -c -o $@ $< diff --git a/zlibWrapper/examples/fitblk.c b/zlibWrapper/examples/fitblk.c index 1f12187bd..12e4c8626 100644 --- a/zlibWrapper/examples/fitblk.c +++ b/zlibWrapper/examples/fitblk.c @@ -57,6 +57,7 @@ //#include "zlib.h" #include "zstd_zlibwrapper.h" +#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/ #define local static /* print nastygram and leave */ @@ -79,14 +80,14 @@ local int partcompress(FILE *in, z_streamp def) flush = Z_SYNC_FLUSH; do { def->avail_in = fread(raw, 1, RAWLEN, in); - printf("partcompress def->avail_in=%d\n", def->avail_in); if (ferror(in)) return Z_ERRNO; def->next_in = raw; if (feof(in)) flush = Z_FINISH; + LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); ret = deflate(def, flush); - printf("partcompress def->avail_out=%d\n", def->avail_out); + LOG_FITBLK("partcompress2 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); assert(ret != Z_STREAM_ERROR); } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); return ret; @@ -105,10 +106,10 @@ local int recompress(z_streamp inf, z_streamp def) do { /* decompress */ inf->avail_out = RAWLEN; - printf("recompress inf->avail_out=%d\n", inf->avail_out); - inf->next_out = raw; + LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); ret = inflate(inf, Z_NO_FLUSH); + LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && ret != Z_NEED_DICT); if (ret == Z_MEM_ERROR) @@ -119,9 +120,10 @@ local int recompress(z_streamp inf, z_streamp def) def->next_in = raw; if (inf->avail_out != 0) flush = Z_FINISH; + LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); ret = deflate(def, flush); + LOG_FITBLK("recompress2deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); assert(ret != Z_STREAM_ERROR); - printf("recompress def->avail_out=%d ret=%d\n", def->avail_out, ret); } while (ret != Z_STREAM_END && def->avail_out != 0); return ret; } @@ -149,8 +151,7 @@ int main(int argc, char **argv) quit("need positive size of 8 or greater"); size = (unsigned)ret; - printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", - ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + printf("zlib version %s\n", ZLIB_VERSION); if (isUsingZSTD()) printf("zstd version %s\n", zstdVersion()); /* allocate memory for buffers and compression engine */ @@ -165,11 +166,12 @@ int main(int argc, char **argv) /* compress from stdin until output full, or no more input */ def.avail_out = size + EXCESS; def.next_out = blk; + LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); ret = partcompress(stdin, &def); + LOG_FITBLK("partcompress2 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); if (ret == Z_ERRNO) quit("error reading input"); -printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret); /* if it all fit, then size was undersubscribed -- done! */ if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { /* write block to stdout */ @@ -205,9 +207,9 @@ printf("partcompress def.total_out=%d ret=%d\n", (int)def.total_out, ret); inf.next_in = blk; def.avail_out = size + EXCESS; def.next_out = tmp; -printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); ret = recompress(&inf, &def); -printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); if (ret == Z_MEM_ERROR) quit("out of memory"); @@ -222,9 +224,9 @@ printf("recompress1 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail inf.next_in = tmp; def.avail_out = size; def.next_out = blk; -printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); ret = recompress(&inf, &def); -printf("recompress2 inf.avail_in=%d def.avail_out=%d\n", inf.avail_in, def.avail_out); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); if (ret == Z_MEM_ERROR) quit("out of memory"); assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ diff --git a/zlibWrapper/zstd_zlibwrapper.c b/zlibWrapper/zstd_zlibwrapper.c index 28f42526e..90ec590d0 100644 --- a/zlibWrapper/zstd_zlibwrapper.c +++ b/zlibWrapper/zstd_zlibwrapper.c @@ -23,7 +23,7 @@ #define ZWRAP_DEFAULT_CLEVEL 5 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ #define LOG_WRAPPERC(...) /*printf(__VA_ARGS__)*/ -#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ +#define LOG_WRAPPERD(...) /*printf(__VA_ARGS__)*/ #define FINISH_WITH_GZ_ERR(msg) { \ @@ -323,6 +323,9 @@ typedef struct { ZSTD_DStream* zbd; char headerBuf[16]; /* should be equal or bigger than ZSTD_frameHeaderSize_min */ int errorCount; + int decompState; + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; /* zlib params */ int stream_size; @@ -330,11 +333,16 @@ typedef struct { int windowBits; ZSTD_customMem customMem; z_stream allocFunc; /* copy of zalloc, zfree, opaque */ - ZSTD_inBuffer inBuffer; - ZSTD_outBuffer outBuffer; } ZWRAP_DCtx; +void ZWRAP_initDCtx(ZWRAP_DCtx* zwd) +{ + zwd->errorCount = zwd->decompState = 0; + zwd->outBuffer.pos = 0; + zwd->outBuffer.size = 0; +} + ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) { ZWRAP_DCtx* zwd; @@ -353,9 +361,8 @@ ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) memset(zwd, 0, sizeof(ZWRAP_DCtx)); memcpy(&zwd->customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); } - zwd->outBuffer.pos = 0; - zwd->outBuffer.size = 0; + ZWRAP_initDCtx(zwd); return zwd; } @@ -422,6 +429,7 @@ ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) if (zwd == NULL) return Z_STREAM_ERROR; { size_t const errorCode = ZSTD_resetDStream(zwd->zbd); if (ZSTD_isError(errorCode)) return ZWRAPD_finish_with_error(zwd, strm, 0); } + ZWRAP_initDCtx(zwd); } strm->total_in = 0; @@ -470,7 +478,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (!strm->reserved) { LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); return res; } @@ -479,6 +487,9 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) ZWRAP_DCtx* zwd = (ZWRAP_DCtx*) strm->state; if (strm->state == NULL) return Z_STREAM_ERROR; LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + + if (zwd->decompState == Z_STREAM_END) return Z_STREAM_END; + // if (((strm->avail_in < ZSTD_HEADERSIZE) || (strm->total_in > 0)) && (strm->total_in < ZLIB_HEADERSIZE)) if (strm->total_in < ZLIB_HEADERSIZE) { @@ -524,7 +535,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) if (flush == Z_INFLATE_SYNC) res = inflateSync(strm); else res = inflate(strm, flush); - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); return res; } } @@ -558,7 +569,7 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); + LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(errorCode)); goto error; } // LOG_WRAPPERD("1srcSize=%d inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)srcSize, (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); @@ -573,26 +584,30 @@ ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) zwd->outBuffer.size = strm->avail_out; zwd->outBuffer.pos = 0; errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); - // LOG_WRAPPERD("2 inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos); LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", (int)errorCode, (int)strm->avail_in, (int)strm->avail_out); if (ZSTD_isError(errorCode)) { - LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", ZSTD_getErrorName(errorCode)); zwd->errorCount++; + LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", ZSTD_getErrorName(errorCode), zwd->errorCount); if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; } + LOG_WRAPPERD("inflate inpos=%d inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", (int)inPos, (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size); strm->next_out += zwd->outBuffer.pos; strm->total_out += zwd->outBuffer.pos; strm->avail_out -= zwd->outBuffer.pos; strm->total_in += zwd->inBuffer.pos - inPos; strm->next_in += zwd->inBuffer.pos - inPos; strm->avail_in -= zwd->inBuffer.pos - inPos; - if (errorCode == 0) { LOG_WRAPPERD("inflate Z_STREAM_END1 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); return Z_STREAM_END; } + if (errorCode == 0) { + LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + zwd->decompState = Z_STREAM_END; + return Z_STREAM_END; + } goto finish; error: return ZWRAPD_finish_with_error(zwd, strm, 0); } finish: - LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); return Z_OK; }