fileio can use small blocks params

diff --git a/lib/zstdhc.c b/lib/zstdhc.c
index 866db183712e066cfdd7bad1cd57a36bf1a52085..fdf6dc95ddd78515b89c0c21245f1c2931e887e1 100644 (file)
--- a/lib/zstdhc.c
+++ b/lib/zstdhc.c
@@ -111,7 +111,7 @@ size_t ZSTD_HC_freeCCtx(ZSTD_HC_CCtx* cctx)
 /** ZSTD_HC_validateParams
     correct params value to remain within authorized range
     optimize for srcSize if srcSize > 0 */
-void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
+void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint)
 {
     const U32 btPlus = (params->strategy == ZSTD_HC_btlazy2);
 
@@ -120,9 +120,9 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
     if (params->windowLog   < ZSTD_HC_WINDOWLOG_MIN) params->windowLog = ZSTD_HC_WINDOWLOG_MIN;
 
     /* correct params, to use less memory */
-    if (srcSize > 0)
+    if ((srcSizeHint > 0) && (srcSizeHint < (1<<ZSTD_HC_WINDOWLOG_MAX)))
     {
-        U32 srcLog = ZSTD_highbit((U32)srcSize-1) + 1;
+        U32 srcLog = ZSTD_highbit((U32)srcSizeHint-1) + 1;
         if (params->windowLog > srcLog) params->windowLog = srcLog;
     }
 
@@ -139,9 +139,10 @@ void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize)
 
 
 static size_t ZSTD_HC_resetCCtx_advanced (ZSTD_HC_CCtx* zc,
-                                          ZSTD_HC_parameters params)
+                                          ZSTD_HC_parameters params,
+                                          U64 srcSizeHint)
 {
-    ZSTD_HC_validateParams(&params, 0);
+    ZSTD_HC_validateParams(&params, srcSizeHint);
 
     /* reserve table memory */
     {
@@ -930,7 +931,7 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
     if (ip != ctxPtr->end)
     {
         if (ctxPtr->end != NULL)
-            ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params);
+            ZSTD_HC_resetCCtx_advanced(ctxPtr, ctxPtr->params, srcSize);
         ctxPtr->base = ip;
     }
 
@@ -941,23 +942,24 @@ size_t ZSTD_HC_compressContinue (ZSTD_HC_CCtx* ctxPtr,
 
 size_t ZSTD_HC_compressBegin_advanced(ZSTD_HC_CCtx* ctx,
                                       void* dst, size_t maxDstSize,
-                                      const ZSTD_HC_parameters params)
+                                      const ZSTD_HC_parameters params,
+                                      U64 srcSizeHint)
 {
     size_t errorCode;
     if (maxDstSize < 4) return ERROR(dstSize_tooSmall);
-    errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params);
+    errorCode = ZSTD_HC_resetCCtx_advanced(ctx, params, srcSizeHint);
     if (ZSTD_isError(errorCode)) return errorCode;
     MEM_writeLE32(dst, ZSTD_magicNumber); /* Write Header */
     return 4;
 }
 
 
-size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel)
+size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint)
 {
-    int tableID = 1;
+    int tableID = ((srcSizeHint-1) > 128 KB);   /* intentional underflow for 0 */
     if (compressionLevel<=0) compressionLevel = 1;
     if (compressionLevel > ZSTD_HC_MAX_CLEVEL) compressionLevel = ZSTD_HC_MAX_CLEVEL;
-    return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel]);
+    return ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, ZSTD_HC_defaultParameters[tableID][compressionLevel], srcSizeHint);
 }
 
 
@@ -995,7 +997,7 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
     }
 
     /* Header */
-    oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params);
+    oSize = ZSTD_HC_compressBegin_advanced(ctx, dst, maxDstSize, params, srcSize);
     if(ZSTD_isError(oSize)) return oSize;
     op += oSize;
     maxDstSize -= oSize;

diff --git a/lib/zstdhc_static.h b/lib/zstdhc_static.h
index 534f2a01188f0e18210657a33c7914011f12a4db..f1973fc01f9e686158f1baabc3913e724ccacc80 100644 (file)
--- a/lib/zstdhc_static.h
+++ b/lib/zstdhc_static.h
@@ -83,14 +83,14 @@ size_t ZSTD_HC_compress_advanced (ZSTD_HC_CCtx* ctx,
 
 /** ZSTD_HC_validateParams
     correct params value to remain within authorized range
-    optimize for srcSize if srcSize > 0 */
-void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, size_t srcSize);
+    srcSizeHint value is optional, select 0 if not known */
+void ZSTD_HC_validateParams(ZSTD_HC_parameters* params, U64 srcSizeHint);
 
 
 /* *************************************
 *  Streaming functions
 ***************************************/
-size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel);
+size_t ZSTD_HC_compressBegin(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, int compressionLevel, U64 srcSizeHint);
 size_t ZSTD_HC_compressContinue(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
 size_t ZSTD_HC_compressEnd(ZSTD_HC_CCtx* ctx, void* dst, size_t maxDstSize);
 

diff --git a/programs/bench.c b/programs/bench.c
index 0009e4ae549ba8b18f9991cbf8620e6d9add1c80..5d10f9fb2b41a05aa9d3399b9adf26c9201aee08 100644 (file)
--- a/programs/bench.c
+++ b/programs/bench.c
@@ -380,7 +380,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
 }
 
 
-static U64 BMK_GetFileSize(char* infilename)
+static U64 BMK_GetFileSize(const char* infilename)
 {
     int r;
 #if defined(_MSC_VER)

diff --git a/programs/fileio.c b/programs/fileio.c
index d168f8a4bb52c4d97832efab1688d5bed44e1218..a7a394e0c54f0dd121b9a0807b8dda5cde4c41f8 100644 (file)
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -36,7 +36,7 @@
 /**LEGACY_SUPPORT :
 *  decompressor can decode older formats (starting from Zstd 0.1+) */
 #  define ZSTD_LEGACY_SUPPORT 1
-#endif // ZSTD_LEGACY_SUPPORT
+#endif
 
 
 /* *************************************
@@ -58,11 +58,13 @@
 /* *************************************
 *  Includes
 ***************************************/
-#include <stdio.h>     /* fprintf, fopen, fread, _fileno, stdin, stdout */
-#include <stdlib.h>    /* malloc, free */
-#include <string.h>    /* strcmp, strlen */
-#include <time.h>      /* clock */
-#include <errno.h>     /* errno */
+#include <stdio.h>      /* fprintf, fopen, fread, _fileno, stdin, stdout */
+#include <stdlib.h>     /* malloc, free */
+#include <string.h>     /* strcmp, strlen */
+#include <time.h>       /* clock */
+#include <errno.h>      /* errno */
+#include <sys/types.h>  /* stat64 */
+#include <sys/stat.h>   /* stat64 */
 #include "mem.h"
 #include "fileio.h"
 #include "zstd_static.h"
@@ -215,19 +217,35 @@ static void FIO_getFileHandles(FILE** pfinput, FILE** pfoutput, const char* inpu
     if ( *pfoutput==0) EXM_THROW(13, "Pb opening dst : %s", output_filename);
 }
 
+
+static U64 FIO_getFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+#endif
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;
+    return (U64)statbuf.st_size;
+}
+
+
 typedef void* (*FIO_createC) (void);
 static void* local_ZSTD_createCCtx(void) { return (void*) ZSTD_createCCtx(); }
 static void* local_ZSTD_HC_createCCtx(void) { return (void*) ZSTD_HC_createCCtx(); }
 
-typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel);
-static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
+typedef size_t (*FIO_initC) (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint);
+static size_t local_ZSTD_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
 {
-    (void)cLevel;
+    (void)cLevel; (void)srcSizeHint;
     return ZSTD_compressBegin((ZSTD_CCtx*)ctx, dst, maxDstSize);
 }
-static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel)
+static size_t local_ZSTD_HC_compressBegin (void* ctx, void* dst, size_t maxDstSize, int cLevel, U64 srcSizeHint)
 {
-    return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel);
+    return ZSTD_HC_compressBegin((ZSTD_HC_CCtx*)ctx, dst, maxDstSize, cLevel, srcSizeHint);
 }
 
 typedef size_t (*FIO_continueC) (void* ctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize);
@@ -294,6 +312,7 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
         freeC = local_ZSTD_HC_freeCCtx;
     }
     FIO_getFileHandles(&finput, &foutput, input_filename, output_filename);
+    filesize = FIO_getFileSize(input_filename);
 
     /* Allocate Memory */
     ctx = createC();
@@ -304,12 +323,13 @@ unsigned long long FIO_compressFilename(const char* output_filename, const char*
     inEnd = inBuff + inBuffSize;
 
     /* Write Frame Header */
-    cSize = initC(ctx, outBuff, outBuffSize, cLevel);
+    cSize = initC(ctx, outBuff, outBuffSize, cLevel, filesize);
     if (ZSTD_isError(cSize)) EXM_THROW(22, "Compression error : cannot create frame header");
 
     sizeCheck = fwrite(outBuff, 1, cSize, foutput);
     if (sizeCheck!=cSize) EXM_THROW(23, "Write error : cannot write header into %s", output_filename);
     compressedfilesize += cSize;
+    filesize = 0;
 
     /* Main compression loop */
     while (1)