From 19e6b8f4963855b5ab77564da2f488e83081bbcc Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Feb 2018 18:33:43 +0000 Subject: [PATCH] Add support for compression methods "zstd" and "zstd_global_dict". FossilOrigin-Name: a993a50bb8d5a3bf7cf79e09204814e172ba0bf9b3949e81912ef83f0d4bb44e --- ext/zonefile/zonefile.c | 338 ++++++++++++++++++++++++++++++------ ext/zonefile/zonefile1.test | 56 +++++- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 343 insertions(+), 67 deletions(-) diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index 4bfb3e5e6c..371971f9c9 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -35,14 +35,16 @@ typedef unsigned long u32; #endif #endif /* SQLITE_AMALGAMATION */ +#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) +#define ZONEFILE_DEFAULT_ENCRYPTION 0 +#define ZONEFILE_DEFAULT_COMPRESSION 0 +#define ZONEFILE_DEFAULT_DICTSIZE (64*1024) + #define ZONEFILE_MAGIC_NUMBER 0x464B3138 #define ZONEFILE_SZ_HEADER 32 #define ZONEFILE_SZ_KEYOFFSETS_ENTRY 20 -#define ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE (64*1024) -#define ZONEFILE_DEFAULT_ENCRYPTION 0 -#define ZONEFILE_DEFAULT_COMPRESSION 0 #define ZONEFILE_COMPRESSION_NONE 0 #define ZONEFILE_COMPRESSION_ZSTD 1 @@ -66,11 +68,13 @@ static u32 zonefileGet32(const u8 *aBuf){ + (((u32)aBuf[3]) << 0); } -#ifdef SQLITE_HAVE_ZLIB +#include +#include +#include +#ifdef SQLITE_HAVE_ZLIB #include - -static int zfZlibOpen(void **pp){ +static int zfZlibOpen(void **pp, u8 *aDict, int nDict){ *pp = 0; return SQLITE_OK; } @@ -98,56 +102,191 @@ static int zfZlibUncompressSize( } static int zfZlibUncompress( void *p, - u8 *aDest, int *pnDest, + u8 *aDest, int nDest, const u8 *aSrc, int nSrc ){ - uLongf destLen = (uLongf)(*pnDest); + uLongf destLen = (uLongf)nDest; int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4); - *pnDest = (int)destLen; return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; } #endif +#ifdef SQLITE_HAVE_ZSTD +#include +static int zfZstdOpen(void **pp, u8 *aDict, int nDict){ + *pp = 0; + return SQLITE_OK; +} +static void zfZstdClose(void *p){ +} +static int zfZstdCompressBound(void *p, int nSrc){ + return (int)ZSTD_compressBound((size_t)nSrc); +} +static int zfZstdCompress( + void *p, + u8 *aDest, int *pnDest, + const u8 *aSrc, int nSrc +){ + size_t szDest = (size_t)(*pnDest); + size_t rc = ZSTD_compress(aDest, szDest, aSrc, (size_t)nSrc, 1); + if( ZSTD_isError(rc) ) return SQLITE_ERROR; + *pnDest = (int)rc; + return SQLITE_OK; +} +static int zfZstdUncompressSize( + void *p, + const u8 *aSrc, int nSrc +){ + return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc); +} +static int zfZstdUncompress( + void *p, + u8 *aDest, int nDest, + const u8 *aSrc, int nSrc +){ + size_t rc = ZSTD_decompress(aDest, (size_t)nDest, aSrc, (size_t)nSrc); + if( rc!=(size_t)nDest ) return SQLITE_ERROR; + return SQLITE_OK; +} + +#include +typedef struct ZfZstddict ZfZstddict; +struct ZfZstddict { + ZSTD_CDict *pCDict; + ZSTD_CCtx *pCCtx; + ZSTD_DDict *pDDict; + ZSTD_DCtx *pDCtx; +}; + +static void zfZstddictClose(void *p){ + if( p ){ + ZfZstddict *pCmp = (ZfZstddict*)p; + if( pCmp->pCDict ) ZSTD_freeCDict(pCmp->pCDict); + if( pCmp->pCCtx ) ZSTD_freeCCtx(pCmp->pCCtx); + if( pCmp->pDCtx ) ZSTD_freeDCtx(pCmp->pDCtx); + sqlite3_free(pCmp); + } +} +static int zfZstddictOpen(void **pp, u8 *aDict, int nDict){ + int rc = SQLITE_OK; + ZfZstddict *pDict = (ZfZstddict*)sqlite3_malloc(sizeof(ZfZstddict)); + if( pDict==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pDict, 0, sizeof(ZfZstddict)); + if( aDict ){ + pDict->pDDict = ZSTD_createDDict(aDict, nDict); + pDict->pDCtx = ZSTD_createDCtx(); + if( pDict->pDDict==0 || pDict->pDCtx==0 ){ + zfZstddictClose((void*)pDict); + pDict = 0; + rc = SQLITE_ERROR; + } + } + } + *pp = (void*)pDict; + return rc; +} +static int zfZstddictTrain( + void *p, /* Compressor handle */ + u8 *aDict, int *pnDict, /* OUT: Dictionary buffer */ + u8 *aSamp, size_t *aSz, int nSamp /* IN: Training samples */ +){ + ZfZstddict *pCmp = (ZfZstddict*)p; + size_t sz = ZDICT_trainFromBuffer(aDict, (size_t)*pnDict, aSamp, aSz, nSamp); + if( ZDICT_isError(sz) ) return SQLITE_ERROR; + pCmp->pCDict = ZSTD_createCDict(aDict, sz, 1); + pCmp->pCCtx = ZSTD_createCCtx(); + if( pCmp->pCDict==0 || pCmp->pCCtx==0 ) return SQLITE_ERROR; + *pnDict = (int)sz; + return SQLITE_OK; +} +static int zfZstddictCompressBound(void *p, int nSrc){ + return (int)ZSTD_compressBound((size_t)nSrc); +} +static int zfZstddictCompress( + void *p, + u8 *aDest, int *pnDest, + const u8 *aSrc, int nSrc +){ + ZfZstddict *pCmp = (ZfZstddict*)p; + size_t szDest = (size_t)(*pnDest); + size_t rc; + assert( pCmp && pCmp->pCDict && pCmp->pCCtx ); + rc = ZSTD_compress_usingCDict( + pCmp->pCCtx, aDest, szDest, aSrc, (size_t)nSrc, pCmp->pCDict + ); + if( ZSTD_isError(rc) ) return SQLITE_ERROR; + *pnDest = (int)rc; + return SQLITE_OK; +} +static int zfZstddictUncompressSize( + void *p, + const u8 *aSrc, int nSrc +){ + return (int)ZSTD_getFrameContentSize(aSrc, (size_t)nSrc); +} +static int zfZstddictUncompress( + void *p, + u8 *aDest, int nDest, + const u8 *aSrc, int nSrc +){ + ZfZstddict *pCmp = (ZfZstddict*)p; + size_t rc = ZSTD_decompress_usingDDict( + pCmp->pDCtx, aDest, (size_t)nDest, aSrc, (size_t)nSrc, pCmp->pDDict + ); + if( rc!=(size_t)nDest ) return SQLITE_ERROR; + return SQLITE_OK; +} +#endif typedef struct ZonefileCompress ZonefileCompress; static struct ZonefileCompress { int eType; const char *zName; - int (*xOpen)(void**); + int (*xOpen)(void**, u8 *aDict, int nDict); void (*xClose)(void*); + int (*xTrain)(void*, u8 *aDict, int *pnDict, u8 *a, size_t *aSz, int n); int (*xCompressBound)(void*, int nSrc); int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc); int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc); - int (*xUncompress)(void*, u8 *aDest, int *pnDest,const u8 *aSrc,int nSrc); + int (*xUncompress)(void*, u8 *aDest, int nDest, const u8 *aSrc, int nSrc); } aZonefileCompress[] = { { ZONEFILE_COMPRESSION_NONE, "none", - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0 }, #ifdef SQLITE_HAVE_ZSTD { ZONEFILE_COMPRESSION_ZSTD, "zstd", - 0, 0, 0, 0, 0, 0 + zfZstdOpen, zfZstdClose, + 0, + zfZstdCompressBound, zfZstdCompress, + zfZstdUncompressSize, zfZstdUncompress }, { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict", - 0, 0, 0, 0, 0, 0 + zfZstddictOpen, zfZstddictClose, + zfZstddictTrain, + zfZstddictCompressBound, zfZstddictCompress, + zfZstddictUncompressSize, zfZstddictUncompress }, #endif /* SQLITE_HAVE_ZSTD */ #ifdef SQLITE_HAVE_ZLIB { ZONEFILE_COMPRESSION_ZLIB, "zlib", zfZlibOpen, zfZlibClose, + 0, zfZlibCompressBound, zfZlibCompress, zfZlibUncompressSize, zfZlibUncompress }, #endif /* SQLITE_HAVE_ZLIB */ #ifdef SQLITE_HAVE_BROTLI { ZONEFILE_COMPRESSION_BROTLI, "brotli", - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0 }, #endif /* SQLITE_HAVE_BROTLI */ #ifdef SQLITE_HAVE_LZ4 { ZONEFILE_COMPRESSION_LZ4, "lz4", - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0 }, { ZONEFILE_COMPRESSION_LZ4HC, "lz4hc", - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0 }, #endif /* SQLITE_HAVE_LZ4 */ }; @@ -195,9 +334,6 @@ static ZonefileCompress *zonefileCompressByValue(int eType){ ")" -#include -#include -#include /* ** A structure to store the parameters for a single zonefile_write() @@ -492,21 +628,22 @@ static void zonefileFileClose(FILE *pFd){ static int zonefileAppendCompressed( sqlite3_context *pCtx, /* Leave any error message here */ - ZonefileCompress *pCmp, + ZonefileCompress *pMethod, /* Compression method object */ + void *pCmp, /* Compression handle */ ZonefileBuffer *pTo, /* Append new data here */ ZonefileBuffer *pFrom /* Input buffer */ ){ int rc = SQLITE_OK; - if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){ + if( pMethod->eType==ZONEFILE_COMPRESSION_NONE ){ if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){ rc = SQLITE_ERROR; }else{ zonefileAppendBlob(pTo, pFrom->a, pFrom->n); } }else{ - int nReq = pCmp->xCompressBound(0, pFrom->n); + int nReq = pMethod->xCompressBound(pCmp, pFrom->n); if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR; - rc = pCmp->xCompress(0, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n); + rc = pMethod->xCompress(pCmp, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n); pTo->n += nReq; if( rc!=SQLITE_OK ){ return rc; @@ -534,11 +671,16 @@ static void zonefileWriteFunc( int rc; sqlite3_value *pPrev = 0; char *zErr = 0; + void *pCmp = 0; /* Data compressor handle */ + u32 iOff; - ZonefileBuffer sFrameIdx = {0, 0, 0}; - ZonefileBuffer sKeyIdx = {0, 0, 0}; + ZonefileBuffer sFrameIdx = {0, 0, 0}; /* Array of frame offsets */ + ZonefileBuffer sKeyIdx = {0, 0, 0}; /* Array of key locations */ ZonefileBuffer sData = {0, 0, 0}; /* All completed frames so far */ ZonefileBuffer sFrame = {0, 0, 0}; /* Current frame (uncompressed) */ + ZonefileBuffer sDict = {0, 0, 0}; /* Compression model (if any) */ + ZonefileBuffer sSample = {0,0,0}; /* Sample data for compressor */ + size_t *aSample = 0; /* Array of sample sizes */ u8 aHdr[ZONEFILE_SZ_HEADER]; /* Space to assemble zonefile header */ @@ -550,20 +692,84 @@ static void zonefileWriteFunc( } if( zonefileGetParams(pCtx, zJson, &sWrite) ) return; + /* Check that the index-data compressor is not one that uses an external + ** dictionary. This is not permitted as there is no sense in using a + ** dictionary to compress a single blob of data, and the index-data + ** compressor only ever compresses a single frame. Also, there is nowhere + ** in the file-format to store such a dictionary for the index-data. */ + if( sWrite.pCmpIdx->xTrain ){ + zonefileCtxError(pCtx, + "compressor \"%s\" may not be used to compress the zonefile index", + sWrite.pCmpIdx->zName + ); + goto zone_write_out; + } + + if( sWrite.pCmpData->xOpen ){ + rc = sWrite.pCmpData->xOpen(&pCmp, 0, 0); + if( rc!=SQLITE_OK ){ + zonefileCtxError(pCtx, "error in compressor construction"); + goto zone_write_out; + } + } + /* Prepare the SQL statement used to read data from the source table. This ** also serves to verify the suitability of the source table schema. */ pStmt = zonefileCtxPrepare(pCtx, "SELECT k, frame, v FROM %Q ORDER BY frame, idx, k", zTbl ); - if( pStmt==0 ) return; + if( pStmt==0 ) goto zone_write_out; /* Open a file-handle used to write out the zonefile */ pFd = zonefileFileOpen(zFile, 1, &zErr); if( pFd==0 ){ sqlite3_result_error(pCtx, zErr, -1); - sqlite3_finalize(pStmt); sqlite3_free(zErr); - return; + goto zone_write_out; + } + + /* If the data compressor uses a global dictionary, create the dictionary + ** now. */ + if( sWrite.pCmpData->xTrain ){ + int nSample = 0; + + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nByte = sqlite3_column_bytes(pStmt, 2); + const u8 *aByte = (const u8*)sqlite3_column_blob(pStmt, 2); + if( zonefileBufferGrow(pCtx, &sSample, nByte) ){ + goto zone_write_out; + } + if( (nSample & (nSample-1))==0 ){ + int nNew = nSample ? nSample*2 : 8; + size_t *aNew = (size_t*)sqlite3_realloc(aSample, sizeof(size_t) * nNew); + if( aNew==0 ){ + sqlite3_result_error_nomem(pCtx); + goto zone_write_out; + } + aSample = aNew; + } + aSample[nSample] = nByte; + zonefileAppendBlob(&sSample, aByte, nByte); + nSample++; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + zonefileTransferError(pCtx); + goto zone_write_out; + } + + if( zonefileBufferGrow(pCtx, &sDict, ZONEFILE_DEFAULT_DICTSIZE) ){ + goto zone_write_out; + } + sDict.n = sDict.nAlloc; + + rc = sWrite.pCmpData->xTrain( + pCmp, sDict.a, &sDict.n, sSample.a, aSample, nSample + ); + if( rc!=SQLITE_OK ){ + zonefileCtxError(pCtx, "error generating dictionary"); + goto zone_write_out; + } } while( SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -578,7 +784,7 @@ static void zonefileWriteFunc( ){ /* Add new entry to sFrame */ if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) - || zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) ){ goto zone_write_out; } @@ -607,10 +813,10 @@ static void zonefileWriteFunc( zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } - if( sFrame.n>0 ){ - if( zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) ){ - goto zone_write_out; - } + if( sFrame.n>0 + && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) + ){ + goto zone_write_out; } sqlite3_value_free(pPrev); pPrev = 0; @@ -620,7 +826,7 @@ static void zonefileWriteFunc( if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out; zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n); zonefileBufferFree(&sKeyIdx); - rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, &sKeyIdx, &sFrameIdx); + rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, 0, &sKeyIdx,&sFrameIdx); sFrameIdx.n = 0; if( rc ) goto zone_write_out; } @@ -630,8 +836,9 @@ static void zonefileWriteFunc( zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER); aHdr[4] = sWrite.pCmpIdx->eType; aHdr[5] = sWrite.pCmpData->eType; - zonefilePut32(&aHdr[6], 0); /* Compression dictionary byte offset */ - zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); + iOff = ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n; + zonefilePut32(&aHdr[6], sDict.n ? iOff : 0); + zonefilePut32(&aHdr[10], iOff + sDict.n); zonefilePut32(&aHdr[14], nFrame); zonefilePut32(&aHdr[18], nKey); aHdr[22] = sWrite.encryptionType; @@ -643,6 +850,7 @@ static void zonefileWriteFunc( rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n); + if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sDict.a, sDict.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n); if( rc ){ zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile); @@ -655,12 +863,16 @@ static void zonefileWriteFunc( pFd = 0; zone_write_out: + if( pCmp ) sWrite.pCmpData->xClose(pCmp); if( pFd ) fclose(pFd); sqlite3_finalize(pStmt); zonefileBufferFree(&sFrameIdx); zonefileBufferFree(&sKeyIdx); zonefileBufferFree(&sFrame); + zonefileBufferFree(&sDict); zonefileBufferFree(&sData); + zonefileBufferFree(&sSample); + sqlite3_free(aSample); } typedef struct ZonefileFilesTab ZonefileFilesTab; @@ -984,19 +1196,20 @@ static int zonefileReadHeader( } static int zonefileUncompress( - ZonefileCompress *pCmp, + ZonefileCompress *pMethod, + void *pCmp, u8 *aIn, int nIn, u8 **paOut, int *pnOut ){ int rc; - int nOut = pCmp->xUncompressSize(0, aIn, nIn); + int nOut = pMethod->xUncompressSize(pCmp, aIn, nIn); u8 *aOut = sqlite3_malloc(nOut); - assert( pCmp->eType!=ZONEFILE_COMPRESSION_NONE ); + assert( pMethod->eType!=ZONEFILE_COMPRESSION_NONE ); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ - rc = pCmp->xUncompress(0, aOut, &nOut, aIn, nIn); + rc = pMethod->xUncompress(pCmp, aOut, nOut, aIn, nIn); if( rc ){ sqlite3_free(aOut); aOut = 0; @@ -1052,7 +1265,7 @@ static int zonefileLoadIndex( if( rc==SQLITE_OK && pCmp ){ u8 *aUn = 0; int nUn = 0; - rc = zonefileUncompress(pCmp, aIdx, nIdx, &aUn, &nUn); + rc = zonefileUncompress(pCmp, 0, aIdx, nIdx, &aUn, &nUn); if( rc==SQLITE_ERROR ){ *pzErr = sqlite3_mprintf("failed to uncompress index"); } @@ -1524,7 +1737,8 @@ static void zonefileFree(void *p){ static int zonefileCtxUncompress( sqlite3_context *pCtx, - ZonefileCompress *pCmp, + ZonefileCompress *pMethod, + void *pCmp, u8 *aBuf, int nBuf, int iKeyOff, int nKey ){ @@ -1532,7 +1746,7 @@ static int zonefileCtxUncompress( u8 *aUn = 0; int nUn = 0; - rc = zonefileUncompress(pCmp, aBuf, nBuf, &aUn, &nUn); + rc = zonefileUncompress(pMethod, pCmp, aBuf, nBuf, &aUn, &nUn); if( rc==SQLITE_OK ){ sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT); }else if( rc==SQLITE_ERROR ){ @@ -1554,7 +1768,8 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ int iKeyOff = 0; /* Offset of record within frame */ int szKey = 0; /* Uncompressed size of record in bytes */ ZonefileHeader hdr; - ZonefileCompress *pCmp = 0; + ZonefileCompress *pCmpMethod = 0; + void *pCmp = 0; if( pTab->pIdToName==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, @@ -1592,9 +1807,27 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr); } - /* Find the compression method */ + /* Find the compression method and open the compressor instance */ if( rc==SQLITE_OK ){ - rc = zfFindCompress(hdr.compressionTypeContent, &pCmp, &zErr); + rc = zfFindCompress(hdr.compressionTypeContent, &pCmpMethod, &zErr); + } + if( pCmpMethod ){ + int nDict = 0; + u8 *aDict = 0; + assert( rc==SQLITE_OK ); + if( hdr.byteOffsetDictionary ){ + nDict = hdr.byteOffsetFrames - hdr.byteOffsetDictionary; + aDict = sqlite3_malloc(nDict); + if( aDict==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = zonefileFileRead(pFd, aDict, nDict, hdr.byteOffsetDictionary); + } + } + if( rc==SQLITE_OK ){ + rc = pCmpMethod->xOpen(&pCmp, aDict, nDict); + } + sqlite3_free(aDict); } /* Find the offset (iOff) and size (szFrame) of the frame that the @@ -1625,19 +1858,21 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ ** the required record is read. Otherwise, the entire frame is read ** into memory. */ if( rc==SQLITE_OK ){ - int sz = (pCmp ? (int)szFrame : szKey); - i64 ofst = iOff + (pCmp ? 0 : iKeyOff); + int sz = (pCmpMethod ? (int)szFrame : szKey); + i64 ofst = iOff + (pCmpMethod ? 0 : iKeyOff); u8 *aBuf = sqlite3_malloc(sz); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst); if( rc==SQLITE_OK ){ - if( pCmp==0 ){ + if( pCmpMethod==0 ){ sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree); aBuf = 0; }else{ - rc = zonefileCtxUncompress(pCtx, pCmp, aBuf, szFrame, iKeyOff, szKey); + rc = zonefileCtxUncompress( + pCtx, pCmpMethod, pCmp, aBuf, szFrame, iKeyOff, szKey + ); } }else{ zErr = sqlite3_mprintf( @@ -1656,6 +1891,7 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ sqlite3_free(zErr); } zonefileFileClose(pFd); + if( pCmpMethod ) pCmpMethod->xClose(pCmp); return rc; } diff --git a/ext/zonefile/zonefile1.test b/ext/zonefile/zonefile1.test index f47d595b7f..bdcb4540ff 100644 --- a/ext/zonefile/zonefile1.test +++ b/ext/zonefile/zonefile1.test @@ -65,16 +65,56 @@ do_execsql_test 1.6 { SELECT count(*) FROM z1_shadow_idx } 0 do_execsql_test 1.7 { DROP TABLE z1 } #------------------------------------------------------------------------- - -foreach {tn cmp cmpidx} { - 1 none none - 2 zlib none - 3 none zlib - 4 zlib zlib +# Figure out which compression algorithms, if any, are supported by +# this build. Populate the global list $COMPRESSION_METHODS with the +# result. +reset_db +load_static_extension db zonefile +do_execsql_test 2.0 { + CREATE TABLE bb( + k INTEGER PRIMARY KEY, + frame INTEGER DEFAULT -1, + idx INTEGER DEFAULT -1, + v BLOB + ); + INSERT INTO bb(k, v) VALUES(1, randomblob(100)); +} +set COMPRESSION_METHODS [list] +foreach cmp { + none zlib zstd zstd_global_dict lz4 lz4hc brotli } { + set res [catchsql { + WITH p(n,v) AS ( + VALUES('compressionTypeContent', $cmp) + ) + SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p; + }] + + if {[lindex $res 0]==0} { + lappend COMPRESSION_METHODS $cmp + } +} + +# Check that it is not possible to use zstd_global_dict to compress +# the zonefile index. +# +if {[lsearch $COMPRESSION_METHODS zstd_global_dict]>=0} { + do_catchsql_test 2.1 { + WITH p(n,v) AS ( + VALUES('compressionTypeIndexData', 'zstd_global_dict') + ) + SELECT zonefile_write('test.zonefile', 'bb', json_group_object(n,v)) FROM p; + } {1 {compressor "zstd_global_dict" may not be used to compress the zonefile index}} +} +# puts $COMPRESSION_METHODS + +foreach cmp $COMPRESSION_METHODS { foreach cmpidx $COMPRESSION_METHODS { + if {$cmpidx == "zstd_global_dict"} continue reset_db load_static_extension db zonefile - + + set tn "$cmp/$cmpidx" + do_execsql_test 2.$tn.0 { CREATE TABLE zz( k INTEGER PRIMARY KEY, @@ -156,7 +196,7 @@ foreach {tn cmp cmpidx} { test20.zonefile 2 2 test21.zonefile 7 4 } -} +}} finish_test diff --git a/manifest b/manifest index 541978cf90..544b0d61ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\szlib\scompression\sto\sthe\szonefile\smodule. -D 2018-02-15T20:37:58.786 +C Add\ssupport\sfor\scompression\smethods\s"zstd"\sand\s"zstd_global_dict". +D 2018-02-17T18:33:43.145 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -409,8 +409,8 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4 -F ext/zonefile/zonefile.c b870b52d83d22a1f75969be99246974f69f8a103b2608322ac9f759fe53776f8 -F ext/zonefile/zonefile1.test 7d5348391d3558546d27b4625a9210c22d18db0b03ce864e03da081143b9d625 +F ext/zonefile/zonefile.c 3954cdaeae3ce5018be5e75022cbd5d142245828a8396ae35db64c2b379f3205 +F ext/zonefile/zonefile1.test 0f84e56cd4f7b2c05443d1a2632c20ef635cc7f92de2868d92a2a6c09a9258ad F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fb1c2277912c55cfae30c18b5434bc193748746395fa7df983cd8a29e5741ff9 -R f62d689272bec55b3ef6a167acd15166 +P 72b8a7ef98d84460718378b9d17477599df39b4216015f8967674dd02b54b406 +R 3137d370c8b0ed559bfb7417aaf06af5 U dan -Z a99bb0a8f616d21dc3e95b3e50af7ee0 +Z cfd4dcc32b639975b3448fc063f18d5e diff --git a/manifest.uuid b/manifest.uuid index ddfa70ef2c..b20cc577b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72b8a7ef98d84460718378b9d17477599df39b4216015f8967674dd02b54b406 \ No newline at end of file +a993a50bb8d5a3bf7cf79e09204814e172ba0bf9b3949e81912ef83f0d4bb44e \ No newline at end of file -- 2.47.2