From: dan Date: Thu, 15 Feb 2018 20:37:58 +0000 (+0000) Subject: Add support for zlib compression to the zonefile module. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=21f433e82189a9f3064b4cb02094ac398b232c82;p=thirdparty%2Fsqlite.git Add support for zlib compression to the zonefile module. FossilOrigin-Name: 72b8a7ef98d84460718378b9d17477599df39b4216015f8967674dd02b54b406 --- diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index fecb7fe95e..4bfb3e5e6c 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -44,6 +44,138 @@ typedef unsigned long u32; #define ZONEFILE_DEFAULT_ENCRYPTION 0 #define ZONEFILE_DEFAULT_COMPRESSION 0 +#define ZONEFILE_COMPRESSION_NONE 0 +#define ZONEFILE_COMPRESSION_ZSTD 1 +#define ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT 2 +#define ZONEFILE_COMPRESSION_ZLIB 3 +#define ZONEFILE_COMPRESSION_BROTLI 4 +#define ZONEFILE_COMPRESSION_LZ4 5 +#define ZONEFILE_COMPRESSION_LZ4HC 6 + + +static void zonefilePut32(u8 *aBuf, u32 v){ + aBuf[0] = (v >> 24) & 0xFF; + aBuf[1] = (v >> 16) & 0xFF; + aBuf[2] = (v >> 8) & 0xFF; + aBuf[3] = v & 0xFF; +} +static u32 zonefileGet32(const u8 *aBuf){ + return (((u32)aBuf[0]) << 24) + + (((u32)aBuf[1]) << 16) + + (((u32)aBuf[2]) << 8) + + (((u32)aBuf[3]) << 0); +} + +#ifdef SQLITE_HAVE_ZLIB + +#include + +static int zfZlibOpen(void **pp){ + *pp = 0; + return SQLITE_OK; +} +static void zfZlibClose(void *p){ +} +static int zfZlibCompressBound(void *p, int nSrc){ + return (int)compressBound((uLong)nSrc) + 4; +} +static int zfZlibCompress( + void *p, + u8 *aDest, int *pnDest, + const u8 *aSrc, int nSrc +){ + uLongf destLen = (uLongf)(*pnDest)-4; + int rc = compress(&aDest[4], &destLen, aSrc, (uLong)nSrc); + *pnDest = (int)(destLen+4); + zonefilePut32(aDest, nSrc); + return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; +} +static int zfZlibUncompressSize( + void *p, + const u8 *aSrc, int nSrc +){ + return (int)zonefileGet32(aSrc); +} +static int zfZlibUncompress( + void *p, + u8 *aDest, int *pnDest, + const u8 *aSrc, int nSrc +){ + uLongf destLen = (uLongf)(*pnDest); + int rc = uncompress(aDest, &destLen, &aSrc[4], (uLong)nSrc-4); + *pnDest = (int)destLen; + return rc==Z_OK ? SQLITE_OK : SQLITE_ERROR; +} +#endif + +typedef struct ZonefileCompress ZonefileCompress; +static struct ZonefileCompress { + int eType; + const char *zName; + int (*xOpen)(void**); + void (*xClose)(void*); + int (*xCompressBound)(void*, int nSrc); + int (*xCompress)(void*, u8 *aDest, int *pnDest, const u8 *aSrc, int nSrc); + int (*xUncompressSize)(void*, const u8 *aSrc, int nSrc); + int (*xUncompress)(void*, u8 *aDest, int *pnDest,const u8 *aSrc,int nSrc); +} aZonefileCompress[] = { + { ZONEFILE_COMPRESSION_NONE, "none", + 0, 0, 0, 0, 0, 0 + }, +#ifdef SQLITE_HAVE_ZSTD + { ZONEFILE_COMPRESSION_ZSTD, "zstd", + 0, 0, 0, 0, 0, 0 + }, + { ZONEFILE_COMPRESSION_ZSTD_GLOBAL_DICT, "zstd_global_dict", + 0, 0, 0, 0, 0, 0 + }, +#endif /* SQLITE_HAVE_ZSTD */ +#ifdef SQLITE_HAVE_ZLIB + { ZONEFILE_COMPRESSION_ZLIB, "zlib", + zfZlibOpen, zfZlibClose, + zfZlibCompressBound, zfZlibCompress, + zfZlibUncompressSize, zfZlibUncompress + }, +#endif /* SQLITE_HAVE_ZLIB */ +#ifdef SQLITE_HAVE_BROTLI + { ZONEFILE_COMPRESSION_BROTLI, "brotli", + 0, 0, 0, 0, 0, 0 + }, +#endif /* SQLITE_HAVE_BROTLI */ +#ifdef SQLITE_HAVE_LZ4 + { ZONEFILE_COMPRESSION_LZ4, "lz4", + 0, 0, 0, 0, 0, 0 + }, + { ZONEFILE_COMPRESSION_LZ4HC, "lz4hc", + 0, 0, 0, 0, 0, 0 + }, +#endif /* SQLITE_HAVE_LZ4 */ +}; + +static ZonefileCompress *zonefileCompress(const char *zName){ + int i; + for(i=0; imaxAutoFrameSize = ZONEFILE_DEFAULT_MAXAUTOFRAMESIZE; + p->pCmpData = p->pCmpIdx = zonefileCompressByValue(0); rc = zonefilePrepare(db, &pStmt, &zErr,"SELECT key, value FROM json_each(?)"); if( rc==SQLITE_OK ){ @@ -238,10 +371,20 @@ static int zonefileGetParams( p->maxAutoFrameSize = iVal; }else if( sqlite3_stricmp("compressionTypeIndexData", zKey)==0 ){ - p->compressionTypeIndexData = iVal; + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + p->pCmpIdx = zonefileCompress(zName); + if( p->pCmpIdx==0 ){ + rc = SQLITE_ERROR; + zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName); + } }else if( sqlite3_stricmp("compressionTypeContent", zKey)==0 ){ - p->compressionTypeContent = iVal; + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + p->pCmpData = zonefileCompress(zName); + if( p->pCmpData==0 ){ + rc = SQLITE_ERROR; + zErr = sqlite3_mprintf("unknown compression scheme: \"%s\"", zName); + } }else if( sqlite3_stricmp("encryptionType", zKey)==0 ){ p->encryptionType = iVal; @@ -293,20 +436,6 @@ static void zonefileBufferFree(ZonefileBuffer *pBuf){ memset(pBuf, 0, sizeof(ZonefileBuffer)); } -static void zonefilePut32(u8 *aBuf, u32 v){ - aBuf[0] = (v >> 24) & 0xFF; - aBuf[1] = (v >> 16) & 0xFF; - aBuf[2] = (v >> 8) & 0xFF; - aBuf[3] = v & 0xFF; -} - -static u32 zonefileGet32(u8 *aBuf){ - return (((u32)aBuf[0]) << 24) - + (((u32)aBuf[1]) << 16) - + (((u32)aBuf[2]) << 8) - + (((u32)aBuf[3]) << 0); -} - static u64 zonefileGet64(u8 *aBuf){ return (((u64)aBuf[0]) << 56) + (((u64)aBuf[1]) << 48) @@ -361,6 +490,31 @@ static void zonefileFileClose(FILE *pFd){ if( pFd ) fclose(pFd); } +static int zonefileAppendCompressed( + sqlite3_context *pCtx, /* Leave any error message here */ + ZonefileCompress *pCmp, + ZonefileBuffer *pTo, /* Append new data here */ + ZonefileBuffer *pFrom /* Input buffer */ +){ + int rc = SQLITE_OK; + if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){ + if( zonefileBufferGrow(pCtx, pTo, pFrom->n) ){ + rc = SQLITE_ERROR; + }else{ + zonefileAppendBlob(pTo, pFrom->a, pFrom->n); + } + }else{ + int nReq = pCmp->xCompressBound(0, pFrom->n); + if( zonefileBufferGrow(pCtx, pTo, nReq) ) return SQLITE_ERROR; + rc = pCmp->xCompress(0, &pTo->a[pTo->n], &nReq, pFrom->a, pFrom->n); + pTo->n += nReq; + if( rc!=SQLITE_OK ){ + return rc; + } + } + return SQLITE_OK; +} + /* ** Function: zonefile_write(F,T[,J]) */ @@ -375,7 +529,6 @@ static void zonefileWriteFunc( ZonefileWrite sWrite; /* Decoded JSON parameters */ int nKey = 0; /* Number of keys in new zonefile */ int nFrame = 0; /* Number of frames in new zonefile */ - int szFrame = 0; /* Size of current frame */ sqlite3_stmt *pStmt = 0; /* SQL used to read data from source table */ FILE *pFd = 0; int rc; @@ -384,7 +537,9 @@ static void zonefileWriteFunc( ZonefileBuffer sFrameIdx = {0, 0, 0}; ZonefileBuffer sKeyIdx = {0, 0, 0}; - ZonefileBuffer sFrames = {0, 0, 0}; + ZonefileBuffer sData = {0, 0, 0}; /* All completed frames so far */ + ZonefileBuffer sFrame = {0, 0, 0}; /* Current frame (uncompressed) */ + u8 aHdr[ZONEFILE_SZ_HEADER]; /* Space to assemble zonefile header */ assert( objc==2 || objc==3 ); @@ -419,12 +574,16 @@ static void zonefileWriteFunc( int bAuto = zonefileIsAutoFrame(pFrame); if( zonefileCompareValue(pFrame, pPrev) - || (bAuto && szFrame && (szFrame+nBlob)>sWrite.maxAutoFrameSize) + || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) ){ - /* Add new entry to sFrameIdx */ - szFrame = 0; - if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) ) goto zone_write_out; - zonefileAppend32(&sFrameIdx, sFrames.n); + /* Add new entry to sFrame */ + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) + ){ + goto zone_write_out; + } + sFrame.n = 0; + zonefileAppend32(&sFrameIdx, sData.n); sqlite3_value_free(pPrev); pPrev = sqlite3_value_dup(pFrame); if( pPrev==0 ){ @@ -440,23 +599,37 @@ static void zonefileWriteFunc( } zonefileAppend64(&sKeyIdx, k); zonefileAppend32(&sKeyIdx, nFrame-1); - zonefileAppend32(&sKeyIdx, szFrame); + zonefileAppend32(&sKeyIdx, sFrame.n); zonefileAppend32(&sKeyIdx, nBlob); - /* Add data for new entry to sFrames */ - if( zonefileBufferGrow(pCtx, &sFrames, nBlob) ) goto zone_write_out; - zonefileAppendBlob(&sFrames, pBlob, nBlob); - szFrame += nBlob; + /* Add uncompressed data for new entry to sFrame */ + if( zonefileBufferGrow(pCtx, &sFrame, nBlob) ) goto zone_write_out; + zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } + if( sFrame.n>0 ){ + if( zonefileAppendCompressed(pCtx, sWrite.pCmpData, &sData, &sFrame) ){ + goto zone_write_out; + } + } sqlite3_value_free(pPrev); pPrev = 0; + /* If a compression method was specified, compress the key-index here */ + if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){ + if( zonefileBufferGrow(pCtx, &sFrameIdx, sKeyIdx.n) ) goto zone_write_out; + zonefileAppendBlob(&sFrameIdx, sKeyIdx.a, sKeyIdx.n); + zonefileBufferFree(&sKeyIdx); + rc = zonefileAppendCompressed(pCtx, sWrite.pCmpIdx, &sKeyIdx, &sFrameIdx); + sFrameIdx.n = 0; + if( rc ) goto zone_write_out; + } + /* Create the zonefile header in the in-memory buffer */ memset(aHdr, 0, ZONEFILE_SZ_HEADER); zonefilePut32(&aHdr[0], ZONEFILE_MAGIC_NUMBER); - aHdr[4] = sWrite.compressionTypeIndexData; - aHdr[5] = sWrite.compressionTypeContent; + aHdr[4] = sWrite.pCmpIdx->eType; + aHdr[5] = sWrite.pCmpData->eType; zonefilePut32(&aHdr[6], 0); /* Compression dictionary byte offset */ zonefilePut32(&aHdr[10], ZONEFILE_SZ_HEADER + sFrameIdx.n + sKeyIdx.n); zonefilePut32(&aHdr[14], nFrame); @@ -470,7 +643,7 @@ static void zonefileWriteFunc( rc = zonefileFileWrite(pFd, aHdr, ZONEFILE_SZ_HEADER); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrameIdx.a, sFrameIdx.n); if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sKeyIdx.a, sKeyIdx.n); - if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sFrames.a, sFrames.n); + if( rc==SQLITE_OK ) rc = zonefileFileWrite(pFd, sData.a, sData.n); if( rc ){ zonefileCtxError(pCtx, "error writing file \"%s\" (fwrite())", zFile); goto zone_write_out; @@ -486,7 +659,8 @@ static void zonefileWriteFunc( sqlite3_finalize(pStmt); zonefileBufferFree(&sFrameIdx); zonefileBufferFree(&sKeyIdx); - zonefileBufferFree(&sFrames); + zonefileBufferFree(&sFrame); + zonefileBufferFree(&sData); } typedef struct ZonefileFilesTab ZonefileFilesTab; @@ -809,6 +983,96 @@ static int zonefileReadHeader( return rc; } +static int zonefileUncompress( + ZonefileCompress *pCmp, + u8 *aIn, int nIn, + u8 **paOut, int *pnOut +){ + int rc; + int nOut = pCmp->xUncompressSize(0, aIn, nIn); + u8 *aOut = sqlite3_malloc(nOut); + + assert( pCmp->eType!=ZONEFILE_COMPRESSION_NONE ); + if( aOut==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = pCmp->xUncompress(0, aOut, &nOut, aIn, nIn); + if( rc ){ + sqlite3_free(aOut); + aOut = 0; + } + } + + *paOut = aOut; + *pnOut = nOut; + return rc; +} + +static int zfFindCompress(int eType, ZonefileCompress **pp, char **pzErr){ + int rc = SQLITE_OK; + ZonefileCompress *pCmp; + pCmp = zonefileCompressByValue(eType); + if( pCmp==0 ){ + *pzErr = sqlite3_mprintf("unsupported compression method: %d", eType); + rc = SQLITE_ERROR; + }else if( pCmp->eType==ZONEFILE_COMPRESSION_NONE ){ + pCmp = 0; + } + *pp = pCmp; + return rc; +} + +static int zonefileLoadIndex( + ZonefileHeader *pHdr, + FILE *pFd, + u8 **paIdx, int *pnIdx, + char **pzErr +){ + ZonefileCompress *pCmp = 0; + int rc; + u8 *aIdx = 0; + int nIdx = 0; + + rc = zfFindCompress(pHdr->compressionTypeIndexData, &pCmp, pzErr); + if( rc==SQLITE_OK ){ + if( pHdr->byteOffsetDictionary ){ + nIdx = pHdr->byteOffsetDictionary - ZONEFILE_SZ_HEADER; + }else{ + nIdx = pHdr->byteOffsetFrames - ZONEFILE_SZ_HEADER; + } + aIdx = (u8*)sqlite3_malloc(nIdx); + + if( aIdx==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = zonefileFileRead(pFd, aIdx, nIdx, ZONEFILE_SZ_HEADER); + } + } + + if( rc==SQLITE_OK && pCmp ){ + u8 *aUn = 0; + int nUn = 0; + rc = zonefileUncompress(pCmp, aIdx, nIdx, &aUn, &nUn); + if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("failed to uncompress index"); + } + sqlite3_free(aIdx); + aIdx = aUn; + nIdx = nUn; + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(aIdx); + aIdx = 0; + nIdx = 0; + } + + *paIdx = aIdx; + *pnIdx = nIdx; + return SQLITE_OK; +} + + static int zonefilePopulateIndex( ZonefileFilesTab *pTab, const char *zFile, @@ -825,21 +1089,12 @@ static int zonefilePopulateIndex( } if( rc==SQLITE_OK && hdr.numKeys>0 ){ - /* TODO: Deal with encrypted and compressed ZonefileIndex objects */ - i64 iOff; /* Offset of keyOffsets array */ u8 *aKey; /* Entire KeyOffsets array */ int nKey; /* Size of buffer aKey[] in bytes */ int i; - assert( hdr.encryptionType==0 && hdr.compressionTypeIndexData==0 ); - iOff = ZONEFILE_SZ_HEADER + (hdr.numFrames * 4); - nKey = ZONEFILE_SZ_KEYOFFSETS_ENTRY * hdr.numKeys; - aKey = (u8*)sqlite3_malloc(nKey); - if( aKey==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = zonefileFileRead(pFd, aKey, nKey, iOff); - } + assert( hdr.encryptionType==0 ); + rc = zonefileLoadIndex(&hdr, pFd, &aKey, &nKey, &pTab->base.zErrMsg); if( rc==SQLITE_OK && pTab->pInsertIdx==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pInsertIdx, &pTab->base.zErrMsg, @@ -850,7 +1105,7 @@ static int zonefilePopulateIndex( } for(i=0; ipInsertIdx, 1, (i64)zonefileGet64(&aEntry[0])); sqlite3_bind_int64(pTab->pInsertIdx, 2, iFileid); @@ -1267,6 +1522,27 @@ static void zonefileFree(void *p){ sqlite3_free(p); } +static int zonefileCtxUncompress( + sqlite3_context *pCtx, + ZonefileCompress *pCmp, + u8 *aBuf, int nBuf, + int iKeyOff, int nKey +){ + int rc; + u8 *aUn = 0; + int nUn = 0; + + rc = zonefileUncompress(pCmp, aBuf, nBuf, &aUn, &nUn); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(pCtx, &aUn[iKeyOff], nKey, SQLITE_TRANSIENT); + }else if( rc==SQLITE_ERROR ){ + zonefileCtxError(pCtx, "failed to uncompress frame"); + } + + sqlite3_free(aUn); + return rc; +} + static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ ZonefileTab *pTab = (ZonefileTab*)pCsr->base.pVtab; const char *zFile = 0; @@ -1274,7 +1550,11 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ FILE *pFd = 0; int rc = SQLITE_OK; u32 iOff = 0; /* Offset of frame in file */ + u32 szFrame = 0; /* Size of frame in bytes */ + int iKeyOff = 0; /* Offset of record within frame */ + int szKey = 0; /* Uncompressed size of record in bytes */ ZonefileHeader hdr; + ZonefileCompress *pCmp = 0; if( pTab->pIdToName==0 ){ rc = zonefilePrepare(pTab->db, &pTab->pIdToName, &pTab->base.zErrMsg, @@ -1287,6 +1567,9 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ } } + iKeyOff = sqlite3_column_int(pCsr->pSelect, 3); + szKey = sqlite3_column_int(pCsr->pSelect, 4); + /* Open the file to read the blob from */ sqlite3_bind_int64(pTab->pIdToName, 1, sqlite3_column_int64(pCsr->pSelect,1)); if( SQLITE_ROW==sqlite3_step(pTab->pIdToName) ){ @@ -1309,30 +1592,60 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ rc = zonefileReadHeader(pFd, zFile, &hdr, &zErr); } - /* Calculate the offset of the frame to read the blob from */ + /* Find the compression method */ + if( rc==SQLITE_OK ){ + rc = zfFindCompress(hdr.compressionTypeContent, &pCmp, &zErr); + } + + /* Find the offset (iOff) and size (szFrame) of the frame that the + ** record is stored in. */ if( rc==SQLITE_OK ){ - u8 aOff[4] = {0,0,0,0}; int iFrame = sqlite3_column_int(pCsr->pSelect, 2); - rc = zonefileFileRead(pFd, aOff, 4, ZONEFILE_SZ_HEADER + 4 * iFrame); + u8 aSpace[8] = {0,0,0,0,0,0,0,0}; + u8 *aOff = aSpace; + u8 *aFree = 0; + if( hdr.compressionTypeIndexData ){ + int nFree = 0; + rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr); + if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame]; + }else{ + rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame); + } iOff = zonefileGet32(aOff); + if( iFrame+1pSelect, 4); - int ofst = sqlite3_column_int(pCsr->pSelect, 3); + int sz = (pCmp ? (int)szFrame : szKey); + i64 ofst = iOff + (pCmp ? 0 : iKeyOff); u8 *aBuf = sqlite3_malloc(sz); if( aBuf==0 ){ rc = SQLITE_NOMEM; }else{ - rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + iOff + ofst); + rc = zonefileFileRead(pFd, aBuf, sz, hdr.byteOffsetFrames + ofst); if( rc==SQLITE_OK ){ - sqlite3_result_blob(pCtx, aBuf, sz, zonefileFree); + if( pCmp==0 ){ + sqlite3_result_blob(pCtx, aBuf, szKey, zonefileFree); + aBuf = 0; + }else{ + rc = zonefileCtxUncompress(pCtx, pCmp, aBuf, szFrame, iKeyOff, szKey); + } }else{ zErr = sqlite3_mprintf( - "failed to read %d bytes from file \"%s\"", sz, zFile + "failed to read %d bytes at offset %d from file \"%s\"", + sz, (int)(hdr.byteOffsetFrames+ofst), zFile ); } + sqlite3_free(aBuf); } } diff --git a/ext/zonefile/zonefile1.test b/ext/zonefile/zonefile1.test index 97fdab5858..f47d595b7f 100644 --- a/ext/zonefile/zonefile1.test +++ b/ext/zonefile/zonefile1.test @@ -66,85 +66,96 @@ do_execsql_test 1.7 { DROP TABLE z1 } #------------------------------------------------------------------------- -reset_db -load_static_extension db zonefile - -do_execsql_test 2.0 { - CREATE TABLE zz( - k INTEGER PRIMARY KEY, - frame INTEGER DEFAULT -1, - idx INTEGER DEFAULT -1, - v BLOB - ); - CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB); - CREATE VIRTUAL TABLE zone USING zonefile; -} - -set nMinByte 0 -set nMaxByte 444 -foreach {zonefile lKey} { - test1.zonefile {195 1238 298 405 297} - test2.zonefile {124 1624 82 1929} - test3.zonefile {932 683 1751 410 41} - test4.zonefile {427 1491} - test5.zonefile {1004 473 801 394 1672 816 1577} - test6.zonefile {1374 1454 1005} - test7.zonefile {450 241 319 133} - test8.zonefile {1414 900 1406 1917 127 673} - test9.zonefile {1192 226 988 1292 718 1345 1675} - test10.zonefile {314} - test11.zonefile {1177 1597 60 532 291 1164 812} - test12.zonefile {1168 1290 1585 939 1916} - test13.zonefile {644 1784 1476 1283 433 506} - test14.zonefile {1141 1547 1506 364} - test15.zonefile {1756 1885 844 1880 1896 354} - test16.zonefile {1383 1928 1371} - test17.zonefile {93} - test18.zonefile {1067} - test19.zonefile {642} - test20.zonefile {1380 1857} - test21.zonefile {288 293 1968 1207 1739 231 300} - test22.zonefile {651 1007 607 830 299 1431} - test23.zonefile {81 1651 543 1949 256 119 1088} - test24.zonefile {1278 2024 682 1115 194 636 1804} - test25.zonefile {514 1155 171 2015 791} - test26.zonefile {1615 1228 147 1464} - test27.zonefile {55 1130 781 678 78} - test28.zonefile {1981 1401 1178} - test29.zonefile {1754 864 183 1953 1901} - test30.zonefile {1461 817} - test31.zonefile {1720 1722 686 1833} +foreach {tn cmp cmpidx} { + 1 none none + 2 zlib none + 3 none zlib + 4 zlib zlib } { - forcedelete $zonefile - execsql { DELETE FROM zz; } - foreach k $lKey { - execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) } + reset_db + load_static_extension db zonefile + + do_execsql_test 2.$tn.0 { + CREATE TABLE zz( + k INTEGER PRIMARY KEY, + frame INTEGER DEFAULT -1, + idx INTEGER DEFAULT -1, + v BLOB + ); + CREATE TABLE rt(k INTEGER PRIMARY KEY, v BLOB); + CREATE VIRTUAL TABLE zone USING zonefile; } - execsql { INSERT INTO rt SELECT k, v FROM zz } - breakpoint - execsql { - SELECT zonefile_write($zonefile, 'zz', '{"maxAutoFrameSize":2000}'); - INSERT INTO zone_files(filename) VALUES($zonefile); + + set nMinByte 0 + set nMaxByte 444 + foreach {zonefile lKey} { + test1.zonefile {195 1238 298 405 297} + test2.zonefile {124 1624 82 1929} + test3.zonefile {932 683 1751 410 41} + test4.zonefile {427 1491} + test5.zonefile {1004 473 801 394 1672 816 1577} + test6.zonefile {1374 1454 1005} + test7.zonefile {450 241 319 133} + test8.zonefile {1414 900 1406 1917 127 673} + test9.zonefile {1192 226 988 1292 718 1345 1675} + test10.zonefile {314} + test11.zonefile {1177 1597 60 532 291 1164 812} + test12.zonefile {1168 1290 1585 939 1916} + test13.zonefile {644 1784 1476 1283 433 506} + test14.zonefile {1141 1547 1506 364} + test15.zonefile {1756 1885 844 1880 1896 354} + test16.zonefile {1383 1928 1371} + test17.zonefile {93} + test18.zonefile {1067} + test19.zonefile {642} + test20.zonefile {1380 1857} + test21.zonefile {288 293 1968 1207 1739 231 300} + test22.zonefile {651 1007 607 830 299 1431} + test23.zonefile {81 1651 543 1949 256 119 1088} + test24.zonefile {1278 2024 682 1115 194 636 1804} + test25.zonefile {514 1155 171 2015 791} + test26.zonefile {1615 1228 147 1464} + test27.zonefile {55 1130 781 678 78} + test28.zonefile {1981 1401 1178} + test29.zonefile {1754 864 183 1953 1901} + test30.zonefile {1461 817} + test31.zonefile {1720 1722 686 1833} + } { + forcedelete $zonefile + execsql { DELETE FROM zz; } + foreach k $lKey { + execsql { INSERT INTO zz(k, v) VALUES($k, randomblob($k)) } + } + execsql { INSERT INTO rt SELECT k, v FROM zz } + execsql { + WITH p(n,v) AS ( + VALUES('maxAutoFrameSize', 2000) UNION ALL + VALUES('compressionTypeIndexData', $cmpidx) UNION ALL + VALUES('compressionTypeContent', $cmp) + ) + SELECT zonefile_write($zonefile, 'zz', json_group_object(n, v)) FROM p; + INSERT INTO zone_files(filename) VALUES($zonefile); + } + } + + do_execsql_test 2.$tn.1 { + SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v + } + do_execsql_test 2.$tn.2 { + SELECT count(*) FROM zone JOIN rt USING (k); + } {135} + do_execsql_test 2.$tn.3 { + SELECT filename, + json_extract(header, '$.numKeys'), + json_extract(header, '$.numFrames') + FROM zone_files + WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile') + ORDER BY 1 + } { + test19.zonefile 1 1 + test20.zonefile 2 2 + test21.zonefile 7 4 } -} - -do_execsql_test 2.1 { - SELECT k FROM zone JOIN rt USING (k) WHERE zone.v!=rt.v -} -do_execsql_test 2.2 { - SELECT count(*) FROM zone JOIN rt USING (k); -} {135} -do_execsql_test 2.3 { - SELECT filename, - json_extract(header, '$.numKeys'), - json_extract(header, '$.numFrames') - FROM zone_files - WHERE filename IN ('test19.zonefile', 'test20.zonefile', 'test21.zonefile') - ORDER BY 1 -} { - test19.zonefile 1 1 - test20.zonefile 2 2 - test21.zonefile 7 4 } diff --git a/manifest b/manifest index d23be9ea5b..541978cf90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\spoint\sin\szonefile.c\sso\sthat\sall\sfiles\sare\sopened\sin\seither\s"rb"\sor\n"wb"\smode. -D 2018-02-15T15:24:12.092 +C Add\ssupport\sfor\szlib\scompression\sto\sthe\szonefile\smodule. +D 2018-02-15T20:37:58.786 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -409,8 +409,8 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4 -F ext/zonefile/zonefile.c 88d4ba281f47d35ce5989a7c500cfd05e3c8e348426baebcb7bcef314aee08f2 -F ext/zonefile/zonefile1.test 872ec8d549af0f1423601acdfe29390803fab3e14b8d5715f2f4bbd11431f1f5 +F ext/zonefile/zonefile.c b870b52d83d22a1f75969be99246974f69f8a103b2608322ac9f759fe53776f8 +F ext/zonefile/zonefile1.test 7d5348391d3558546d27b4625a9210c22d18db0b03ce864e03da081143b9d625 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4bb854ddd9c1dc2972fd4f7c2c2b2d121caa662d5085694c2dbb35d331a61444 -R 2c56f94b965e90e1ed8c01e5183c20c8 +P fb1c2277912c55cfae30c18b5434bc193748746395fa7df983cd8a29e5741ff9 +R f62d689272bec55b3ef6a167acd15166 U dan -Z 2cff9a10fd106f92fe182af0aacf7c32 +Z a99bb0a8f616d21dc3e95b3e50af7ee0 diff --git a/manifest.uuid b/manifest.uuid index da03a2339e..ddfa70ef2c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb1c2277912c55cfae30c18b5434bc193748746395fa7df983cd8a29e5741ff9 \ No newline at end of file +72b8a7ef98d84460718378b9d17477599df39b4216015f8967674dd02b54b406 \ No newline at end of file