From: dan Date: Mon, 19 Feb 2018 14:27:24 +0000 (+0000) Subject: Modify the zonefile format in order to avoid depending on the filesize to X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=c65faba468f7f83c76a19f66ced124acc59a07f9;p=thirdparty%2Fsqlite.git Modify the zonefile format in order to avoid depending on the filesize to determine the extent of the final frame. See README.md for details. FossilOrigin-Name: 4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447 --- diff --git a/ext/zonefile/README.md b/ext/zonefile/README.md index 38408a478f..32a556133d 100644 --- a/ext/zonefile/README.md +++ b/ext/zonefile/README.md @@ -111,7 +111,17 @@ key from one of the zonefile files in the index: necessary as we may not know the offset of the start of the frame data until after the ZoneFileIndex structure is compressed. - * Currently there is no support at all for encryption or compression. + * The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the + offsets for the first byte past the end of the corresponding frame. + For example, byteOffsetZoneFrame[] identifies the first byte of the + second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past + the end of the last frame in the file. + + This is better as if we store the starting offset of each frame, there + is no way to determine the size of the last frame in the file without + trusting the filesize itself. + + * Currently there is no support at all for encryption. * Zonefile currently uses json1 to parse the json argument to zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1 diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c index 69e66fde0f..835a1ff09c 100644 --- a/ext/zonefile/zonefile.c +++ b/ext/zonefile/zonefile.c @@ -788,7 +788,7 @@ static void zonefileWriteFunc( ); if( pStmt==0 ) goto zone_write_out; - /* Open a file-handle used to write out the zonefile */ + /* Open the file-handle used to write out the zonefile */ pFd = zonefileFileOpen(zFile, 1, &zErr); if( pFd==0 ){ sqlite3_result_error(pCtx, zErr, -1); @@ -797,7 +797,7 @@ static void zonefileWriteFunc( } /* If the data compressor uses a global dictionary, create the dictionary - ** now. */ + ** and store it in buffer sDict. */ if( sWrite.pCmpData->xTrain ){ int nSample = 0; @@ -847,24 +847,30 @@ static void zonefileWriteFunc( const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2); int bAuto = zonefileIsAutoFrame(pFrame); - if( zonefileCompareValue(pFrame, pPrev) - || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) - ){ - /* Add new entry to sFrame */ - if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) - || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) + if( sFrame.n>0 ){ + if( zonefileCompareValue(pFrame, pPrev) + || (bAuto && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize) ){ - goto zone_write_out; + /* Add new entry to sFrame */ + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp,&sData,&sFrame) + ){ + goto zone_write_out; + } + sFrame.n = 0; + zonefileAppend32(&sFrameIdx, sData.n); + sqlite3_value_free(pPrev); + pPrev = 0; + nFrame++; } - sFrame.n = 0; - zonefileAppend32(&sFrameIdx, sData.n); - sqlite3_value_free(pPrev); + } + + if( pPrev==0 ){ pPrev = sqlite3_value_dup(pFrame); if( pPrev==0 ){ sqlite3_result_error_nomem(pCtx); goto zone_write_out; } - nFrame++; } /* Add new entry to sKeyIdx */ @@ -872,7 +878,7 @@ static void zonefileWriteFunc( goto zone_write_out; } zonefileAppend64(&sKeyIdx, k); - zonefileAppend32(&sKeyIdx, nFrame-1); + zonefileAppend32(&sKeyIdx, nFrame); zonefileAppend32(&sKeyIdx, sFrame.n); zonefileAppend32(&sKeyIdx, nBlob); @@ -881,13 +887,16 @@ static void zonefileWriteFunc( zonefileAppendBlob(&sFrame, pBlob, nBlob); nKey++; } - if( sFrame.n>0 - && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) - ){ - goto zone_write_out; + + if( sFrame.n>0 ){ + if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) + || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) + ){ + goto zone_write_out; + } + zonefileAppend32(&sFrameIdx, sData.n); + nFrame++; } - sqlite3_value_free(pPrev); - pPrev = 0; /* If a compression method was specified, compress the key-index here */ if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){ @@ -933,6 +942,7 @@ static void zonefileWriteFunc( zone_write_out: if( pCmp ) sWrite.pCmpData->xClose(pCmp); if( pFd ) fclose(pFd); + sqlite3_value_free(pPrev); sqlite3_finalize(pStmt); zonefileBufferFree(&sFrameIdx); zonefileBufferFree(&sKeyIdx); @@ -1908,16 +1918,14 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){ if( hdr.compressionTypeIndexData ){ int nFree = 0; rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr); - if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame]; + if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)]; }else{ - rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame); + rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * (iFrame-1)); } - iOff = zonefileGet32(aOff); - if( iFrame+10 ){ + iOff = zonefileGet32(aOff); + szFrame = szFrame - iOff; } sqlite3_free(aFree); } diff --git a/manifest b/manifest index d5bc229376..62ca0de21b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\s"brotli"\scompression\sto\sthe\szonefile\smodule. -D 2018-02-17T20:22:23.498 +C Modify\sthe\szonefile\sformat\sin\sorder\sto\savoid\sdepending\son\sthe\sfilesize\sto\s\ndetermine\sthe\sextent\sof\sthe\sfinal\sframe.\sSee\sREADME.md\sfor\sdetails. +D 2018-02-19T14:27:24.815 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea @@ -408,8 +408,8 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f -F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4 -F ext/zonefile/zonefile.c 95233e917f4bdcc28fead61c4ad991b1ee18ea44fdba30a7e50e6484ea79fe31 +F ext/zonefile/README.md 1a95a93c865e196bc6301581e5a702f449ea9ce0e307cdbdbbdfd58377f1ec7e +F ext/zonefile/zonefile.c 755903797e593dde7876438002d694e3c8cbdf589f2ebda7f646345281e81d99 F ext/zonefile/zonefile1.test 0f84e56cd4f7b2c05443d1a2632c20ef635cc7f92de2868d92a2a6c09a9258ad F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 @@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bbe5b21ffab3cd312680ca9f179c5847790c17fb91d4174985153c6c398d48e3 -R 4ceb71e701bfbb226e3bec90adf489aa +P 3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac +R d6252e6001118befbde12b4bc598ec3c U dan -Z 3bc85025f17818043a52cc73d7feb53a +Z cbe2e5c769641b120eaa207e1d50335e diff --git a/manifest.uuid b/manifest.uuid index a0926326ba..f4e9fee9a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac \ No newline at end of file +4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447 \ No newline at end of file