Modify the zonefile format in order to avoid depending on the filesize to

determine the extent of the final frame. See README.md for details.

FossilOrigin-Name: 4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447

diff --git a/ext/zonefile/README.md b/ext/zonefile/README.md
index 38408a478fda324d63f8815ee605c6910b38947f..32a556133db7b2773a5425cf2c58cb4a1984ece2 100644 (file)
--- a/ext/zonefile/README.md
+++ b/ext/zonefile/README.md
@@ -111,7 +111,17 @@ key from one of the zonefile files in the index:
      necessary as we may not know the offset of the start of the frame data
      until after the ZoneFileIndex structure is compressed.
 
-  *  Currently there is no support at all for encryption or compression.
+  *  The offsets in the ZoneFileIndex.byteOffsetZoneFrame[] array are the
+     offsets for the first byte past the end of the corresponding frame.
+     For example, byteOffsetZoneFrame[] identifies the first byte of the
+     second frame, and byteOffsetZoneFrame[numFrames-1] is one byte past
+     the end of the last frame in the file.
+
+     This is better as if we store the starting offset of each frame, there
+     is no way to determine the size of the last frame in the file without
+     trusting the filesize itself.
+
+  *  Currently there is no support at all for encryption.
 
   *  Zonefile currently uses json1 to parse the json argument to
      zonefile\_write(). And so must be used with an SQLITE\_ENABLE\_JSON1

diff --git a/ext/zonefile/zonefile.c b/ext/zonefile/zonefile.c
index 69e66fde0f969f372f5efdece479e9b91ae0909c..835a1ff09c6cea307745a464da8ce9692a190a85 100644 (file)
--- a/ext/zonefile/zonefile.c
+++ b/ext/zonefile/zonefile.c
@@ -788,7 +788,7 @@ static void zonefileWriteFunc(
   );
   if( pStmt==0 ) goto zone_write_out;
 
-  /* Open a file-handle used to write out the zonefile */ 
+  /* Open the file-handle used to write out the zonefile */ 
   pFd = zonefileFileOpen(zFile, 1, &zErr);
   if( pFd==0 ){
     sqlite3_result_error(pCtx, zErr, -1);
@@ -797,7 +797,7 @@ static void zonefileWriteFunc(
   }
 
   /* If the data compressor uses a global dictionary, create the dictionary
-  ** now.  */
+  ** and store it in buffer sDict.  */
   if( sWrite.pCmpData->xTrain ){
     int nSample = 0;
 
@@ -847,24 +847,30 @@ static void zonefileWriteFunc(
     const u8 *pBlob = (const u8*)sqlite3_column_blob(pStmt, 2);
 
     int bAuto = zonefileIsAutoFrame(pFrame);
-    if( zonefileCompareValue(pFrame, pPrev) 
-     || (bAuto && sFrame.n && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
-    ){
-      /* Add new entry to sFrame */
-      if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
-       || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
+    if( sFrame.n>0 ){
+      if( zonefileCompareValue(pFrame, pPrev) 
+       || (bAuto && (sFrame.n+nBlob)>sWrite.maxAutoFrameSize)
       ){
-        goto zone_write_out;
+        /* Add new entry to sFrame */
+        if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
+         || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp,&sData,&sFrame)
+        ){
+          goto zone_write_out;
+        }
+        sFrame.n = 0;
+        zonefileAppend32(&sFrameIdx, sData.n);
+        sqlite3_value_free(pPrev);
+        pPrev = 0;
+        nFrame++;
       }
-      sFrame.n = 0;
-      zonefileAppend32(&sFrameIdx, sData.n);
-      sqlite3_value_free(pPrev);
+    }
+
+    if( pPrev==0 ){
       pPrev = sqlite3_value_dup(pFrame);
       if( pPrev==0 ){
         sqlite3_result_error_nomem(pCtx);
         goto zone_write_out;
       }
-      nFrame++;
     }
 
     /* Add new entry to sKeyIdx */
@@ -872,7 +878,7 @@ static void zonefileWriteFunc(
       goto zone_write_out;
     }
     zonefileAppend64(&sKeyIdx, k);
-    zonefileAppend32(&sKeyIdx, nFrame-1);
+    zonefileAppend32(&sKeyIdx, nFrame);
     zonefileAppend32(&sKeyIdx, sFrame.n);
     zonefileAppend32(&sKeyIdx, nBlob);
 
@@ -881,13 +887,16 @@ static void zonefileWriteFunc(
     zonefileAppendBlob(&sFrame, pBlob, nBlob);
     nKey++;
   }
-  if( sFrame.n>0
-   && zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame) 
-  ){
-    goto zone_write_out;
+
+  if( sFrame.n>0 ){
+    if( zonefileBufferGrow(pCtx, &sFrameIdx, 4) 
+     || zonefileAppendCompressed(pCtx, sWrite.pCmpData, pCmp, &sData, &sFrame)
+    ){
+      goto zone_write_out;
+    }
+    zonefileAppend32(&sFrameIdx, sData.n);
+    nFrame++;
   }
-  sqlite3_value_free(pPrev);
-  pPrev = 0;
 
   /* If a compression method was specified, compress the key-index here */
   if( sWrite.pCmpIdx->eType!=ZONEFILE_COMPRESSION_NONE ){
@@ -933,6 +942,7 @@ static void zonefileWriteFunc(
  zone_write_out:
   if( pCmp ) sWrite.pCmpData->xClose(pCmp);
   if( pFd ) fclose(pFd);
+  sqlite3_value_free(pPrev);
   sqlite3_finalize(pStmt);
   zonefileBufferFree(&sFrameIdx);
   zonefileBufferFree(&sKeyIdx);
@@ -1908,16 +1918,14 @@ static int zonefileGetValue(sqlite3_context *pCtx, ZonefileCsr *pCsr){
     if( hdr.compressionTypeIndexData ){
       int nFree = 0;
       rc = zonefileLoadIndex(&hdr, pFd, &aFree, &nFree, &zErr);
-      if( rc==SQLITE_OK ) aOff = &aFree[4*iFrame];
+      if( rc==SQLITE_OK ) aOff = &aFree[4*(iFrame-1)];
     }else{
-      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * iFrame);
+      rc = zonefileFileRead(pFd, aOff, 8, ZONEFILE_SZ_HEADER + 4 * (iFrame-1));
     }
-    iOff = zonefileGet32(aOff);
-    if( iFrame+1<hdr.numFrames ){
-      szFrame = zonefileGet32(&aOff[4]) - iOff;
-    }else{
-      fseek(pFd, 0, SEEK_END);
-      szFrame = (u32)ftell(pFd) - iOff - hdr.byteOffsetFrames;
+    szFrame = zonefileGet32(&aOff[4]);
+    if( iFrame>0 ){
+      iOff = zonefileGet32(aOff);
+      szFrame = szFrame - iOff;
     }
     sqlite3_free(aFree);
   }

diff --git a/manifest b/manifest
index d5bc229376060aeaf295047f34b1d3a46993a837..62ca0de21b9a276b856ec01d2879755280b3fa1e 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\ssupport\sfor\s"brotli"\scompression\sto\sthe\szonefile\smodule.
-D 2018-02-17T20:22:23.498
+C Modify\sthe\szonefile\sformat\sin\sorder\sto\savoid\sdepending\son\sthe\sfilesize\sto\s\ndetermine\sthe\sextent\sof\sthe\sfinal\sframe.\sSee\sREADME.md\sfor\sdetails.
+D 2018-02-19T14:27:24.815
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 7a3f714b4fcf793108042b7b0a5c720b0b310ec84314d61ba7f3f49f27e550ea
@@ -408,8 +408,8 @@ F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f
-F ext/zonefile/README.md 387ad2b748e98eeea21fd4dbb609fefe313263fadb3fc6c01c512b4c95e55ae4
-F ext/zonefile/zonefile.c 95233e917f4bdcc28fead61c4ad991b1ee18ea44fdba30a7e50e6484ea79fe31
+F ext/zonefile/README.md 1a95a93c865e196bc6301581e5a702f449ea9ce0e307cdbdbbdfd58377f1ec7e
+F ext/zonefile/zonefile.c 755903797e593dde7876438002d694e3c8cbdf589f2ebda7f646345281e81d99
 F ext/zonefile/zonefile1.test 0f84e56cd4f7b2c05443d1a2632c20ef635cc7f92de2868d92a2a6c09a9258ad
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
@@ -1708,7 +1708,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bbe5b21ffab3cd312680ca9f179c5847790c17fb91d4174985153c6c398d48e3
-R 4ceb71e701bfbb226e3bec90adf489aa
+P 3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac
+R d6252e6001118befbde12b4bc598ec3c
 U dan
-Z 3bc85025f17818043a52cc73d7feb53a
+Z cbe2e5c769641b120eaa207e1d50335e

diff --git a/manifest.uuid b/manifest.uuid
index a0926326bab8e0d546cf07a899eb780202bc30c9..f4e9fee9a429803eb5bf5718d3bc92ca27d5eec6 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3eb25b3fa5733b4418e7e2633be34b763e2c70342bb9c418a07c9f7d4b196fac
\ No newline at end of file
+4dbe0cba3fad9a752834d795127cf35eed21fab63b18a48f75d5c1e96ca77447
\ No newline at end of file