The xTruncate method for the multiplexor now changes surplus overflow

files to zero-length by default.  Or if the "truncate" query parameter
is used, it actually deletes the surplus overflow files.  This allows
VACUUM to shrink the database again.

FossilOrigin-Name: 5aefef04d6cc3b734c3f05e036b3869a52248f4d

diff --git a/manifest b/manifest
index 0d3df0583b0f897e41541cecf942834e74b961ef..a12b10d33a131dae4e64c3171ac1da58dbb4f7fe 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Call\ssqlite3_log()\swith\san\sappropriate\smessage\sif\sunable\sto\sfind\sa\sunique\nmaster-journal\sfilename.
-D 2011-12-16T01:30:15.388
+C The\sxTruncate\smethod\sfor\sthe\smultiplexor\snow\schanges\ssurplus\soverflow\nfiles\sto\szero-length\sby\sdefault.\s\sOr\sif\sthe\s"truncate"\squery\sparameter\s\nis\sused,\sit\sactually\sdeletes\sthe\ssurplus\soverflow\sfiles.\s\sThis\sallows\nVACUUM\sto\sshrink\sthe\sdatabase\sagain.
+D 2011-12-16T04:57:32.390
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207
 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff
 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e
 F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5
-F src/test_multiplex.c c87d27283e2f578c17a8db7a8cac095f576f4323
+F src/test_multiplex.c 993f930c9c8e1aa0f4fe7b5a74c9a60fbf3447e4
 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d
 F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e
 F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec
@@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 2685c2b949061f18bf6a4940eac8c8148873abb6
-R 45a748a7c5e87682cd34056f04633da7
+P e9177f7d4ec19f0841d0613990d2fb5725700ba1
+R c09d355f93cdab856d4274295151320d
 U drh
-Z 8c7f37ab067ed94fde60690f325712e2
+Z 4efbc5088c9568c2b7090122e09b39c8

diff --git a/manifest.uuid b/manifest.uuid
index 57de583d493ffdc14dd8abe688c919f55a1eac89..4852f049679c529245e707506eb3163846cac399 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-e9177f7d4ec19f0841d0613990d2fb5725700ba1
\ No newline at end of file
+5aefef04d6cc3b734c3f05e036b3869a52248f4d
\ No newline at end of file

diff --git a/src/test_multiplex.c b/src/test_multiplex.c
index d844604ab5c6687ca15c76e5a3e8d92bd889add2..f01a7f585b8369b4b1c138db076edb55bd7b19f2 100644 (file)
--- a/src/test_multiplex.c
+++ b/src/test_multiplex.c
@@ -133,7 +133,8 @@ struct multiplexGroup {
   int nName;                       /* Length of base filename */
   int flags;                       /* Flags used for original opening */
   unsigned int szChunk;            /* Chunk size used for this group */
-  int bEnabled;                    /* TRUE to use Multiplex VFS for this file */
+  unsigned char bEnabled;          /* TRUE to use Multiplex VFS for this file */
+  unsigned char bTruncate;         /* TRUE to enable truncation of databases */
   multiplexGroup *pNext, *pPrev;   /* Doubly linked list of all group objects */
 };
 
@@ -276,12 +277,17 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){
 
 /* Translate an sqlite3_file* that is really a multiplexGroup* into
 ** the sqlite3_file* for the underlying original VFS.
+**
+** For chunk 0, the pGroup->flags determines whether or not a new file
+** is created if it does not already exist.  For chunks 1 and higher, the
+** file is created only if createFlag is 1.
 */
 static sqlite3_file *multiplexSubOpen(
-  multiplexGroup *pGroup,
-  int iChunk,
-  int *rc,
-  int *pOutFlags
+  multiplexGroup *pGroup,    /* The multiplexor group */
+  int iChunk,                /* Which chunk to open.  0==original file */
+  int *rc,                   /* Result code in and out */
+  int *pOutFlags,            /* Output flags */
+  int createFlag             /* True to create if iChunk>0 */
 ){
   sqlite3_file *pSubOpen = 0;
   sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;        /* Real VFS */
@@ -299,15 +305,28 @@ static sqlite3_file *multiplexSubOpen(
 
   *rc = multiplexSubFilename(pGroup, iChunk);
   if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){
+    int flags, rc2, bExists;
+    if( iChunk==0 ) createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0;
+    flags = pGroup->flags;
+    if( createFlag ){
+      flags |= SQLITE_OPEN_CREATE;
+    }else if( pGroup->aReal[iChunk].z==0 ){
+      return 0;
+    }else{
+      *rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[iChunk].z,
+                              SQLITE_ACCESS_EXISTS, &bExists);
+      if( *rc || !bExists ) return 0;
+      flags &= ~SQLITE_OPEN_CREATE;
+    }
     pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile );
     if( pSubOpen==0 ){
       *rc = SQLITE_NOMEM;
       return 0;
     }
     pGroup->aReal[iChunk].p = pSubOpen;
-    *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen,
-                          pGroup->flags, pOutFlags);
-    if( *rc!=SQLITE_OK ){
+    rc2 = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen,
+                          flags, pOutFlags);
+    if( rc2!=SQLITE_OK ){
       sqlite3_free(pSubOpen);
       pGroup->aReal[iChunk].p = 0;
       return 0;
@@ -316,6 +335,25 @@ static sqlite3_file *multiplexSubOpen(
   return pSubOpen;
 }
 
+/*
+** Return the size, in bytes, of chunk number iChunk.  If that chunk
+** does not exist, then return 0.  This function does not distingish between
+** non-existant files and zero-length files.
+*/
+static sqlite3_int64 multiplexSubSize(
+  multiplexGroup *pGroup,    /* The multiplexor group */
+  int iChunk,                /* Which chunk to open.  0==original file */
+  int *rc                    /* Result code in and out */
+){
+  sqlite3_file *pSub;
+  sqlite3_int64 sz = 0;
+
+  pSub = multiplexSubOpen(pGroup, iChunk, rc, NULL, 0);
+  if( pSub==0 ) return 0;
+  *rc = pSub->pMethods->xFileSize(pSub, &sz);
+  return sz;
+}    
+
 /*
 ** This is the implementation of the multiplex_control() SQL function.
 */
@@ -455,6 +493,7 @@ static int multiplexOpen(
     memset(pGroup, 0, sz);
     pMultiplexOpen->pGroup = pGroup;
     pGroup->bEnabled = -1;
+    pGroup->bTruncate = (flags & SQLITE_OPEN_MAIN_DB)==0;
     pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE;
 
     if( zName ){
@@ -475,6 +514,7 @@ static int multiplexOpen(
             pGroup->bEnabled = 0;
           }
         }
+        if( sqlite3_uri_parameter(zName, "truncate") ) pGroup->bTruncate = 1;
       }
       pGroup->zName = p;
       memcpy(pGroup->zName, zName, nName+1);
@@ -493,8 +533,8 @@ static int multiplexOpen(
     pGroup->flags = flags;
     rc = multiplexSubFilename(pGroup, 1);
     if( rc==SQLITE_OK ){
-      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags);
-      assert( pSubOpen || rc!=SQLITE_OK );
+      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags, 0);
+      if( pSubOpen==0 && rc==SQLITE_OK ) rc = SQLITE_CANTOPEN;
     }
     if( rc==SQLITE_OK ){
       sqlite3_int64 sz;
@@ -522,17 +562,18 @@ static int multiplexOpen(
           }
         }else{
           /* If the first overflow file exists and if the size of the main file
-           ** is different from the chunk size, that means the chunk size is set
-           ** set incorrectly.  So fix it.
-           **
-           ** Or, if the first overflow file does not exist and the main file is
-           ** larger than the chunk size, that means the chunk size is too small.
-           ** But we have no way of determining the intended chunk size, so 
-           ** just disable the multiplexor all togethre.
-           */
+          ** is different from the chunk size, that means the chunk size is set
+          ** set incorrectly.  So fix it.
+          **
+          ** Or, if the first overflow file does not exist and the main file is
+          ** larger than the chunk size, that means the chunk size is too small.
+          ** But we have no way of determining the intended chunk size, so 
+          ** just disable the multiplexor all togethre.
+          */
           rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z,
               SQLITE_ACCESS_EXISTS, &bExists);
-          if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0
+          bExists = multiplexSubSize(pGroup, 1, &rc)>0;
+          if( rc==SQLITE_OK && bExists  && sz==(sz&0xffff0000) && sz>0
               && sz!=pGroup->szChunk ){
             pGroup->szChunk = sz;
           }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){
@@ -674,7 +715,7 @@ static int multiplexRead(
   int rc = SQLITE_OK;
   multiplexEnter();
   if( !pGroup->bEnabled ){
-    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
+    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
       rc = SQLITE_IOERR_READ;
     }else{
@@ -683,7 +724,7 @@ static int multiplexRead(
   }else{
     while( iAmt > 0 ){
       int i = (int)(iOfst / pGroup->szChunk);
-      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
+      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1);
       if( pSubOpen ){
         int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk;
         if( extra<0 ) extra = 0;
@@ -719,7 +760,7 @@ static int multiplexWrite(
   int rc = SQLITE_OK;
   multiplexEnter();
   if( !pGroup->bEnabled ){
-    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
+    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
       rc = SQLITE_IOERR_WRITE;
     }else{
@@ -728,7 +769,7 @@ static int multiplexWrite(
   }else{
     while( rc==SQLITE_OK && iAmt>0 ){
       int i = (int)(iOfst / pGroup->szChunk);
-      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
+      sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1);
       if( pSubOpen ){
         int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) -
                     pGroup->szChunk;
@@ -756,28 +797,35 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){
   int rc = SQLITE_OK;
   multiplexEnter();
   if( !pGroup->bEnabled ){
-    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
+    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
       rc = SQLITE_IOERR_TRUNCATE;
     }else{
       rc = pSubOpen->pMethods->xTruncate(pSubOpen, size);
     }
-  }else if( (pGroup->flags & SQLITE_OPEN_MAIN_DB)==0 ){
-    int rc2;
+  }else{
     int i;
+    int iBaseGroup = (int)(size / pGroup->szChunk);
     sqlite3_file *pSubOpen;
     sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;   /* Real VFS */
     /* delete the chunks above the truncate limit */
-    for(i=(int)(size / pGroup->szChunk)+1; i<pGroup->nReal; i++){
-      multiplexSubClose(pGroup, i, pOrigVfs);
+    for(i = pGroup->nReal-1; i>iBaseGroup && rc==SQLITE_OK; i--){
+      if( pGroup->bTruncate ){
+        multiplexSubClose(pGroup, i, pOrigVfs);
+      }else{
+        pSubOpen = multiplexSubOpen(pGroup, i, &rc, 0, 0);
+        if( pSubOpen ){
+          rc = pSubOpen->pMethods->xTruncate(pSubOpen, 0);
+        }
+      }
     }
-    pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0);
-    if( pSubOpen ){
-      rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk);
-      if( rc2!=SQLITE_OK ) rc = rc2;
-    }else{
-      rc = SQLITE_IOERR_TRUNCATE;
+    if( rc==SQLITE_OK ){
+      pSubOpen = multiplexSubOpen(pGroup, iBaseGroup, &rc, 0, 0);
+      if( pSubOpen ){
+        rc = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk);
+      }
     }
+    if( rc ) rc = SQLITE_IOERR_TRUNCATE;
   }
   multiplexLeave();
   return rc;
@@ -812,42 +860,18 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){
   int i;
   multiplexEnter();
   if( !pGroup->bEnabled ){
-    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
+    sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
     if( pSubOpen==0 ){
       rc = SQLITE_IOERR_FSTAT;
     }else{
       rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize);
     }
   }else{
-    sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs;
     *pSize = 0;
     for(i=0; rc==SQLITE_OK; i++){
-      sqlite3_file *pSubOpen = 0;
-      int exists = 0;
-      rc = multiplexSubFilename(pGroup, i);
-      if( rc!=SQLITE_OK ) break;
-      if( pGroup->nReal>i && pGroup->aReal[i].p!=0 ){
-        exists = 1;
-      }else if( (pGroup->flags & SQLITE_OPEN_DELETEONCLOSE)==0 ){
-        const char *zReal = pGroup->aReal[i].z;
-        rc = pOrigVfs->xAccess(pOrigVfs, zReal, SQLITE_ACCESS_EXISTS, &exists);
-      }
-      if( exists==0 ){
-        /* stop at first "gap" or IO error. */
-        break;
-      }
-      if( rc==SQLITE_OK ){
-        pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL);
-      }
-      assert( pSubOpen || rc!=SQLITE_OK );
-      if( rc==SQLITE_OK ){
-        sqlite3_int64 sz = 0;
-        rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz);
-        if( rc==SQLITE_OK && sz>pGroup->szChunk ){
-          rc = SQLITE_IOERR_FSTAT;
-        }
-        *pSize = i*(sqlite3_int64)pGroup->szChunk + sz;
-      }
+      sqlite3_int64 sz = multiplexSubSize(pGroup, i, &rc);
+      if( sz==0 ) break;
+      *pSize = i*(sqlite3_int64)pGroup->szChunk + sz;
     }
   }
   multiplexLeave();
@@ -859,7 +883,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){
 static int multiplexLock(sqlite3_file *pConn, int lock){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xLock(pSubOpen, lock);
   }
@@ -871,7 +895,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){
 static int multiplexUnlock(sqlite3_file *pConn, int lock){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xUnlock(pSubOpen, lock);
   }
@@ -883,7 +907,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){
 static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut);
   }
@@ -931,7 +955,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
       rc = SQLITE_OK;
       break;
     default:
-      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL);
+      pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0);
       if( pSubOpen ){
         rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg);
         if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){
@@ -948,7 +972,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){
 static int multiplexSectorSize(sqlite3_file *pConn){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen && pSubOpen->pMethods->xSectorSize ){
     return pSubOpen->pMethods->xSectorSize(pSubOpen);
   }
@@ -960,7 +984,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){
 static int multiplexDeviceCharacteristics(sqlite3_file *pConn){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen);
   }
@@ -978,7 +1002,7 @@ static int multiplexShmMap(
 ){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp);
   }
@@ -995,7 +1019,7 @@ static int multiplexShmLock(
 ){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags);
   }
@@ -1007,7 +1031,7 @@ static int multiplexShmLock(
 static void multiplexShmBarrier(sqlite3_file *pConn){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     pSubOpen->pMethods->xShmBarrier(pSubOpen);
   }
@@ -1018,7 +1042,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){
 static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){
   multiplexConn *p = (multiplexConn*)pConn;
   int rc;
-  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL);
+  sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0);
   if( pSubOpen ){
     return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag);
   }