From: drh Date: Fri, 16 Dec 2011 04:57:32 +0000 (+0000) Subject: The xTruncate method for the multiplexor now changes surplus overflow X-Git-Tag: version-3.7.10~19^2~55^2~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e7d9f13d99667f89606a617eb9ffc01eb0fc8649;p=thirdparty%2Fsqlite.git The xTruncate method for the multiplexor now changes surplus overflow files to zero-length by default. Or if the "truncate" query parameter is used, it actually deletes the surplus overflow files. This allows VACUUM to shrink the database again. FossilOrigin-Name: 5aefef04d6cc3b734c3f05e036b3869a52248f4d --- diff --git a/manifest b/manifest index 0d3df0583b..a12b10d33a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\ssqlite3_log()\swith\san\sappropriate\smessage\sif\sunable\sto\sfind\sa\sunique\nmaster-journal\sfilename. -D 2011-12-16T01:30:15.388 +C The\sxTruncate\smethod\sfor\sthe\smultiplexor\snow\schanges\ssurplus\soverflow\nfiles\sto\szero-length\sby\sdefault.\s\sOr\sif\sthe\s"truncate"\squery\sparameter\s\nis\sused,\sit\sactually\sdeletes\sthe\ssurplus\soverflow\sfiles.\s\sThis\sallows\nVACUUM\sto\sshrink\sthe\sdatabase\sagain. +D 2011-12-16T04:57:32.390 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5b4a3e12a850b021547e43daf886b25133b44c07 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 8d416f29ad8573f32601f6056c9d2b17472e9ad5 -F src/test_multiplex.c c87d27283e2f578c17a8db7a8cac095f576f4323 +F src/test_multiplex.c 993f930c9c8e1aa0f4fe7b5a74c9a60fbf3447e4 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -978,7 +978,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 2685c2b949061f18bf6a4940eac8c8148873abb6 -R 45a748a7c5e87682cd34056f04633da7 +P e9177f7d4ec19f0841d0613990d2fb5725700ba1 +R c09d355f93cdab856d4274295151320d U drh -Z 8c7f37ab067ed94fde60690f325712e2 +Z 4efbc5088c9568c2b7090122e09b39c8 diff --git a/manifest.uuid b/manifest.uuid index 57de583d49..4852f04967 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9177f7d4ec19f0841d0613990d2fb5725700ba1 \ No newline at end of file +5aefef04d6cc3b734c3f05e036b3869a52248f4d \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index d844604ab5..f01a7f585b 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -133,7 +133,8 @@ struct multiplexGroup { int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ unsigned int szChunk; /* Chunk size used for this group */ - int bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bEnabled; /* TRUE to use Multiplex VFS for this file */ + unsigned char bTruncate; /* TRUE to enable truncation of databases */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -276,12 +277,17 @@ static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. +** +** For chunk 0, the pGroup->flags determines whether or not a new file +** is created if it does not already exist. For chunks 1 and higher, the +** file is created only if createFlag is 1. */ static sqlite3_file *multiplexSubOpen( - multiplexGroup *pGroup, - int iChunk, - int *rc, - int *pOutFlags + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc, /* Result code in and out */ + int *pOutFlags, /* Output flags */ + int createFlag /* True to create if iChunk>0 */ ){ sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ @@ -299,15 +305,28 @@ static sqlite3_file *multiplexSubOpen( *rc = multiplexSubFilename(pGroup, iChunk); if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + int flags, rc2, bExists; + if( iChunk==0 ) createFlag = (pGroup->flags & SQLITE_OPEN_CREATE)!=0; + flags = pGroup->flags; + if( createFlag ){ + flags |= SQLITE_OPEN_CREATE; + }else if( pGroup->aReal[iChunk].z==0 ){ + return 0; + }else{ + *rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[iChunk].z, + SQLITE_ACCESS_EXISTS, &bExists); + if( *rc || !bExists ) return 0; + flags &= ~SQLITE_OPEN_CREATE; + } pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); if( pSubOpen==0 ){ *rc = SQLITE_NOMEM; return 0; } pGroup->aReal[iChunk].p = pSubOpen; - *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, - pGroup->flags, pOutFlags); - if( *rc!=SQLITE_OK ){ + rc2 = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + flags, pOutFlags); + if( rc2!=SQLITE_OK ){ sqlite3_free(pSubOpen); pGroup->aReal[iChunk].p = 0; return 0; @@ -316,6 +335,25 @@ static sqlite3_file *multiplexSubOpen( return pSubOpen; } +/* +** Return the size, in bytes, of chunk number iChunk. If that chunk +** does not exist, then return 0. This function does not distingish between +** non-existant files and zero-length files. +*/ +static sqlite3_int64 multiplexSubSize( + multiplexGroup *pGroup, /* The multiplexor group */ + int iChunk, /* Which chunk to open. 0==original file */ + int *rc /* Result code in and out */ +){ + sqlite3_file *pSub; + sqlite3_int64 sz = 0; + + pSub = multiplexSubOpen(pGroup, iChunk, rc, NULL, 0); + if( pSub==0 ) return 0; + *rc = pSub->pMethods->xFileSize(pSub, &sz); + return sz; +} + /* ** This is the implementation of the multiplex_control() SQL function. */ @@ -455,6 +493,7 @@ static int multiplexOpen( memset(pGroup, 0, sz); pMultiplexOpen->pGroup = pGroup; pGroup->bEnabled = -1; + pGroup->bTruncate = (flags & SQLITE_OPEN_MAIN_DB)==0; pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; if( zName ){ @@ -475,6 +514,7 @@ static int multiplexOpen( pGroup->bEnabled = 0; } } + if( sqlite3_uri_parameter(zName, "truncate") ) pGroup->bTruncate = 1; } pGroup->zName = p; memcpy(pGroup->zName, zName, nName+1); @@ -493,8 +533,8 @@ static int multiplexOpen( pGroup->flags = flags; rc = multiplexSubFilename(pGroup, 1); if( rc==SQLITE_OK ){ - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); - assert( pSubOpen || rc!=SQLITE_OK ); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags, 0); + if( pSubOpen==0 && rc==SQLITE_OK ) rc = SQLITE_CANTOPEN; } if( rc==SQLITE_OK ){ sqlite3_int64 sz; @@ -522,17 +562,18 @@ static int multiplexOpen( } }else{ /* If the first overflow file exists and if the size of the main file - ** is different from the chunk size, that means the chunk size is set - ** set incorrectly. So fix it. - ** - ** Or, if the first overflow file does not exist and the main file is - ** larger than the chunk size, that means the chunk size is too small. - ** But we have no way of determining the intended chunk size, so - ** just disable the multiplexor all togethre. - */ + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ rc = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, SQLITE_ACCESS_EXISTS, &bExists); - if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 + bExists = multiplexSubSize(pGroup, 1, &rc)>0; + if( rc==SQLITE_OK && bExists && sz==(sz&0xffff0000) && sz>0 && sz!=pGroup->szChunk ){ pGroup->szChunk = sz; }else if( rc==SQLITE_OK && !bExists && sz>pGroup->szChunk ){ @@ -674,7 +715,7 @@ static int multiplexRead( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_READ; }else{ @@ -683,7 +724,7 @@ static int multiplexRead( }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; @@ -719,7 +760,7 @@ static int multiplexWrite( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_WRITE; }else{ @@ -728,7 +769,7 @@ static int multiplexWrite( }else{ while( rc==SQLITE_OK && iAmt>0 ){ int i = (int)(iOfst / pGroup->szChunk); - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL, 1); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; @@ -756,28 +797,35 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_TRUNCATE; }else{ rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); } - }else if( (pGroup->flags & SQLITE_OPEN_MAIN_DB)==0 ){ - int rc2; + }else{ int i; + int iBaseGroup = (int)(size / pGroup->szChunk); sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->szChunk)+1; inReal; i++){ - multiplexSubClose(pGroup, i, pOrigVfs); + for(i = pGroup->nReal-1; i>iBaseGroup && rc==SQLITE_OK; i--){ + if( pGroup->bTruncate ){ + multiplexSubClose(pGroup, i, pOrigVfs); + }else{ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, 0); + } + } } - pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); - if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - rc = SQLITE_IOERR_TRUNCATE; + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, iBaseGroup, &rc, 0, 0); + if( pSubOpen ){ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); + } } + if( rc ) rc = SQLITE_IOERR_TRUNCATE; } multiplexLeave(); return rc; @@ -812,42 +860,18 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen==0 ){ rc = SQLITE_IOERR_FSTAT; }else{ rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); } }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; for(i=0; rc==SQLITE_OK; i++){ - sqlite3_file *pSubOpen = 0; - int exists = 0; - rc = multiplexSubFilename(pGroup, i); - if( rc!=SQLITE_OK ) break; - if( pGroup->nReal>i && pGroup->aReal[i].p!=0 ){ - exists = 1; - }else if( (pGroup->flags & SQLITE_OPEN_DELETEONCLOSE)==0 ){ - const char *zReal = pGroup->aReal[i].z; - rc = pOrigVfs->xAccess(pOrigVfs, zReal, SQLITE_ACCESS_EXISTS, &exists); - } - if( exists==0 ){ - /* stop at first "gap" or IO error. */ - break; - } - if( rc==SQLITE_OK ){ - pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); - } - assert( pSubOpen || rc!=SQLITE_OK ); - if( rc==SQLITE_OK ){ - sqlite3_int64 sz = 0; - rc = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( rc==SQLITE_OK && sz>pGroup->szChunk ){ - rc = SQLITE_IOERR_FSTAT; - } - *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; - } + sqlite3_int64 sz = multiplexSubSize(pGroup, i, &rc); + if( sz==0 ) break; + *pSize = i*(sqlite3_int64)pGroup->szChunk + sz; } } multiplexLeave(); @@ -859,7 +883,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } @@ -871,7 +895,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } @@ -883,7 +907,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } @@ -931,7 +955,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); if( op==SQLITE_FCNTL_VFSNAME && rc==SQLITE_OK ){ @@ -948,7 +972,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen && pSubOpen->pMethods->xSectorSize ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } @@ -960,7 +984,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } @@ -978,7 +1002,7 @@ static int multiplexShmMap( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } @@ -995,7 +1019,7 @@ static int multiplexShmLock( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } @@ -1007,7 +1031,7 @@ static int multiplexShmLock( static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } @@ -1018,7 +1042,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); }