From: dan Date: Mon, 18 Apr 2016 18:18:18 +0000 (+0000) Subject: Fix some zipvfs related problems in RBU vacuum. X-Git-Tag: version-3.13.0~75^2~8 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=beccf1d16f3cf00c58349dd1a6114b13a920ee4b;p=thirdparty%2Fsqlite.git Fix some zipvfs related problems in RBU vacuum. FossilOrigin-Name: d76f4aaa4caab713460421bd27365a82ac986c20 --- diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 280b64bb2a..f98a836852 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -190,6 +190,8 @@ typedef sqlite3_int64 i64; #define WAL_LOCK_CKPT 1 #define WAL_LOCK_READ0 3 +#define SQLITE_FCNTL_RBUCNT 5149216 + /* ** A structure to store values read from the rbu_state table in memory. */ @@ -393,6 +395,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2325,6 +2328,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ + int nRbu = 0; assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); assert( rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2332,6 +2336,10 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + if( rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, &nRbu); + } + /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ if( p->zState ){ @@ -2355,7 +2363,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, 1); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, nRbu<=1); } p->eStage = 0; @@ -2363,20 +2371,13 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else{ - int frc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_ZIPVFS, 0); char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, frc!=SQLITE_OK); + p->dbMain = rbuOpenDbhandle(p, zTarget, nRbu<=1); sqlite3_free(zTarget); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, - "PRAGMA journal_mode=off; PRAGMA zipvfs_journal_mode = off;" - "BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 - ); - } } } @@ -3445,7 +3446,7 @@ static sqlite3rbu *openRbuHandle( ** created at this point. */ p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } /* Check if the main database is a zipvfs db. If it is, set the upper @@ -3787,6 +3788,11 @@ static void rbuPutU32(u8 *aBuf, u32 iVal){ aBuf[3] = (iVal >> 0) & 0xFF; } +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3812,6 +3818,37 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( p->pRbu && rbuIsVacuum(p->pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + sqlite3_file *pFd = 0; + rc = sqlite3_file_control( + p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd + ); + if( rc==SQLITE_OK ){ + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + } + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + rbuPutU32(&aBuf[52], 0); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + + if( iAmt>100 ){ + assert( iAmt>=101 ); + memset(&aBuf[101], 0, iAmt-101); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3893,7 +3930,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3905,7 +3955,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3968,6 +4020,11 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + int *pnRbu = (int*)pArg; + (*pnRbu)++; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ diff --git a/manifest b/manifest index 6840069cfa..d23cec1ffd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-vacuum\sswitch\sto\sthe\s"rbu"\sdemonstration\sprogram. -D 2016-04-18T09:17:05.073 +C Fix\ssome\szipvfs\srelated\sproblems\sin\sRBU\svacuum. +D 2016-04-18T18:18:18.881 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 721c6c116018b5d02f6318b6bbb7834098bc6a07 +F ext/rbu/sqlite3rbu.c 372ed3aaa396e3edfacdb8976d49aafdbe330cfa F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bae7f875f476f6e01112751cb404fef42ba0a01c -R 446c70a62698b2fec86a1f84d13dd5af +P 9a0078a538c7e73a009960347b8953c5af99fefd +R dc7cff23cf20a157412c6c371d56e10b U dan -Z be7942fa7aee76e8d89cc36d886361a8 +Z 72aacc0891d9c46e37fe5c65eb96b678 diff --git a/manifest.uuid b/manifest.uuid index c5a45627c9..e3fa7de1a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a0078a538c7e73a009960347b8953c5af99fefd \ No newline at end of file +d76f4aaa4caab713460421bd27365a82ac986c20 \ No newline at end of file