From: adam Date: Mon, 10 Oct 2011 22:11:44 +0000 (+0000) Subject: Merging in cherry picked diffs for persist wal, alloc padding, wal-safe vacuum and... X-Git-Tag: mountain-lion~17^2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=aec336af26ba4529d808c75deb201bdef31faa5e;p=thirdparty%2Fsqlite.git Merging in cherry picked diffs for persist wal, alloc padding, wal-safe vacuum and sqlite3_file_control based lockstate checking FossilOrigin-Name: db5b7b778c09c57501cb8266895a0ea4f2de7649 --- diff --git a/manifest b/manifest index c3cd1f857e..f5f83fd9dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cherrypick\sthe\ssqlite_data_count()\schanges\sfrom\s[d4f95b3b6e]\sand\n[9913996e7b]\sinto\sthe\sapple-osx\sbranch\sfor\sversion\s3.7.7. -D 2011-10-10T18:59:05.365 +C Merging\sin\scherry\spicked\sdiffs\sfor\spersist\swal,\salloc\spadding,\swal-safe\svacuum\sand\ssqlite3_file_control\sbased\slockstate\schecking +D 2011-10-10T22:11:44.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8410b02448997eb43bdf0ffa482c9bc2d2624e45 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,10 +124,10 @@ F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5 F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 986c15232757f2873dff35ee3b35cbf935fc573c +F src/backup.c 1cf56cc81b035e0a9884ddac13442edefcb0cb46 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c e09181860cc8b32ac3c767b0747a5ee66104817b +F src/btree.c 1870b70e376c5c9ff31e1235e4e4434212709265 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a @@ -149,10 +149,10 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c 015826a958f690302d27e096a68d50b3657e4201 F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 -F src/main.c a5414c66da45e65518e9f84e9cfe4ac1ff30ea06 +F src/main.c f3d35af74fdccb20f7edb31d842006d99887cf2e F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c 46095d62b241466ef51970e592aa3a7a87e443e1 +F src/mem1.c 1abe5a2c20d981e050504684bfdd3a126356e6d9 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 9b237d911ba9904142a804be727cc6664873f8a3 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f @@ -168,9 +168,9 @@ F src/os.c 1663f3754a3da185369a121a4f417762f77880b6 F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c af3aa6c091f501a590b45c95aa6af269ce728a94 -F src/os_win.c eafcd6b91cf204a7ef29ac1ef2a1b7132e132e58 -F src/pager.c e3688b37e781e8e069ed6375299da82a34a41794 +F src/os_unix.c 8f60f53930d4c9e781c46d803b3534d004282442 +F src/os_win.c daa67fe04bd162fbcc387214e9614a7faa6b90b1 +F src/pager.c 4bee5b51c33855cd8de4094709912b000f2bccff F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce @@ -184,15 +184,15 @@ F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c 75b7dd8c6a5ff9b8ec3f7ed51d31af8b30f114bc F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 -F src/sqlite.h.in 7289ab65b3eaf9dad5284b100764f3db1bd667a9 -F src/sqlite3_private.h 1d18557420cb0cc51ff31ec0a3fcce11e5cd6f5a +F src/sqlite.h.in 05e72174ea58476dc71db4bb6816f5b79a100f76 +F src/sqlite3_private.h e3b586e0aa329075d99be7198df9bc80c5b19e2d F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 -F src/sqliteInt.h 7240bba17d8f49a667d67bdbbe4cdf898536fc61 +F src/sqliteInt.h 2f4919d0105a821f04433a181f877c8a70201f9a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 5db825be61708b1a2b3f8f6e185e9b753829acef -F src/test1.c 4671911a936bc2c9a845bf32f70d770e7a5cd0b8 +F src/test1.c 1ce078b76ff2876c32b6ae4d1bbf754f704c87cb F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -246,12 +246,12 @@ F src/vdbe.c 80d511d7096918f4f6be6a63988a885c54dd1048 F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 7f01db7d26758b2be316116f8325a0b9f0d8edd6 -F src/vdbeaux.c ebe8ca774b6bb9fb4252037f3bb53883b632704e +F src/vdbeaux.c cfb5207d13d0bc3d2753bba67900c462b60ecb08 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 4b92fe7355f682368203d29b2be7125cbab85e79 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 -F src/wal.c 876f9b68df44e65a32c5106010a0a95ee554a284 +F src/wal.c eea77c324942f7e31ce9c3a5e6e86c4a0424fa09 F src/wal.h e75d87752bd5df3dc4152ee2cb3b0dcd0b309e5e F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 @@ -897,6 +897,7 @@ F test/walfault.test 7db81f3dac64ce8897196f199c2909078bcabf8d F test/walhook.test c934ac5219fee2b4e7653d291db9107b8dc73bba F test/walmode.test 8fe643a94841a1f95186e91e220181429f1188c1 F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 +F test/walpersist.test 45fb0c94fb63908e2d66b1d99ce4645bfce0fa1e F test/walro.test 05769ae10ddce1b6ad41bdd5326ab7ef951c8278 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test 989854bc5c214700a9f2d545bb158643813b8881 @@ -954,10 +955,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P c6158b254fabd02a3d58b1a047a3c5fa979b41a8 -R 698ad78198c29aa949534e4c4e347713 -T *branch * apple-osx-377 -T *sym-apple-osx-377 * -T -sym-apple-osx * -U drh -Z 06f34d41d072554967ad572a94c4a640 +P aef7945c423a8338374572eeda9444866c64569b +R d7448c6862ea7b8b001f5bf968511903 +U adam +Z 39deebc534cda2d5bd1c84327ddbee27 diff --git a/manifest.uuid b/manifest.uuid index 65c398afe7..1d16fcb865 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aef7945c423a8338374572eeda9444866c64569b \ No newline at end of file +db5b7b778c09c57501cb8266895a0ea4f2de7649 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 4d7ae31834..322ff87af6 100644 --- a/src/backup.c +++ b/src/backup.c @@ -417,6 +417,15 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ if( p->pDestDb ){ sqlite3ResetInternalSchema(p->pDestDb, -1); + } + + if( destMode==PAGER_JOURNALMODE_WAL ){ + /* This call cannot fail. The success of the BtreeUpdateMeta() + ** method above indicates that a write transaction has been opened + ** and page 1 is already dirty. Therefore this always succeeds. + */ + TESTONLY(int rc2 =) sqlite3BtreeSetVersion(p->pDest, 2); + assert( rc2==SQLITE_OK ); } /* Set nDestTruncate to the final number of pages in the destination diff --git a/src/btree.c b/src/btree.c index 256221daee..7e7dfdb092 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8169,7 +8169,6 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - assert( pBtree->inTrans==TRANS_NONE ); assert( iVersion==1 || iVersion==2 ); /* If setting the version fields to 1, do not automatically open the diff --git a/src/main.c b/src/main.c index e03c9b6e86..0c6ab65051 100644 --- a/src/main.c +++ b/src/main.c @@ -2979,144 +2979,26 @@ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ return 0; } -#if (SQLITE_ENABLE_APPLE_SPI>0) -#define SQLITE_FILE_HEADER_LEN 16 -#include -#include "sqlite3_private.h" -#include "btreeInt.h" -#include -#include - -/* Check for a conflicting lock. If one is found, print an this - ** on standard output using the format string given and return 1. - ** If there are no conflicting locks, return 0. - */ -static int isLocked( - pid_t pid, /* PID to test for lock owner */ - int h, /* File descriptor to check */ - int type, /* F_RDLCK or F_WRLCK */ - unsigned int iOfst, /* First byte of the lock */ - unsigned int iCnt, /* Number of bytes in the lock range */ - const char *zType /* Type of lock */ -){ - struct flock lk; - int err; - - memset(&lk, 0, sizeof(lk)); - lk.l_type = type; - lk.l_whence = SEEK_SET; - lk.l_start = iOfst; - lk.l_len = iCnt; - - if( pid!=SQLITE_LOCKSTATE_ANYPID ){ -#ifndef F_GETLKPID -# warning F_GETLKPID undefined, _sqlite3_lockstate falling back to F_GETLK - err = fcntl(h, F_GETLK, &lk); -#else - lk.l_pid = pid; - err = fcntl(h, F_GETLKPID, &lk); -#endif - }else{ - err = fcntl(h, F_GETLK, &lk); - } - - if( err==(-1) ){ - fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); - return -1; - } - - if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ -#ifdef SQLITE_DEBUG - fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); -#endif - return 1; - } - return 0; -} +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) -/* - ** Location of locking bytes in the database file - */ -#ifndef PENDING_BYTE -# define PENDING_BYTE (0x40000000) -# define RESERVED_BYTE (PENDING_BYTE+1) -# define SHARED_FIRST (PENDING_BYTE+2) -# define SHARED_SIZE 510 -#endif /* PENDING_BYTE */ - -/* - ** Lock locations for shared-memory locks used by WAL mode. - */ -#ifndef SHM_BASE -# define SHM_BASE 120 -# define SHM_WRITE SHM_BASE -# define SHM_CHECKPOINT (SHM_BASE+1) -# define SHM_RECOVER (SHM_BASE+2) -# define SHM_READ_FIRST (SHM_BASE+3) -# define SHM_READ_SIZE 5 -#endif /* SHM_BASE */ +#include "sqlite3_private.h" /* ** Testing a file path for sqlite locks held by a process ID. ** Returns SQLITE_LOCKSTATE_ON if locks are present on path ** that would prevent writing to the database. -** -** This test only works for lock testing on unix/posix VFS. -** Adapted from tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce */ int _sqlite3_lockstate(const char *path, pid_t pid){ - int hDb; /* File descriptor for the open database file */ - int hShm; /* File descriptor for WAL shared-memory file */ - ssize_t got; /* Bytes read from header */ - int isWal; /* True if in WAL mode */ - int nLock = 0; /* Number of locks held */ - unsigned char aHdr[100]; /* Database header */ - - /* Open the file at path and make sure we are dealing with a database file */ - hDb = open(path, O_RDONLY | O_NOCTTY); - if( hDb<0 ){ - return SQLITE_LOCKSTATE_ERROR; - } - assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); - got = pread(hDb, aHdr, 100, 0); - if( got<0 ){ - close(hDb); - return SQLITE_LOCKSTATE_ERROR; - } - if( got!=100 || memcmp(aHdr, SQLITE_FILE_HEADER, SQLITE_FILE_HEADER_LEN)!=0 ){ - close(hDb); - return SQLITE_LOCKSTATE_NOTADB; - } + sqlite3 *db = NULL; - /* First check for an exclusive lock */ - nLock += isLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE"); - isWal = aHdr[18]==2; - if( nLock==0 && isWal==0 ){ - /* Rollback mode */ - nLock += isLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, "PENDING|RESERVED|SHARED"); - } - close(hDb); - if( nLock==0 && isWal!=0 ){ - char zShm[MAXPATHLEN]; - - close(hDb); - /* WAL mode */ - strlcpy(zShm, path, MAXPATHLEN); - strlcat(zShm, "-shm", MAXPATHLEN); - hShm = open(zShm, O_RDONLY, 0); - if( hShm<0 ){ - return SQLITE_LOCKSTATE_OFF; - } - if( isLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || - isLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ - nLock = 1; - } - close(hShm); - } - if( nLock>0 ){ - return SQLITE_LOCKSTATE_ON; + if( sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK ){ + LockstatePID lockstate = {pid, -1}; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_LOCKSTATE_PID, &lockstate); + sqlite3_close(db); + int state = lockstate.state; + return state; } - return SQLITE_LOCKSTATE_OFF; + return SQLITE_LOCKSTATE_ERROR; } #endif /* SQLITE_ENABLE_APPLE_SPI */ diff --git a/src/mem1.c b/src/mem1.c index 6b1c30afb5..43a463d447 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -45,6 +45,7 @@ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); #define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) #endif @@ -60,11 +61,8 @@ static void *sqlite3MemMalloc(int nByte){ sqlite3_int64 *p; assert( nByte>0 ); nByte = ROUND8(nByte); - p = SQLITE_MALLOC( nByte+8 ); - if( p ){ - p[0] = nByte; - p++; - }else{ + p = SQLITE_MALLOC( nByte ); + if( !p ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); } @@ -82,7 +80,6 @@ static void *sqlite3MemMalloc(int nByte){ static void sqlite3MemFree(void *pPrior){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 ); - p--; SQLITE_FREE(p); } @@ -93,9 +90,7 @@ static void sqlite3MemFree(void *pPrior){ static int sqlite3MemSize(void *pPrior){ sqlite3_int64 *p; if( pPrior==0 ) return 0; - p = (sqlite3_int64*)pPrior; - p--; - return (int)p[0]; + return (int)SQLITE_MALLOCSIZE(pPrior); } /* @@ -112,12 +107,8 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ - p--; - p = SQLITE_REALLOC(p, nByte+8 ); - if( p ){ - p[0] = nByte; - p++; - }else{ + p = SQLITE_REALLOC(p, nByte ); + if( !p ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed memory resize %u to %u bytes", diff --git a/src/os_unix.c b/src/os_unix.c index 85e17598bb..9ec16348a5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -258,8 +258,9 @@ struct unixFile { /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ -#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ -#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ /* ** Include code that is common to all os_*.c files @@ -1539,6 +1540,8 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ return rc; } +static int _unixFileLock(unixFile *pFile, struct flock *pLock, int retry); + /* ** Attempt to set a system-lock on the file pFile. The lock is ** described by pLock. @@ -1559,7 +1562,15 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ ** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). */ static int unixFileLock(unixFile *pFile, struct flock *pLock){ - int rc; + return _unixFileLock(pFile, pLock, 0); +} + +static int unixFileLock2(unixFile *pFile, struct flock *pLock){ + return _unixFileLock(pFile, pLock, 10); +} + +static int _unixFileLock(unixFile *pFile, struct flock *pLock, int retry) { + int rc = 0; unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); assert( pInode!=0 ); @@ -1581,7 +1592,13 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ rc = 0; } }else{ - rc = osFcntl(pFile->h, F_SETLK, pLock); + int i = 0; + do { + rc = osFcntl(pFile->h, F_SETLK, pLock); + if (rc && retry) { + usleep(100 * (++i)); + } + } while (!rc && retry--); } return rc; } @@ -1752,7 +1769,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ lock.l_start = PENDING_BYTE; lock.l_len = 1L; lock.l_type = F_UNLCK; - if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + if( unixFileLock2(pFile, &lock) && rc==SQLITE_OK ){ /* This could happen with a network mount */ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR @@ -1920,15 +1937,15 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ assert( handleNFSUnlock==0 ); #endif #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + int tErrno; /* Error code from system call errors */ if( handleNFSUnlock ){ - int tErrno; /* Error code from system call errors */ off_t divSize = SHARED_SIZE - 1; lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); @@ -1944,9 +1961,13 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); +#else + rc = SQLITE_IOERR_UNLOCK; +#endif if( IS_LOCK_ERROR(rc) ){ pFile->lastErrno = tErrno; } @@ -1956,7 +1977,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST+divSize; lock.l_len = SHARED_SIZE-divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); @@ -1975,13 +1996,10 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; - if( unixFileLock(pFile, &lock) ){ -#if OSLOCKING_CHECK_BUSY_IOERR + if( unixFileLock2(pFile, &lock) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } #else /* In theory, the call to unixFileLock() cannot fail because another ** process is holding an incompatible lock. If it does, this @@ -1990,8 +2008,11 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ ** SQLITE_BUSY would confuse the upper layer (in practice it causes ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; - pFile->lastErrno = errno; + pFile->lastErrno = tErrno; #endif + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } goto end_unlock; } } @@ -2000,7 +2021,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_whence = SEEK_SET; lock.l_start = PENDING_BYTE; lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock2(pFile, &lock)==0 ){ pInode->eFileLock = SHARED_LOCK; }else{ #if OSLOCKING_CHECK_BUSY_IOERR @@ -2029,7 +2050,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ SimulateIOErrorBenign(1); SimulateIOError( h=(-1) ) SimulateIOErrorBenign(0); - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock2(pFile, &lock)==0 ){ pInode->eFileLock = NO_LOCK; }else{ #if OSLOCKING_CHECK_BUSY_IOERR @@ -3845,25 +3866,271 @@ static int getDbPathForUnixFile(unixFile *pFile, char *dbPath); #endif static int isProxyLockingMode(unixFile *); +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static int unixTruncateDatabase(unixFile *, int); + +static int unixInvalidateSupportFiles(unixFile *, int); + +static int unixReplaceDatabase(unixFile *pFile, sqlite3 *srcdb) { + sqlite3_file *id = (sqlite3_file *)pFile; + Btree *pSrcBtree = NULL; + sqlite3_file *src_file = NULL; + unixFile *pSrcFile = NULL; + char srcWalPath[MAXPATHLEN+5]; + int srcWalFD = -1; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + sqlite3 *srcdb2 = NULL; + copyfile_state_t s; + int corruptSrcFileLock = 0; + int corruptDstFileLock = 0; + int isSrcCorrupt = 0; + int isDstCorrupt = 0; + + if( !sqlite3SafetyCheckOk(srcdb) ){ + return SQLITE_MISUSE; + } + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isDstCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptDstFileLock); + } + if( rc ){ + return rc; + } + } + /* get the src file descriptor adhering to the db struct access rules + ** this code is modeled after sqlite3_file_control() in main.c + */ + sqlite3_mutex_enter(srcdb->mutex); + if( srcdb->nDb>0 ){ + pSrcBtree = srcdb->aDb[0].pBt; + } + if( pSrcBtree ){ + Pager *pSrcPager; + sqlite3BtreeEnter(pSrcBtree); + pSrcPager = sqlite3BtreePager(pSrcBtree); + assert( pSrcPager!=0 ); + src_file = sqlite3PagerFile(pSrcPager); + assert( src_file!=0 ); + if( src_file->pMethods ){ + int srcFlags = 0; + pSrcFile = (unixFile *)src_file; + /* wal mode db cannot be opened readonly */ + if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { + srcFlags = SQLITE_OPEN_READWRITE; + } else { + srcFlags = SQLITE_OPEN_READONLY; + } +#if SQLITE_ENABLE_DATA_PROTECTION + srcFlags |= pSrcFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pSrcFile) ){ + srcFlags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); + if( rc==SQLITE_OK ){ + /* start a deferred transaction and read to establish a read lock */ + rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isSrcCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, &corruptSrcFileLock); + } + } + } + } + if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ + rc = SQLITE_INTERNAL; + } + if( rc!=SQLITE_OK ){ + goto end_replace_database; + } + /* both databases are locked appropriately, copy the src wal journal if + ** one exists and then the actual database file + */ + strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); + strlcat(srcWalPath, "-wal", MAXPATHLEN+5); + srcWalFD = open(srcWalPath, O_RDONLY); + if( !(srcWalFD<0) ){ + char dstWalPath[MAXPATHLEN+5]; + int dstWalFD = -1; + strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); + strlcat(dstWalPath, "-wal", MAXPATHLEN+5); + dstWalFD = open(dstWalPath, O_RDWR|O_CREAT, SQLITE_DEFAULT_FILE_PERMISSIONS); + if( !(dstWalFD<0) ){ + s = copyfile_state_alloc(); + lseek(srcWalFD, 0, SEEK_SET); + lseek(dstWalFD, 0, SEEK_SET); + if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_ALL) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + pFile->lastErrno = err; + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + close(dstWalFD); + } + close(srcWalFD); + } + if( rc==SQLITE_OK ){ + /* before we copy, ensure that the file change counter will be modified */ + uint32_t srcChange = 0; + uint32_t dstChange = 0; + pread(pSrcFile->h, &srcChange, 4, 24); + pread(pFile->h, &dstChange, 4, 24); + + /* copy the actual database */ + s = copyfile_state_alloc(); + lseek(pSrcFile->h, 0, SEEK_SET); + lseek(pFile->h, 0, SEEK_SET); + if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_ALL) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + pFile->lastErrno = err; + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + + if (srcChange == dstChange) { + /* modify the change counter to force page zero to be reloaded */ + dstChange ++; + pwrite(pFile->h, &dstChange, 4, 24); + } + } + if( isSrcCorrupt ){ + sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); + }else{ + /* done with the source db so end the transaction */ + sqlite3_exec(srcdb2, "COMMIT", 0, 0, 0); + } + /* zero out any old journal clutter */ + if( rc==SQLITE_OK ){ + int skipWAL = (srcWalFD<0)?0:1; + unixInvalidateSupportFiles(pFile, skipWAL); + } + +end_replace_database: + if( pSrcBtree ){ + sqlite3_close(srcdb2); + sqlite3BtreeLeave(pSrcBtree); + } + sqlite3_mutex_leave(srcdb->mutex); + if( isDstCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); + }else{ + sqlite3demo_superunlock(pLock); + } + return rc; +} + +#define SQLITE_FILE_HEADER_LEN 16 +#include "btreeInt.h" +/* Check for a conflicting lock. If one is found, print an this + ** on standard output using the format string given and return 1. + ** If there are no conflicting locks, return 0. + */ +static int unixIsLocked( + pid_t pid, /* PID to test for lock owner */ + int h, /* File descriptor to check */ + int type, /* F_RDLCK or F_WRLCK */ + unsigned int iOfst, /* First byte of the lock */ + unsigned int iCnt, /* Number of bytes in the lock range */ + const char *zType /* Type of lock */ +){ + struct flock lk; + int err; + + memset(&lk, 0, sizeof(lk)); + lk.l_type = type; + lk.l_whence = SEEK_SET; + lk.l_start = iOfst; + lk.l_len = iCnt; + + if( pid!=SQLITE_LOCKSTATE_ANYPID ){ +#ifndef F_GETLKPID +# warning F_GETLKPID undefined, _sqlite3_lockstate falling back to F_GETLK + err = fcntl(h, F_GETLK, &lk); +#else + lk.l_pid = pid; + err = fcntl(h, F_GETLKPID, &lk); +#endif + }else{ + err = fcntl(h, F_GETLK, &lk); + } + + if( err==(-1) ){ + fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); + return -1; + } + + if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ +#ifdef SQLITE_DEBUG + fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); +#endif + return 1; + } + return 0; +} + +static int unixLockstatePid(unixFile *, pid_t, int *); + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + /* ** Information and control of an open file handle. */ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((unixFile*)id)->eFileLock; + *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { - *(int*)pArg = ((unixFile*)id)->lastErrno; + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((unixFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + return fcntlSizeHint(pFile, *(i64 *)pArg); + } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + *(int*)pArg = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; + }else if( bPersist==0 ){ + pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL; + }else{ + pFile->ctrlFlags |= UNIXFILE_PERSIST_WAL; + } + return SQLITE_OK; } #ifndef NDEBUG /* The pager calls this method to signal that it has done @@ -3877,267 +4144,27 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - case SQLITE_SET_LOCKPROXYFILE: - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) - case SQLITE_TRUNCATE_DATABASE: { - unixFile *pFile = (unixFile*)id; - int rc = SQLITE_OK; - void *pLock = NULL; - int flags = 0; - int corruptFileLock = 0; - int isCorrupt = 0; - -#if SQLITE_ENABLE_DATA_PROTECTION - flags |= pFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pFile) ){ - flags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); - if( rc ){ - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptFileLock); - } - if( rc ){ - return rc; - } - } - rc = pFile->pMethod->xTruncate(id, ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); - - if( rc==SQLITE_OK ){ - char jPath[MAXPATHLEN+9]; - int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); - if( zLenpMethod->xSync(id, SQLITE_SYNC_FULL); - } - if( isCorrupt ){ - sqlite3demo_superunlock_corrupt(id, corruptFileLock); - }else{ - sqlite3demo_superunlock(pLock); - } - return rc; + case SQLITE_FCNTL_TRUNCATE_DATABASE: { + return unixTruncateDatabase(pFile, (pArg ? (*(int *)pArg) : 0)); } + case SQLITE_FCNTL_REPLACE_DATABASE: { + return unixReplaceDatabase(pFile, (sqlite3 *)pArg); + } + case SQLITE_FCNTL_LOCKSTATE_PID: { + LockstatePID *pLockstate; + int rc; - case SQLITE_REPLACE_DATABASE: { - unixFile *pFile = (unixFile*)id; - sqlite3 *srcdb = (sqlite3 *)pArg; - Btree *pSrcBtree = NULL; - sqlite3_file *src_file = NULL; - unixFile *pSrcFile = NULL; - char srcWalPath[MAXPATHLEN+5]; - int srcWalFD = -1; - int rc = SQLITE_OK; - void *pLock = NULL; - int flags = 0; - sqlite3 *srcdb2 = NULL; - copyfile_state_t s; - int corruptSrcFileLock = 0; - int corruptDstFileLock = 0; - int isSrcCorrupt = 0; - int isDstCorrupt = 0; - - if( !sqlite3SafetyCheckOk(srcdb) ){ + if( pArg==NULL ){ return SQLITE_MISUSE; } - -#if SQLITE_ENABLE_DATA_PROTECTION - flags |= pFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pFile) ){ - flags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); - if( rc ){ - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isDstCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptDstFileLock); - } - if( rc ){ - return rc; - } - } - /* get the src file descriptor adhering to the db struct access rules - ** this code is modeled after sqlite3_file_control() in main.c - */ - sqlite3_mutex_enter(srcdb->mutex); - if( srcdb->nDb>0 ){ - pSrcBtree = srcdb->aDb[0].pBt; - } - if( pSrcBtree ){ - Pager *pSrcPager; - sqlite3BtreeEnter(pSrcBtree); - pSrcPager = sqlite3BtreePager(pSrcBtree); - assert( pSrcPager!=0 ); - src_file = sqlite3PagerFile(pSrcPager); - assert( src_file!=0 ); - if( src_file->pMethods ){ - int srcFlags = 0; - pSrcFile = (unixFile *)src_file; - /* wal mode db cannot be opened readonly */ - if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { - srcFlags = SQLITE_OPEN_READWRITE; - } else { - srcFlags = SQLITE_OPEN_READONLY; - } -#if SQLITE_ENABLE_DATA_PROTECTION - srcFlags |= pSrcFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pSrcFile) ){ - srcFlags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); - if( rc==SQLITE_OK ){ - /* start a deferred transaction and read to establish a read lock */ - rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isSrcCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, &corruptSrcFileLock); - } - } - } - } - if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ - rc = SQLITE_INTERNAL; - } - if( rc!=SQLITE_OK ){ - goto end_replace_database; - } - /* both databases are locked appropriately, copy the src wal journal if - ** one exists and then the actual database file - */ - strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); - strlcat(srcWalPath, "-wal", MAXPATHLEN+5); - srcWalFD = open(srcWalPath, O_RDONLY); - if( !(srcWalFD<0) ){ - char dstWalPath[MAXPATHLEN+5]; - int dstWalFD = -1; - strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); - strlcat(dstWalPath, "-wal", MAXPATHLEN+5); - dstWalFD = open(dstWalPath, O_RDWR|O_CREAT, SQLITE_DEFAULT_FILE_PERMISSIONS); - if( !(dstWalFD<0) ){ - s = copyfile_state_alloc(); - lseek(srcWalFD, 0, SEEK_SET); - lseek(dstWalFD, 0, SEEK_SET); - if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_ALL) ){ - int err=errno; - switch(err) { - case ENOMEM: - rc = SQLITE_NOMEM; - break; - default: - pFile->lastErrno = err; - rc = SQLITE_IOERR; - } - } - copyfile_state_free(s); - close(dstWalFD); - } - close(srcWalFD); - } - if( rc==SQLITE_OK ){ - /* before we copy, ensure that the file change counter will be modified */ - uint32_t srcChange = 0; - uint32_t dstChange = 0; - pread(pSrcFile->h, &srcChange, 4, 24); - pread(pFile->h, &dstChange, 4, 24); - - /* copy the actual database */ - s = copyfile_state_alloc(); - lseek(pSrcFile->h, 0, SEEK_SET); - lseek(pFile->h, 0, SEEK_SET); - if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_ALL) ){ - int err=errno; - switch(err) { - case ENOMEM: - rc = SQLITE_NOMEM; - break; - default: - pFile->lastErrno = err; - rc = SQLITE_IOERR; - } - } - copyfile_state_free(s); - - if (srcChange == dstChange) { - /* modify the change counter to force page zero to be reloaded */ - dstChange ++; - pwrite(pFile->h, &dstChange, 4, 24); - } - } - if( isSrcCorrupt ){ - sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); - }else{ - /* done with the source db so end the transaction */ - sqlite3_exec(srcdb2, "COMMIT", 0, 0, 0); - } - /* zero out any old journal clutter */ - if( rc==SQLITE_OK ){ - char jPath[MAXPATHLEN+9]; - int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); - if( zLenpMethod->xSync(id, SQLITE_SYNC_FULL); - } - - end_replace_database: - if( pSrcBtree ){ - sqlite3_close(srcdb2); - sqlite3BtreeLeave(pSrcBtree); - } - sqlite3_mutex_leave(srcdb->mutex); - if( isDstCorrupt ){ - sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); - }else{ - sqlite3demo_superunlock(pLock); - } + pLockstate = (LockstatePID *)pArg; + rc = unixLockstatePid(pFile, pLockstate->pid, &(pLockstate->state)); return rc; } #endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ @@ -4470,8 +4497,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ } if( pInode->bProcessLock==0 ){ - pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + }else{ + pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); + } if( pShmNode->h<0 ){ const char *zRO; zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); @@ -4847,6 +4882,210 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static const char *unixTempFileDir(void); + +static int unixInvalidateSupportFiles(unixFile *pFile, int skipWAL) { + char jPath[MAXPATHLEN+9]; + int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); + if( zLenpInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode && !pShmNode->isReadonly ){ + struct stat sStat; + sqlite3_mutex_enter(pShmNode->mutex); + + if( pShmNode->h>=0 && !osFstat(pShmNode->h, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + bzero(pShmNode->apRegion[0], size); + sqlite3_mutex_leave(pShmNode->mutex); + unixLeaveMutex(); + continue; + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } + unixLeaveMutex(); + } + jLen = strlcpy(&jPath[zLen], extensions[j], 9); + if( jLen < 9 ){ + int jflags = (j<2) ? O_TRUNC : O_RDWR; + int jfd = open(jPath, jflags); + if( jfd==(-1) ){ + if( errno!=ENOENT ){ + perror(jPath); + } + } else { + if( j==2 ){ + struct stat sStat; + if( !osFstat(jfd, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + uint32_t zero = 0; + pwrite(jfd, &zero, (size_t)size, 0); + } + } + } + fsync(jfd); + close(jfd); + } + } + } + } + return SQLITE_OK; +} + +static int unixTruncateDatabase(unixFile *pFile, int bFlags) { + sqlite3_file *id = (sqlite3_file *)pFile; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + int corruptFileLock = 0; + int isCorrupt = 0; + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptFileLock); + } + if( rc ){ + return rc; + } + } + rc = pFile->pMethod->xTruncate(id, ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); + if( rc==SQLITE_OK ){ + unixInvalidateSupportFiles(pFile, 0); + } + pFile->pMethod->xSync(id, SQLITE_SYNC_FULL); + + + if( isCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptFileLock); + }else{ + sqlite3demo_superunlock(pLock); + } + return rc; +} + +/* + ** Lock locations for shared-memory locks used by WAL mode. + */ +#ifndef SHM_BASE +# define SHM_BASE 120 +# define SHM_WRITE SHM_BASE +# define SHM_CHECKPOINT (SHM_BASE+1) +# define SHM_RECOVER (SHM_BASE+2) +# define SHM_READ_FIRST (SHM_BASE+3) +# define SHM_READ_SIZE 5 +#endif /* SHM_BASE */ + +/* +** This test only works for lock testing on unix/posix VFS. +** Adapted from tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +*/ +static int unixLockstatePid(unixFile *pFile, pid_t pid, int *pLockstate){ + int hDb; /* File descriptor for the open database file */ + int hShm = -1; /* File descriptor for WAL shared-memory file */ + ssize_t got; /* Bytes read from header */ + int isWal; /* True if in WAL mode */ + int nLock = 0; /* Number of locks held */ + unsigned char aHdr[100]; /* Database header */ + + assert(pLockstate); + + /* make sure we are dealing with a database file */ + hDb = pFile->h; + if( hDb<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); + got = pread(hDb, aHdr, 100, 0); + if( got<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + if( got!=100 || memcmp(aHdr, SQLITE_FILE_HEADER, SQLITE_FILE_HEADER_LEN)!=0 ){ + *pLockstate = SQLITE_LOCKSTATE_NOTADB; + return SQLITE_NOTADB; + } + + /* First check for an exclusive lock */ + nLock += unixIsLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE"); + isWal = aHdr[18]==2; + if( nLock==0 && isWal==0 ){ + /* Rollback mode */ + nLock += unixIsLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, "PENDING|RESERVED|SHARED"); + } + if( nLock==0 && isWal!=0 ){ + /* lookup the file descriptor for the shared memory file if we have it open in this process */ + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode ){ + sqlite3_mutex_enter(pShmNode->mutex); + + hShm = pShmNode->h; + if( hShm >= 0){ + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + } + + sqlite3_mutex_leave(pShmNode->mutex); + } + + if( hShm<0 ){ + /* the shared memory file isn't open in this process space, open our own FD */ + char zShm[MAXPATHLEN]; + + /* WAL mode */ + strlcpy(zShm, pFile->zPath, MAXPATHLEN); + strlcat(zShm, "-shm", MAXPATHLEN); + hShm = open(zShm, O_RDONLY, 0); + if( hShm<0 ){ + *pLockstate = SQLITE_LOCKSTATE_OFF; + unixLeaveMutex(); + return SQLITE_OK; + } + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + close(hShm); + } + unixLeaveMutex(); + } + if( nLock>0 ){ + *pLockstate = SQLITE_LOCKSTATE_ON; + } else { + *pLockstate = SQLITE_LOCKSTATE_OFF; + } + return SQLITE_OK; +} + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + + + /* ** Here ends the implementation of all sqlite3_file methods. ** diff --git a/src/os_win.c b/src/os_win.c index bd0f2f216a..88db683dbc 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -102,8 +102,9 @@ struct winFile { const sqlite3_io_methods *pMethod; /*** Must be first ***/ sqlite3_vfs *pVfs; /* The VFS used to open this file */ HANDLE h; /* Handle for accessing the file */ - unsigned char locktype; /* Type of lock currently held on this file */ + u8 locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 bPersistWal; /* True to persist WAL files */ DWORD lastErrno; /* The Windows errno from the last I/O error */ DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ @@ -1276,17 +1277,18 @@ static int winUnlock(sqlite3_file *id, int locktype){ ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((winFile*)id)->locktype; + *(int*)pArg = pFile->locktype; return SQLITE_OK; } case SQLITE_LAST_ERRNO: { - *(int*)pArg = (int)((winFile*)id)->lastErrno; + *(int*)pArg = (int)pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((winFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -1296,6 +1298,15 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); return SQLITE_OK; } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + *(int*)pArg = pFile->bPersistWal; + }else{ + pFile->bPersistWal = bPersist!=0; + } + return SQLITE_OK; + } case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; } diff --git a/src/pager.c b/src/pager.c index 22bc145468..5863d0b941 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6739,11 +6739,11 @@ static int pagerOpenWal(Pager *pPager){ if( rc==SQLITE_OK ){ #if SQLITE_ENABLE_DATA_PROTECTION rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, (pPager->vfsFlags & SQLITE_OPEN_FILEPROTECTION_MASK), + pPager->journalSizeLimit, (pPager->vfsFlags & (SQLITE_OPEN_FILEPROTECTION_MASK | SQLITE_OPEN_READONLY)), &pPager->pWal); #else rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, 0, &pPager->pWal); + pPager->journalSizeLimit, (pPager->vfsFlags & SQLITE_OPEN_READONLY), &pPager->pWal); #endif } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2d3676a3d9..6cba1ca76a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -737,16 +737,35 @@ struct sqlite3_io_methods { ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. -*/ -#define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 -#define SQLITE_FCNTL_SIZE_HINT 5 -#define SQLITE_FCNTL_CHUNK_SIZE 6 -#define SQLITE_FCNTL_FILE_POINTER 7 -#define SQLITE_FCNTL_SYNC_OMITTED 8 - +** +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write AHead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_PERSIST_WAL 10 +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO /* ** CAPI3REF: Mutex Handle diff --git a/src/sqlite3_private.h b/src/sqlite3_private.h index 5d39495ee8..dfc3a0e48a 100644 --- a/src/sqlite3_private.h +++ b/src/sqlite3_private.h @@ -27,17 +27,27 @@ */ extern int _sqlite3_lockstate(const char *path, pid_t pid); +/* +** Test an open database connection for sqlite locks held by a process ID, +** if a process has an open database connection this will avoid trashing file +** locks by re-using open file descriptors for the database file and support +** files (-shm) +*/ +#define SQLITE_FCNTL_LOCKSTATE_PID 103 + /* ** Pass the SQLITE_TRUNCATE_DATABASE operation code to sqlite3_file_control() ** to truncate a database and its associated journal file to zero length. */ -#define SQLITE_TRUNCATE_DATABASE 101 +#define SQLITE_FCNTL_TRUNCATE_DATABASE 101 +#define SQLITE_TRUNCATE_DATABASE SQLITE_FCNTL_TRUNCATE_DATABASE /* ** Pass the SQLITE_REPLACE_DATABASE operation code to sqlite3_file_control() ** and a sqlite3 pointer to another open database file to safely copy the ** contents of that database file into the receiving database. */ -#define SQLITE_REPLACE_DATABASE 102 +#define SQLITE_FCNTL_REPLACE_DATABASE 102 +#define SQLITE_REPLACE_DATABASE SQLITE_FCNTL_REPLACE_DATABASE #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 051b95e10c..7ead79b96f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3250,4 +3250,29 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ + +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + +/* +** An instance of the following structure is used to hold the process ID +** and return-by-reference lockstate value. The SQLITE_FCNTL_LOCKSTATE_PID +** requires the 4th argument to sqlite3_file_control to be a pointer to an +** instance of LockstatePID initialized with a LockstatePID.pid value equal +** to a process ID to be tested, or the special value SQLITE_LOCKSTATE_ANYPID +** The Lockstate.state value is always set to one of the following values +** when sqlite3_file_control returns: +** +** SQLITE_LOCKSTATE_OFF no active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_ON active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_NOTADB path points to a file that is not an sqlite db file +** SQLITE_LOCKSTATE_ERROR path was not vaild or was unreadable +*/ +typedef struct LockstatePID LockstatePID; +struct LockstatePID { + pid_t pid; /* Process ID to test */ + int state; /* The state of the lock (return value) */ +}; + +#endif + #endif /* _SQLITEINT_H_ */ diff --git a/src/test1.c b/src/test1.c index dc5f5d2de4..d7b7a010da 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4990,17 +4990,21 @@ static int file_control_truncate_test( Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3 *db; + int flags; int rc; - if( objc!=2 ){ + if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB FLAGS", 0); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ return TCL_ERROR; } - rc = sqlite3_file_control(db, NULL, SQLITE_TRUNCATE_DATABASE, 0); + if( Tcl_GetIntFromObj(interp, objv[2], &flags) ){ + return TCL_ERROR; + } + rc = sqlite3_file_control(db, NULL, SQLITE_TRUNCATE_DATABASE, &flags); if( rc ){ Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_ERROR; @@ -5283,6 +5287,39 @@ static int path_is_dos( return TCL_OK; } +/* +** tclcmd: file_control_persist_wal DB PERSIST-FLAG +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_PERSIST_WAL opcode. +*/ +static int file_control_persist_wal( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int bPersist; + char z[100]; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &bPersist) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, (void*)&bPersist); + sqlite3_snprintf(sizeof(z), z, "%d %d", rc, bPersist); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + + /* ** tclcmd: sqlite3_vfs_list ** @@ -5994,6 +6031,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_persist_wal", file_control_persist_wal, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, { "path_is_local", path_is_local, 0 }, diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 5d5f47aa38..cf0ec69e7f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1508,6 +1508,8 @@ void sqlite3VdbeMakeReady( zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; + /* Memory for registers, parameters, cursor, etc, is allocated in two ** passes. On the first pass, we try to reuse unused space at the ** end of the opcode array. If we are unable to satisfy all memory diff --git a/src/wal.c b/src/wal.c index 74334e9688..7176d30d74 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1285,7 +1285,11 @@ int sqlite3WalOpen( pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ - vfsFlags = flags | (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + if( flags&SQLITE_OPEN_READONLY ){ + vfsFlags = flags | SQLITE_OPEN_WAL; + } else { + vfsFlags = flags | (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + } rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, vfsFlags, &vfsFlags); if( rc==SQLITE_OK && vfsFlags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; @@ -1805,13 +1809,15 @@ int sqlite3WalClose( */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ + int bPersistWal = -1; if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = sqlite3WalCheckpoint( pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - if( rc==SQLITE_OK ){ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal); + if( rc==SQLITE_OK && bPersistWal!=1 ){ isDelete = 1; } } @@ -1864,6 +1870,9 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** reordering the reads and writes. */ aHdr = walIndexHdr(pWal); + if( aHdr==NULL ){ + return 1; /* Shouldn't be getting NULL from walIndexHdr, but we are */ + } memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); @@ -2048,7 +2057,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ */ if( cnt>5 ){ int nDelay = 1; /* Pause time in microseconds */ - if( cnt>100 ){ + if( cnt>500 ){ VVA_ONLY( pWal->lockError = 1; ) return SQLITE_PROTOCOL; } diff --git a/test/walpersist.test b/test/walpersist.test new file mode 100644 index 0000000000..3f8b0b8930 --- /dev/null +++ b/test/walpersist.test @@ -0,0 +1,68 @@ +# 2011 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL with persistent WAL file mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walpersist + +do_test walpersist-1.0 { + db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(randomblob(5000)); + } + file exists test.db-wal +} {1} +do_test walpersist-1.1 { + file exists test.db-shm +} {1} +do_test walpersist-1.2 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 0 0} +do_test walpersist-1.3 { + sqlite3 db test.db + db eval {SELECT length(a) FROM t1} +} {5000} +do_test walpersist-1.4 { + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} +do_test walpersist-1.5 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.6 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.7 { + file_control_persist_wal db -1 +} {0 1} +do_test walpersist-1.8 { + file_control_persist_wal db 0 +} {0 0} +do_test walpersist-1.9 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.10 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.11 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} + + + + +finish_test