From 4c1c865a2841eee090b9b135d3041998547e385a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Nov 2019 23:55:14 +0000 Subject: [PATCH] Experimental file-controls for controlling the use of the -shm file. FossilOrigin-Name: 12b8fa233697f9f4026eb14fb7454c180f61aa639b1834ccdc8dcfde4420d3a4 --- manifest | 19 +++--- manifest.uuid | 2 +- src/os_unix.c | 177 ++++++++++++++++++++++++++++-------------------- src/shell.c.in | 14 ++++ src/sqlite.h.in | 2 + 5 files changed, 133 insertions(+), 81 deletions(-) diff --git a/manifest b/manifest index 70c5fd08ab..05a5cdefa5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Abort\sthe\swindow\sfunction\stree\srewrite\searly\sfollowing\san\sOOM. -D 2019-11-23T16:34:40.792 +C Experimental\sfile-controls\sfor\scontrolling\sthe\suse\sof\sthe\s-shm\sfile. +D 2019-11-25T23:55:14.269 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -510,7 +510,7 @@ F src/os.c 669cc3839cc35d20f81faf0be1ab6d4581cea35e9d8f3a9d48a98d6571f7c285 F src/os.h 48388821692e87da174ea198bf96b1b2d9d83be5dfc908f673ee21fafbe0d432 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 3e0e519f27683083a465e948e056759a8340728c222b5c394a135e0c57c220bc +F src/os_unix.c 112995da72acac11ad335489acbbf7df097df5b496602e6ae5e08b9ed20d06ed F src/os_win.c 035a813cbd17f355bdcad7ab894af214a9c13a1db8aeac902365350b98cd45a7 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 30f20d2263d3717f41a0d9a40f7a3d0f48ce1cfab461b875c6187ead9d6ad1c7 @@ -527,8 +527,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 2f8fb48e61d0006031df27e53810b6767972526d768d3cc6888435dc350c4c7a F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c f403b7bd2304d4dfd5ad2614cc0ad3386a97af707922882bdabba4c14ce12975 -F src/shell.c.in 4a3a9e1c11847b1904f2b01d087af1c052f660902755abab457cab1756817ded -F src/sqlite.h.in 4fe42f27a7be44586bbd94f49f2b097ef8a1053c747d82f135456c7f5381c85a +F src/shell.c.in 6ecf8033fccb57b47a86e2bb87b1918f1aa5602f74bb72b1ae8c8ad1f44d2f27 +F src/sqlite.h.in d4fd966968f89426afa3924398896748a3cd82e07e0eef70e87b4aca5002b2c2 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 72af51aa4e912e14cd495fb6e7fac65f0940db80ed950d90911aff292cc47ce2 F src/sqliteInt.h 98bc9562acfc361e34182aa25b00e2c73095732ddd3ba4158f984b94f5601f96 @@ -1851,7 +1851,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 57070c68bbe15fc1d19a765182432e844c082909bdbc63b58fd86b96e2c521dd -R 5ec4b862f5e315c01dbee90c995a71d6 +P d66f95a51530259ab48f78c9f91acc38055caf338b6fee846b99a8c077466e95 +R 7963d80c0fc4f8462d1ea33f7ebfda99 +T *branch * reuse-shm +T *sym-reuse-shm * +T -sym-trunk * U drh -Z b069a61ef07a50420fc34a99d1dc8940 +Z 1fd8281bb0cf67f1ca29940be5022987 diff --git a/manifest.uuid b/manifest.uuid index 418e7ff21d..ca94c9acee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d66f95a51530259ab48f78c9f91acc38055caf338b6fee846b99a8c077466e95 \ No newline at end of file +12b8fa233697f9f4026eb14fb7454c180f61aa639b1834ccdc8dcfde4420d3a4 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 18f2afcb66..94e00f0a3f 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -271,6 +271,77 @@ struct unixFile { #endif }; + +/* +** Object used to represent an shared memory buffer. +** +** When multiple threads all reference the same wal-index, each thread +** has its own unixShm object, but they all point to a single instance +** of this unixShmNode object. In other words, each wal-index is opened +** only once per process. +** +** Each unixShmNode object is connected to a single unixInodeInfo object. +** We could coalesce this object into unixInodeInfo, but that would mean +** every open file that does not use shared memory (in other words, most +** open files) would have to carry around this extra information. So +** the unixInodeInfo object contains a pointer to this unixShmNode object +** and the unixShmNode object is created only when needed. +** +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** +** The following fields are read-only after the object is created: +** +** hShm +** zFilename +** +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. +*/ +struct unixShmNode { + unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ + char *zFilename; /* Name of the mmapped file */ + int hShm; /* Open file descriptor */ + int szRegion; /* Size of shared-memory regions */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + char **apRegion; /* Array of mapped shared-memory regions */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif +}; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** unixShm.pShmNode +** unixShm.id +** +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. +*/ +struct unixShm { + unixShmNode *pShmNode; /* The underlying unixShmNode object */ + unixShm *pNext; /* Next unixShm with the same unixShmNode */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ + u8 id; /* Id of this connection within its unixShmNode */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +}; + /* This variable holds the process id (pid) from when the xRandomness() ** method was called. If xOpen() is called from a different process id, ** indicating that a fork() has occurred, the PRNG will be reset. @@ -292,6 +363,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_REUSESHM 0x100 /* Reuse existing -shm file */ /* ** Include code that is common to all os_*.c files @@ -4042,6 +4114,33 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + case SQLITE_FCNTL_REUSE_SHM: { + /* Set the UNIXFILE_REUSESHM file. If this flag is set before + ** the database file is first read, and the database is in WAL-mode, + ** and this is the first connection to connect to the database, and + ** there exists a -shm file from some prior connection that terminated + ** asynchronous, then this connection will attempt to reuse the existing + ** -shm file. The default behavior (without this flag) is to scan + ** the entire -wal file and reconstruct the -shm file from scratch. + ** The default behavior is safer. But some embedded systems want the + ** option to live dangerously. */ + pFile->ctrlFlags |= UNIXFILE_REUSESHM; + return SQLITE_OK; + } + case SQLITE_FCNTL_SHM_HANDLE: { + /* If this database connection has a -shm file open, then set + ** *pArg to the file descriptor for that file. If the -shm file is + ** not open for any reason (not opened yet, not in WAL mode, uses + ** an in-memory -shm file, etc) then set *pArg to -1. */ + unixShm *pShm = pFile->pShm; + int h = -1; + if( pShm ){ + unixShmNode *pNode = pShm->pShmNode; + if( pNode ) h = pNode->hShm; + } + *(int*)pArg = h; + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -4210,76 +4309,6 @@ static int unixGetpagesize(void){ #ifndef SQLITE_OMIT_WAL -/* -** Object used to represent an shared memory buffer. -** -** When multiple threads all reference the same wal-index, each thread -** has its own unixShm object, but they all point to a single instance -** of this unixShmNode object. In other words, each wal-index is opened -** only once per process. -** -** Each unixShmNode object is connected to a single unixInodeInfo object. -** We could coalesce this object into unixInodeInfo, but that would mean -** every open file that does not use shared memory (in other words, most -** open files) would have to carry around this extra information. So -** the unixInodeInfo object contains a pointer to this unixShmNode object -** and the unixShmNode object is created only when needed. -** -** unixMutexHeld() must be true when creating or destroying -** this object or while reading or writing the following fields: -** -** nRef -** -** The following fields are read-only after the object is created: -** -** hShm -** zFilename -** -** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and -** unixMutexHeld() is true when reading or writing any other field -** in this structure. -*/ -struct unixShmNode { - unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ - sqlite3_mutex *pShmMutex; /* Mutex to access this object */ - char *zFilename; /* Name of the mmapped file */ - int hShm; /* Open file descriptor */ - int szRegion; /* Size of shared-memory regions */ - u16 nRegion; /* Size of array apRegion */ - u8 isReadonly; /* True if read-only */ - u8 isUnlocked; /* True if no DMS lock held */ - char **apRegion; /* Array of mapped shared-memory regions */ - int nRef; /* Number of unixShm objects pointing to this */ - unixShm *pFirst; /* All unixShm objects pointing to this */ -#ifdef SQLITE_DEBUG - u8 exclMask; /* Mask of exclusive locks held */ - u8 sharedMask; /* Mask of shared locks held */ - u8 nextShmId; /* Next available unixShm.id value */ -#endif -}; - -/* -** Structure used internally by this VFS to record the state of an -** open shared memory connection. -** -** The following fields are initialized when this object is created and -** are read-only thereafter: -** -** unixShm.pShmNode -** unixShm.id -** -** All other fields are read/write. The unixShm.pShmNode->pShmMutex must -** be held while accessing any read/write fields. -*/ -struct unixShm { - unixShmNode *pShmNode; /* The underlying unixShmNode object */ - unixShm *pNext; /* Next unixShm with the same unixShmNode */ - u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ - u8 id; /* Id of this connection within its unixShmNode */ - u16 sharedMask; /* Mask of shared locks held */ - u16 exclMask; /* Mask of exclusive locks held */ -}; - /* ** Constants used for locking */ @@ -4457,8 +4486,12 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ ** -shm header size) rather than 0 as a system debugging aid, to ** help detect if a -shm file truncation is legitimate or is the work ** or a rogue process. */ - if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + if( rc==SQLITE_OK + && (pDbFd->ctrlFlags & UNIXFILE_REUSESHM)==0 + && robust_ftruncate(pShmNode->hShm, 3) + ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate", + pShmNode->zFilename); } } }else if( lock.l_type==F_WRLCK ){ diff --git a/src/shell.c.in b/src/shell.c.in index e6b6f1a1b3..165be73b7e 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7386,6 +7386,8 @@ static int do_meta_command(char *zLine, ShellState *p){ { "tempfilename", SQLITE_FCNTL_TEMPFILENAME, "" }, { "has_moved", SQLITE_FCNTL_HAS_MOVED, "" }, { "lock_timeout", SQLITE_FCNTL_LOCK_TIMEOUT, "MILLISEC" }, + { "reuse_shm", SQLITE_FCNTL_REUSE_SHM, "" }, + { "shm_handle", SQLITE_FCNTL_SHM_HANDLE, "" }, }; int filectrl = -1; int iCtrl = -1; @@ -7480,6 +7482,18 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 2; break; } + case SQLITE_FCNTL_REUSE_SHM: { + sqlite3_file_control(p->db, 0, filectrl, 0); + isOk = 2; + break; + } + case SQLITE_FCNTL_SHM_HANDLE: { + int h = -2; + sqlite3_file_control(p->db, 0, filectrl, &h); + isOk = 1; + iRes = h; + break; + } } } if( isOk==0 && iCtrl>=0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 53fb22651c..19980979ad 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1147,6 +1147,8 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_LOCK_TIMEOUT 34 #define SQLITE_FCNTL_DATA_VERSION 35 #define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_REUSE_SHM 37 +#define SQLITE_FCNTL_SHM_HANDLE 38 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE -- 2.47.2