From: dan Date: Fri, 16 Jul 2021 18:30:27 +0000 (+0000) Subject: Experiment with having SQLITE_ENABLE_SETLK_TIMEOUT builds on unix use a condition... X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=740fe883b45b7c9c31d238b7eeaa2e5fa79eb778;p=thirdparty%2Fsqlite.git Experiment with having SQLITE_ENABLE_SETLK_TIMEOUT builds on unix use a condition variable to wait for wal locks held by other threads. FossilOrigin-Name: 4a9f5ce79d349a5a352c4a5524ace82611ab1058a3e6cb012ee7d0071f9c8cd2 --- diff --git a/manifest b/manifest index 8d6d4c6f24..9c6cab4b9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\suse-after-free\serror\sin\sioerr.test\scaused\sby\san\serror\sin\stest\scode. -D 2021-07-14T21:18:31.520 +C Experiment\swith\shaving\sSQLITE_ENABLE_SETLK_TIMEOUT\sbuilds\son\sunix\suse\sa\scondition\svariable\sto\swait\sfor\swal\slocks\sheld\sby\sother\sthreads. +D 2021-07-16T18:30:27.066 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -528,7 +528,7 @@ F src/os.c 59ed1f503347e8b5434c0ce7d7d0f02a3f24a72fea8b26d0bba2de8dfaef778b F src/os.h 26890f540b475598cd9881dcc68931377b8d429d3ea3e2eeb64470cde64199f8 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c b11e4610769922253dec27d7af4a07ff84f65169d19bda5e9b12a152a706f7f5 +F src/os_unix.c aea96704db0860b61ad95272bd9cb3fc3a4910e6d68506cc6c48d5d8f7922d0b F src/os_win.c 77d39873836f1831a9b0b91894fec45ab0e9ca8e067dc8c549e1d1eca1566fe9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a F src/pager.c 95c255256b13827caf038c8f963d334784073f38ab6ef9d70371d9d04f3c43e0 @@ -1465,7 +1465,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c a70a8e94bef23339d34226eb9521015ef99f4df8 -F test/threadtest3.c e63013af10cf236c7610eb06d33bde08c861806dc64be811940ff4d9ddd34a4f +F test/threadtest3.c f39994e150bc2c5b61e041905249b49623ada3223ed772454343b22f96ef67ab F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/threadtest5.c 9b4d782c58d8915d7e955ff8051f3d03628bda0d33b82971ea8c0f2f2808c421 F test/time-wordcount.sh 8e0b0f8109367827ad5d58f5cc849705731e4b90 @@ -1920,7 +1920,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c378e99250fe06fae8ca40c62185b607f004d6806e07dbb9f964dd849b4e55f8 -R 5dc9e7510bb681a9844c30d96bdfacf1 +P 1594056aab068b9fae82f6b885828c7127d9bedcc37c4340486e32791bc87c7a +R fc7634cfdafbf55388fafa77867b7fbb +T *branch * unix-timed-wait-exp +T *sym-unix-timed-wait-exp * +T -sym-trunk * U dan -Z ab55d4571a2b3927d12f63db9cd8ac86 +Z 91d9b974e91effdc9a593e4e66c579de diff --git a/manifest.uuid b/manifest.uuid index ca45e83989..a04fee8202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1594056aab068b9fae82f6b885828c7127d9bedcc37c4340486e32791bc87c7a \ No newline at end of file +4a9f5ce79d349a5a352c4a5524ace82611ab1058a3e6cb012ee7d0071f9c8cd2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index aa6b3b8e4c..b9dfad0c73 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4271,7 +4271,12 @@ static int unixGetpagesize(void){ */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pthread_cond_t shmcond; + pthread_mutex_t shmmutex; +#else sqlite3_mutex *pShmMutex; /* Mutex to access this object */ +#endif char *zFilename; /* Name of the mmapped file */ int hShm; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ @@ -4289,6 +4294,15 @@ struct unixShmNode { #endif }; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +# define ENTER_SHMNODE_MUTEX(p) pthread_mutex_lock(&p->shmmutex) +# define LEAVE_SHMNODE_MUTEX(p) pthread_mutex_unlock(&p->shmmutex) +#else +# define ENTER_SHMNODE_MUTEX(p) sqlite3_mutex_enter(p->pShmMutex) +# define LEAVE_SHMNODE_MUTEX(p) sqlite3_mutex_leave(p->pShmMutex) +#endif + + /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. @@ -4338,13 +4352,13 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ f.l_start = UNIX_SHM_BASE + 3; f.l_len = SQLITE_SHM_NLOCK - 3; - sqlite3_mutex_enter(pShmNode->pShmMutex); + ENTER_SHMNODE_MUTEX(pShmNode); if( osFcntl(pShmNode->hShm, F_GETLK, &f)<0 ){ rc = SQLITE_IOERR_LOCK; }else{ *piOut = (f.l_type!=F_UNLCK); } - sqlite3_mutex_leave(pShmNode->pShmMutex); + LEAVE_SHMNODE_MUTEX(pShmNode); } return rc; @@ -4358,7 +4372,7 @@ static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ ** otherwise. */ static int unixShmSystemLock( - unixFile *pFile, /* Open connection to the WAL file */ + unixFile *pFile, /* Open connection to the database file */ int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ int ofst, /* First byte of the locking range */ int n /* Number of bytes to lock */ @@ -4369,7 +4383,9 @@ static int unixShmSystemLock( /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); +#endif assert( pShmNode->nRef>0 || unixMutexHeld() ); /* Shared locks never span more than one byte */ @@ -4463,7 +4479,12 @@ static void unixShmPurge(unixFile *pFd){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pthread_cond_destroy(&p->shmcond); + pthread_mutex_destroy(&p->shmmutex); +#else sqlite3_mutex_free(p->pShmMutex); +#endif for(i=0; inRegion; i+=nShmPerMap){ if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); @@ -4639,11 +4660,16 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; if( sqlite3GlobalConfig.bCoreMutex ){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pthread_cond_init(&pShmNode->shmcond, 0); + pthread_mutex_init(&pShmNode->shmmutex, 0); +#else pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->pShmMutex==0 ){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } +#endif } if( pInode->bProcessLock==0 ){ @@ -4688,10 +4714,10 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** at pShmNode->pFirst. This must be done while holding the ** pShmNode->pShmMutex. */ - sqlite3_mutex_enter(pShmNode->pShmMutex); + ENTER_SHMNODE_MUTEX(pShmNode); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->pShmMutex); + LEAVE_SHMNODE_MUTEX(pShmNode); return rc; /* Jump here on any error */ @@ -4743,7 +4769,7 @@ static int unixShmMap( p = pDbFd->pShm; pShmNode = p->pShmNode; - sqlite3_mutex_enter(pShmNode->pShmMutex); + ENTER_SHMNODE_MUTEX(pShmNode); if( pShmNode->isUnlocked ){ rc = unixLockSharedMemory(pDbFd, pShmNode); if( rc!=SQLITE_OK ) goto shmpage_out; @@ -4852,7 +4878,7 @@ shmpage_out: *pp = 0; } if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; - sqlite3_mutex_leave(pShmNode->pShmMutex); + LEAVE_SHMNODE_MUTEX(pShmNode); return rc; } @@ -4867,7 +4893,9 @@ shmpage_out: static int assertLockingArrayOk(unixShmNode *pShmNode){ unixShm *pX; int aLock[SQLITE_SHM_NLOCK]; +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); +#endif memset(aLock, 0, sizeof(aLock)); for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ @@ -4909,6 +4937,11 @@ static int unixShmLock( u16 mask; /* Mask of locks to take or release */ int *aLock = pShmNode->aLock; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int bRetry; + struct timespec sTimespec; +#endif + assert( pShmNode==pDbFd->pInode->pShmNode ); assert( pShmNode->pInode==pDbFd->pInode ); assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); @@ -4941,82 +4974,128 @@ static int unixShmLock( && (ofst!=0 || (p->exclMask|p->sharedMask)<3) && (ofst<3 || (p->exclMask|p->sharedMask)<(1<iBusyTimeout ){ + struct timeval tm; + memset(&sTimespec, 0, sizeof(sTimespec)); + gettimeofday(&tm, 0); + TIMEVAL_TO_TIMESPEC(&tm, &sTimespec); + sTimespec.tv_sec += pDbFd->iBusyTimeout / 1000; + sTimespec.tv_nsec += (pDbFd->iBusyTimeout % 1000) * 1000000; + if( sTimespec.tv_nsec>(1000*1000000) ){ + sTimespec.tv_sec++; + sTimespec.tv_nsec -= (1000*1000000); + } + } #endif mask = (1<<(ofst+n)) - (1<1 || mask==(1<pShmMutex); - assert( assertLockingArrayOk(pShmNode) ); - if( flags & SQLITE_SHM_UNLOCK ){ - if( (p->exclMask|p->sharedMask) & mask ){ - int ii; - int bUnlock = 1; + ENTER_SHMNODE_MUTEX(pShmNode); - for(ii=ofst; ii((p->sharedMask & (1<exclMask|p->sharedMask) & mask ){ + int ii; + int bUnlock = 1; + + for(ii=ofst; ii((p->sharedMask & (1<sharedMask & (1<1 ); + aLock[ofst]--; } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pthread_cond_broadcast(&pShmNode->shmcond); +#endif + } } - - if( bUnlock ){ - rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + }else if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( (p->exclMask & (1<sharedMask & mask)==0 ){ + if( aLock[ofst]<0 ){ + rc = SQLITE_BUSY; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + bRetry = (pDbFd->iBusyTimeout!=0); +#endif + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + } + + /* Get the local shared locks */ if( rc==SQLITE_OK ){ - memset(&aLock[ofst], 0, sizeof(int)*n); + p->sharedMask |= mask; + aLock[ofst]++; } - }else if( ALWAYS(p->sharedMask & (1<1 ); - aLock[ofst]--; } - - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - } - }else if( flags & SQLITE_SHM_SHARED ){ - assert( n==1 ); - assert( (p->exclMask & (1<sharedMask & mask)==0 ){ - if( aLock[ofst]<0 ){ - rc = SQLITE_BUSY; - }else if( aLock[ofst]==0 ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ + int ii; + for(ii=ofst; iisharedMask & mask)==0 ); + if( ALWAYS((p->exclMask & (1<iBusyTimeout!=0); +#endif + break; + } } - - /* Get the local shared locks */ + + /* Get the exclusive locks at the system level. Then if successful + ** also update the in-memory values. */ if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - aLock[ofst]++; - } - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. */ - int ii; - for(ii=ofst; iisharedMask & mask)==0 ); - if( ALWAYS((p->exclMask & (1<sharedMask & mask)==0 ); + p->exclMask |= mask; + for(ii=ofst; iisharedMask & mask)==0 ); - p->exclMask |= mask; - for(ii=ofst; iishmcond, &pShmNode->shmmutex, &sTimespec + ); + if( prc==0 ){ + rc = SQLITE_OK; + }else{ + /* printf("prc=%d (%s)\n", prc, strerror(prc)); */ + bRetry = 0; } } - } + }while( bRetry ); +#endif + assert( assertLockingArrayOk(pShmNode) ); - sqlite3_mutex_leave(pShmNode->pShmMutex); + LEAVE_SHMNODE_MUTEX(pShmNode); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; @@ -5066,14 +5145,14 @@ static int unixShmUnmap( /* Remove connection p from the set of connections associated ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->pShmMutex); + ENTER_SHMNODE_MUTEX(pShmNode); for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} *pp = p->pNext; /* Free the connection p */ sqlite3_free(p); pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->pShmMutex); + LEAVE_SHMNODE_MUTEX(pShmNode); /* If pShmNode->nRef has reached 0, then close the underlying ** shared-memory file, too */ diff --git a/test/threadtest3.c b/test/threadtest3.c index 41c0fb9ac6..a6571f3e31 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1426,6 +1426,93 @@ static void dynamic_triggers(int nMs){ print_and_free_err(&err); } +typedef struct Walthread6 Walthread6; +struct Walthread6 { + int nInsert; + int nBusy; + int nMaxFrame; +}; + +static int walthread6_walhook( + void *pArg, /* Pointer to Walthread6 structure */ + sqlite3 *db, /* Database handle */ + const char *zDb, /* "main" */ + int nFrame /* Frames current in wal file */ +){ + int rc = SQLITE_OK; + Walthread6 *p = (Walthread6*)pArg; + + if( nFrame>p->nMaxFrame ) p->nMaxFrame = nFrame; + if( nFrame>1000 ){ + sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_RESTART, 0, 0); + } + + return rc; +} + +static char *walthread6_thread(int iTid, void *pArg){ + Error err = {0}; + Sqlite db = {0}; + + Walthread6 res; + memset(&res, 0, sizeof(res)); + + opendb(&err, &db, "test.db", 0); + sqlite3_busy_timeout(db.db, 1000); + sqlite3_wal_hook(db.db, walthread6_walhook, (void*)&res); + + while( !timetostop(&err) ){ + int i; + execsql(&err, &db, "BEGIN IMMEDIATE"); + if( err.rc==SQLITE_BUSY ){ + res.nBusy++; + clear_error(&err, SQLITE_BUSY); + }else{ + res.nInsert++; + for(i=0; i<20; i++){ + execsql(&err, &db, "INSERT INTO t1(b) VALUES(random())"); + } + usleep(10*1000); + execsql(&err, &db, "COMMIT"); + } + } + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf( + "%d transactions (%d busy), max-wal-size=%d frames", + res.nInsert, res.nBusy, res.nMaxFrame + ); +} + +/* +** +*/ +static void walthread6(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + int i; + int nThread = 2; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "PRAGMA page_size = 1024;" + "PRAGMA journal_mode = WAL;" + "CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT);" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + for(i=0; i