From: dan Date: Tue, 27 Jun 2017 20:23:07 +0000 (+0000) Subject: Support clients within a single process only. X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=6f3283a427a3ada1bd0dd205dae141ddf98bc396;p=thirdparty%2Fsqlite.git Support clients within a single process only. FossilOrigin-Name: dfa9a4d53e0aa7f7911b11af57a4d37a1df2d835cf60eb9e76d28b6b39c2ba58 --- diff --git a/manifest b/manifest index e6d8b502de..5234b14a43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\stest\stool\s"tserver"\sfinalizes\sall\sstatements\sbefore\sattempting\sto\nclose\sa\sdatabase\shandle. -D 2017-06-20T19:20:07.502 +C Support\sclients\swithin\sa\ssingle\sprocess\sonly. +D 2017-06-27T20:23:07.577 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -386,11 +386,11 @@ F src/os.c add02933b1dce7a39a005b00a2f5364b763e9a24 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 -F src/os_unix.c 30e2c43e4955db990e5b5a81e901f8aa74cc8820 +F src/os_unix.c d3c3a186ceb27d363f7a5e281ece4eb32b56e173a331e153a761d280034e3212 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c f957574dc2f8a66d8ee2b5fd978ecd98760d6669182a023a3901e98ab5e4df33 -F src/pager.h ea05992c581c3366279fb1d436944604b4be17208ebb41fa407306e5e4b34205 +F src/pager.c 3f9c94c6906c624c4db730523f4e91045ffb7bdfa48ead118025c6ae3e176cea +F src/pager.h 81d2c28e92e543cb1ed977f44810d04c34be2a23d7782bb02548d86a4b9ae657 F src/parse.y 0513387ce02fea97897d8caef82d45f347818593f24f1bdc48e0c530a8af122d F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 @@ -403,10 +403,10 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 4f0adefaa5e9417459b07757e0f6060cac97930a86f0fba9797bab233ced66c0 -F src/server.c 63c3799a184cba3d3b9f102f10e815104ba8d1d4a01c70b0e709e8173b5f9ae7 -F src/server.h e1ce2da1e4d21f335904539e3f98a7c24e015e1201b4bb16d61f0044b8bd2884 +F src/server.c 9f1ce178587ae672058923f47de628d97490d3089fae8b9d6e3e6d6c5148320f +F src/server.h 3796b9dcb61c60c4e49e9f579832ca1b4e63c6493dae5314b2970540a7cd22f8 F src/shell.c e5950029da103c5d378e71d548759459b9a7fc76177a71562c22082c705745ab -F src/sqlite.h.in 8d126e4cfbd1f4bc6f4043aacd77f78b45613e7d630185d49a5d099394247483 +F src/sqlite.h.in 61cac6e788c961bbaf2f91f55d63a8175809bddb7b4971a894b588321c1a82a6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 58fd0676d3111d02e62e5a35992a7d3da5d3f88753acc174f2d37b774fbbdd28 F src/sqliteInt.h 50a6bec0b83965edb0a805e1e0f3ebeab536084109a97bdd18a524eb42579a4e @@ -1036,7 +1036,7 @@ F test/parser1.test 391b9bf9a229547a129c61ac345ed1a6f5eb1854 F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test ca1840b39571fc87f432c104e6df00279ab67edeec0f75d8015dcfea648fee2e +F test/permutations.test 676872085bdaa9a4c789bffd2f1ebbebc22262773bf695e05d19d0085de308f4 F test/pragma.test 1e94755164a3a3264cd39836de4bebcb7809e5f8 F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f F test/pragma3.test 14c12bc5352b1e100e0b6b44f371053a81ccf8ed @@ -1112,8 +1112,8 @@ F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/selectG.test e8600e379589e85e9fefd2fe4d44a4cdd63f6982 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 -F test/server2.test 11dda300ebef43b4abe0fdc086b6f51b964beddb529fb65bc1f026db5895c36e -F test/server3.test c33343f2f6bc23f2b4e2f047c3d083579f0cfac2795e0f1eb226ab34758967c0 +F test/server2.test 277b4b1f142232e1f992ea17b587474c32e41e2f321c01acdd49ef5ffee99ee6 +F test/server3.test 3667fea050a248508d46b5eccb8a5561055f802016dc0d701e6b6f035ecc55ae F test/servercrash.test 816c132b26af008067cab2913783f67006d4003e3988f3f3ee1075742f6e0a6c F test/serverwal.test 2d066ba70db35e28f0115a57385cf606513c56d6e2861119c2defb7471e4e2d9 F test/session.test 78fa2365e93d3663a6e933f86e7afc395adf18be @@ -1585,7 +1585,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05b4fc4340011847c8585bb822d339dd7b8351266a12b26fdc85edce38fc9dd3 -R 5abfa2b3bda91a8770a53a2957396fde +P d8568aacf083b63cdcf68d744ddb07a7615d14f169be6c632813a266a1e8a1cf +R f783e0707b8d4ba7e92792e2decf35e1 +T *branch * server-process-edition +T *sym-server-process-edition * +T -sym-server-edition * U dan -Z b4624fe82f09b9b3eec8967be30284e2 +Z c2e6e9d0d91d26e5c223f5dd53e51df2 diff --git a/manifest.uuid b/manifest.uuid index 719bb96e65..ffbe74199f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8568aacf083b63cdcf68d744ddb07a7615d14f169be6c632813a266a1e8a1cf \ No newline at end of file +dfa9a4d53e0aa7f7911b11af57a4d37a1df2d835cf60eb9e76d28b6b39c2ba58 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7f0ebdba6f..472ab08f78 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3777,6 +3777,12 @@ static int unixGetTempname(int nBuf, char *zBuf); static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ + case SQLITE_FCNTL_FILEID: { + i64 *aId = (i64)pArg; + aId[0] = (i64)(pFile->pInode->fileId.dev); + aId[1] = (i64)(pFile->pInode->fileId.ino); + return SQLITE_OK; + } case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; diff --git a/src/pager.c b/src/pager.c index 46650c7ed6..297cdeadc2 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4116,6 +4116,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ ** rollback before accessing the database file. */ if( isOpen(pPager->jfd) ){ +#if 0 if( pagerIsServer(pPager) ){ assert( pPager->journalMode==PAGER_JOURNALMODE_PERSIST ); pPager->journalMode = PAGER_JOURNALMODE_DELETE; @@ -4123,6 +4124,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ ** is deleted by the call to pagerUnlockAndRollback() below. */ if( pPager->eState==PAGER_OPEN ) pPager->eState = PAGER_READER; } +#endif pager_error(pPager, pagerSyncHotJournal(pPager)); } pagerUnlockAndRollback(pPager); @@ -4131,14 +4133,14 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ if( pagerIsServer(pPager) ){ sqlite3ServerDisconnect(pPager->pServer, pPager->fd); pPager->pServer = 0; - sqlite3_free(pPager->zJournal); + }else{ + sqlite3OsClose(pPager->jfd); } #endif sqlite3EndBenignMalloc(); enable_simulated_io_errors(); PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); - IOTRACE(("CLOSE %p\n", pPager)) - sqlite3OsClose(pPager->jfd); + IOTRACE(("CLOSE %p\n", pPager)); sqlite3OsClose(pPager->fd); sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); @@ -4148,7 +4150,7 @@ int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ #endif assert( !pPager->aSavepoint && !pPager->pInJournal ); - assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + assert( !isOpen(pPager->sjfd) ); sqlite3_free(pPager); return SQLITE_OK; @@ -5090,71 +5092,47 @@ static int hasHotJournal(Pager *pPager, int *pExists){ static int pagerServerConnect(Pager *pPager){ int rc = SQLITE_OK; if( pPager->tempFile==0 ){ - int iClient = 0; pPager->noLock = 1; pPager->journalMode = PAGER_JOURNALMODE_PERSIST; - rc = sqlite3ServerConnect(pPager, &pPager->pServer, &iClient); - if( rc==SQLITE_OK ){ - pPager->zJournal = sqlite3_mprintf( - "%s-journal%d", pPager->zFilename, iClient - ); - if( pPager->zJournal==0 ){ - rc = SQLITE_NOMEM_BKPT; - } - } + rc = sqlite3ServerConnect(pPager, &pPager->pServer); } return rc; } -int sqlite3PagerRollbackJournal(Pager *pPager, int iClient){ - int rc; - char *zJrnl = sqlite3_mprintf("%s-journal%d", pPager->zFilename, iClient); +int sqlite3PagerRollbackJournal(Pager *pPager, sqlite3_file *pJfd){ + int rc; /* Return Code */ + sqlite3_file *saved_jfd = pPager->jfd; + u8 saved_eState = pPager->eState; + u8 saved_eLock = pPager->eLock; + i64 saved_journalOff = pPager->journalOff; + i64 saved_journalHdr = pPager->journalHdr; - if( zJrnl ){ - int bExists = 0; - sqlite3_file *jfd = 0; - sqlite3_vfs * const pVfs = pPager->pVfs; + assert( pPager->journalMode==PAGER_JOURNALMODE_PERSIST ); - rc = sqlite3OsAccess(pVfs, zJrnl, SQLITE_ACCESS_EXISTS, &bExists); - if( rc==SQLITE_OK && bExists ){ - int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - rc = sqlite3OsOpenMalloc(pVfs, zJrnl, &jfd, flags, &flags); - } - assert( rc==SQLITE_OK || jfd==0 ); - if( jfd ){ - sqlite3_file *saved_jfd = pPager->jfd; - u8 saved_eState = pPager->eState; - u8 saved_eLock = pPager->eLock; - i64 saved_journalOff = pPager->journalOff; - i64 saved_journalHdr = pPager->journalHdr; - char *saved_zJournal = pPager->zJournal; - - pPager->eLock = EXCLUSIVE_LOCK; - pPager->eState = PAGER_WRITER_DBMOD; - pPager->jfd = jfd; - rc = pagerSyncHotJournal(pPager); - if( rc==SQLITE_OK ) rc = pager_playback(pPager, 1); - - pPager->jfd = saved_jfd; - pPager->eState = saved_eState; - pPager->eLock = saved_eLock; - pPager->journalOff = saved_journalOff; - pPager->journalHdr = saved_journalHdr; - pPager->zJournal = saved_zJournal; - - sqlite3OsCloseFree(jfd); - if( rc==SQLITE_OK ){ - rc = sqlite3OsDelete(pVfs, zJrnl, 0); - } - } - sqlite3_free(zJrnl); - }else{ - rc = SQLITE_NOMEM_BKPT; - } + pPager->eLock = EXCLUSIVE_LOCK; + pPager->eState = PAGER_WRITER_DBMOD; + pPager->jfd = pJfd; + rc = pagerSyncHotJournal(pPager); + if( rc==SQLITE_OK ) rc = pager_playback(pPager, 1); + assert( isOpen(pPager->jfd) ); + pPager->jfd = saved_jfd; + pPager->eState = saved_eState; + pPager->eLock = saved_eLock; + pPager->journalOff = saved_journalOff; + pPager->journalHdr = saved_journalHdr; return rc; } +void sqlite3PagerServerJournal( + Pager *pPager, + sqlite3_file *jfd, + const char *zJournal +){ + pPager->zJournal = zJournal; + pPager->jfd = jfd; +} + #else # define pagerServerConnect(pPager) SQLITE_OK #endif @@ -5389,8 +5367,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ #ifdef SQLITE_SERVER_EDITION if( pagerIsServer(pPager) ){ assert( rc==SQLITE_OK ); + assert( sqlite3PagerRefcount(pPager)==0 ); pager_reset(pPager); rc = sqlite3ServerBegin(pPager->pServer); + if( rc==SQLITE_OK){ + rc = sqlite3ServerLock(pPager->pServer, 1, 0, 0); + } } #endif if( rc==SQLITE_OK && pagerUseWal(pPager) ){ diff --git a/src/pager.h b/src/pager.h index 8d9d25d1b2..ff0038459b 100644 --- a/src/pager.h +++ b/src/pager.h @@ -237,9 +237,10 @@ void *sqlite3PagerCodec(DbPage *); #endif #ifdef SQLITE_SERVER_EDITION - int sqlite3PagerRollbackJournal(Pager*, int); + int sqlite3PagerRollbackJournal(Pager*, sqlite3_file*); int sqlite3PagerIsServer(Pager *pPager); int sqlite3PagerPagelock(Pager *pPager, Pgno, int); + void sqlite3PagerServerJournal(Pager*, sqlite3_file*, const char*); #endif #endif /* SQLITE_PAGER_H */ diff --git a/src/server.c b/src/server.c index 2bc46071ca..c265de5998 100644 --- a/src/server.c +++ b/src/server.c @@ -14,50 +14,37 @@ #include "sqliteInt.h" /* -** HMA file layout: -** -** 4 bytes - DMS slot. All connections read-lock this slot. -** -** 16*4 bytes - locking slots. Connections hold a read-lock on a locking slot -** when they are connected, a write lock when they have an open -** transaction. -** -** N*4 bytes - Page locking slots. N is HMA_PAGELOCK_SLOTS. -** ** Page-locking slot format: ** -** Each page-locking slot provides SHARED/RESERVED/EXCLUSIVE locks on a -** single page. A RESERVED lock is similar to a RESERVED in SQLite's -** rollback mode - existing SHARED locks may continue but new SHARED locks -** may not be established. As in rollback mode, EXCLUSIVE and RESERVED -** locks are mutually exclusive. -** -** Each 32-bit locking slot is divided into two sections - a bitmask for -** read-locks and a single integer field for the write lock. The bitmask -** occupies the least-significant 27 bits of the slot. The integer field -** occupies the remaining 5 bits (so that it can store values from 0-31). +** Assuming HMA_MAX_TRANSACTIONID is set to 16. ** -** Each client has a unique integer client id. Currently these range from -** 0-15 (maximum of 16 concurrent connections). The page-locking slot format -** allows this to be increased to 0-26 (maximum of 26 connections). To -** take a SHARED lock, the corresponding bit is set in the locking slot -** bitmask: +** The least-significant 16 bits are used for read locks. When a read +** lock is taken, the client sets the bit associated with its +** transaction-id. ** -** slot = slot | (1 << iClient); +** The next 8 bits are set to the number of transient-read locks +** currently held on the page. ** -** To take an EXCLUSIVE or RESERVED lock, the integer part of the locking -** slot is set to the client-id of the locker plus one (a value of zero -** indicates that no connection holds a RESERVED or EXCLUSIVE lock): -** -** slot = slot | ((iClient+1) << 27) +** The next 5 bits are set to 0 if no client currently holds a write +** lock. Or to (transaction-id + 1) if a write lock is held. */ #ifdef SQLITE_SERVER_EDITION -#define HMA_CLIENT_SLOTS 16 +/* Number of page-locking slots */ #define HMA_PAGELOCK_SLOTS (256*1024) -#define HMA_FILE_SIZE (4 + 4*HMA_CLIENT_SLOTS + 4*HMA_PAGELOCK_SLOTS) +/* Maximum concurrent read/write transactions */ +#define HMA_MAX_TRANSACTIONID 16 + +/* +** The argument to this macro is the value of a locking slot. It returns +** -1 if no client currently holds the write lock, or the transaction-id +** of the locker otherwise. +*/ +#define slotGetWriter(v) (((int)((v) >> HMA_MAX_TRANSACTIONID) & 0x1f) -1) + +#define slotReaderMask(v) ((v) & ((1 << HMA_MAX_TRANSACTIONID)-1)) #include "unistd.h" #include "fcntl.h" @@ -66,35 +53,45 @@ #include "sys/stat.h" #include "errno.h" -typedef struct ServerHMA ServerHMA; +typedef struct ServerDb ServerDb; +typedef struct ServerJournal ServerJournal; struct ServerGlobal { - ServerHMA *pHma; /* Linked list of all ServerHMA objects */ + ServerDb *pDb; /* Linked list of all ServerHMA objects */ }; static struct ServerGlobal g_server; +struct ServerJournal { + char *zJournal; + sqlite3_file *jfd; +}; + /* ** There is one instance of the following structure for each distinct -** HMA file opened by clients within this process. +** database file opened in server mode by this process. */ -struct ServerHMA { - char *zName; /* hma file path */ - int fd; /* Fd open on hma file */ - int nClient; /* Current number of clients */ - Server *aClient[HMA_CLIENT_SLOTS]; /* Local (this process) clients */ - u32 *aMap; /* MMapped hma file */ - ServerHMA *pNext; /* Next HMA in this process */ - - dev_t st_dev; - ino_t st_ino; +struct ServerDb { + sqlite3_mutex *mutex; /* Non-recursive mutex */ + int nClient; /* Current number of clients */ + int bInit; /* True once initialized */ + u32 transmask; /* Bitmask of taken transaction ids */ + u32 *aSlot; /* Array of page locking slots */ + i64 aFileId[2]; /* Opaque VFS file-id */ + ServerDb *pNext; /* Next db in this process */ + + sqlite3_vfs *pVfs; + ServerJournal aJrnl[HMA_MAX_TRANSACTIONID]; + u8 *aJrnlFdSpace; }; +/* +** Once instance for each client connection open on a server mode database +** in this process. +*/ struct Server { - ServerHMA *pHma; /* Hma file object */ - int iClient; /* Client id */ + ServerDb *pDb; /* Database object */ Pager *pPager; /* Associated pager object */ - i64 nUsWrite; /* Cumulative us holding WRITER lock */ - i64 iUsWrite; /* Time WRITER lock was taken */ + int iTransId; /* Current transaction id (or -1) */ int nAlloc; /* Allocated size of aLock[] array */ int nLock; /* Number of entries in aLock[] */ u32 *aLock; /* Mapped lock file */ @@ -117,379 +114,261 @@ static void serverAssertMutexHeld(void){ assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_APP1)) ); } -static int posixLock(int fd, int iSlot, int eLock, int bBlock){ - int res; - struct flock l; - short aType[4] = {0, F_UNLCK, F_RDLCK, F_WRLCK}; - assert( eLock==SERVER_WRITE_LOCK - || eLock==SERVER_READ_LOCK - || eLock==SERVER_NO_LOCK - ); - memset(&l, 0, sizeof(l)); - l.l_type = aType[eLock]; - l.l_whence = SEEK_SET; - l.l_start = iSlot*sizeof(u32); - l.l_len = 1; - - res = fcntl(fd, (bBlock ? F_SETLKW : F_SETLK), &l); - if( res && bBlock && errno==EDEADLK ){ - return SQLITE_BUSY_DEADLOCK; +static int serverFindDatabase(Server *pNew, i64 *aFileId){ + ServerDb *p; + int rc = SQLITE_OK; + serverEnterMutex(); + for(p=g_server.pDb; p; p=p->pNext){ + if( p->aFileId[0]==aFileId[0] && p->aFileId[1]==aFileId[1] ){ + break; + } } - return (res==0 ? SQLITE_OK : SQLITE_BUSY); -} + if( p==0 ){ + p = (ServerDb*)sqlite3MallocZero(sizeof(ServerDb)); + if( p ){ + p->aSlot = (u32*)sqlite3MallocZero(sizeof(u32)*HMA_PAGELOCK_SLOTS); + if( p->aSlot==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + p->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); +#if SQLITE_THREADSAFE!=0 + if( p->mutex==0 ) rc = SQLITE_NOMEM_BKPT; +#endif + } -static int serverMapFile(ServerHMA *p){ - assert( p->aMap==0 ); - p->aMap = mmap(0, HMA_FILE_SIZE, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd, 0); - if( p->aMap==0 ){ - return SQLITE_ERROR; + if( rc==SQLITE_NOMEM ){ + sqlite3_free(p->aSlot); + sqlite3_free(p); + p = 0; + }else{ + p->nClient = 1; + p->aFileId[0] = aFileId[0]; + p->aFileId[1] = aFileId[1]; + p->pNext = g_server.pDb; + g_server.pDb = p; + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + }else{ + p->nClient++; } - return SQLITE_OK; + pNew->pDb = p; + serverLeaveMutex(); + return rc; } +/* +** Free all resources allocated by serverInitDatabase() associated with the +** object passed as the only argument. +*/ +static void serverShutdownDatabase(ServerDb *pDb){ + int i; -static void serverDecrHmaRefcount(ServerHMA *pHma){ - if( pHma ){ - pHma->nClient--; - if( pHma->nClient<=0 ){ - ServerHMA **pp; - if( pHma->aMap ) munmap(pHma->aMap, HMA_FILE_SIZE); - if( pHma->fd>=0 ) close(pHma->fd); - for(pp=&g_server.pHma; *pp!=pHma; pp=&(*pp)->pNext); - *pp = pHma->pNext; - sqlite3_free(pHma); + for(i=0; iaJrnl[i]; + if( pJ->jfd ){ + sqlite3OsClose(pJ->jfd); + sqlite3OsDelete(pDb->pVfs, pJ->zJournal, 0); } + sqlite3_free(pJ->zJournal); } -} + memset(pDb->aJrnl, 0, sizeof(ServerJournal)*HMA_MAX_TRANSACTIONID); + if( pDb->aJrnlFdSpace ){ + sqlite3_free(pDb->aJrnlFdSpace); + pDb->aJrnlFdSpace = 0; + } -static int serverOpenHma(Pager *pPager, const char *zPath, ServerHMA **ppHma){ - struct stat sStat; /* Structure populated by stat() */ - int res; /* result of stat() */ - int rc = SQLITE_OK; /* Return code */ - ServerHMA *pHma = 0; - - serverAssertMutexHeld(); + sqlite3_free(pDb->aSlot); + pDb->bInit = 0; +} - res = stat(zPath, &sStat); - if( res!=0 ){ - sqlite3_log(SQLITE_CANTOPEN, "Failed to stat(%s)", zPath); - rc = SQLITE_ERROR; +/* +** This function is called when the very first connection to a database +** is established. It is responsible for rolling back any hot journal +** files found in the file-system. +*/ +static int serverInitDatabase(Server *pNew){ + int nByte; + int rc = SQLITE_OK; + ServerDb *pDb = pNew->pDb; + sqlite3_vfs *pVfs; + const char *zFilename = sqlite3PagerFilename(pNew->pPager, 0); + + assert( zFilename ); + pVfs = pDb->pVfs = sqlite3PagerVfs(pNew->pPager); + nByte = ROUND8(pVfs->szOsFile) * HMA_MAX_TRANSACTIONID; + pDb->aJrnlFdSpace = (u8*)sqlite3MallocZero(nByte); + if( pDb->aJrnlFdSpace==0 ){ + rc = SQLITE_NOMEM_BKPT; }else{ - for(pHma=g_server.pHma; pHma; pHma=pHma->pNext){ - if( sStat.st_dev==pHma->st_dev && sStat.st_ino==pHma->st_ino ) break; - } - if( pHma==0 ){ - int nPath = strlen(zPath); - int nByte = sizeof(ServerHMA) + nPath+1 + 4; - - pHma = (ServerHMA*)sqlite3_malloc(nByte); - if( pHma==0 ){ - rc = SQLITE_NOMEM; - }else{ - int i; - memset(pHma, 0, nByte); - pHma->zName = (char*)&pHma[1]; - pHma->nClient = 1; - pHma->st_dev = sStat.st_dev; - pHma->st_ino = sStat.st_ino; - pHma->pNext = g_server.pHma; - g_server.pHma = pHma; - - memcpy(pHma->zName, zPath, nPath); - memcpy(&pHma->zName[nPath], "-hma", 5); - - pHma->fd = open(pHma->zName, O_RDWR|O_CREAT, 0644); - if( pHma->fd<0 ){ - sqlite3_log(SQLITE_CANTOPEN, "Failed to open(%s)", pHma->zName); - rc = SQLITE_ERROR; - } + u8 *a = pDb->aJrnlFdSpace; + int i; + for(i=0; rc==SQLITE_OK && iaJrnl[i]; + pJ->jfd = (sqlite3_file*)&a[ROUND8(pVfs->szOsFile)*i]; + pJ->zJournal = sqlite3_mprintf("%s-journal%d", zFilename, i); + if( pJ->zJournal==0 ){ + rc = SQLITE_NOMEM_BKPT; + break; + } + rc = sqlite3OsAccess(pVfs, pJ->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + rc = sqlite3OsOpen(pVfs, pJ->zJournal, pJ->jfd, flags, &flags); if( rc==SQLITE_OK ){ - /* Write-lock the DMS slot. If successful, initialize the hma file. */ - rc = posixLock(pHma->fd, 0, SERVER_WRITE_LOCK, 0); - if( rc==SQLITE_OK ){ - res = ftruncate(pHma->fd, HMA_FILE_SIZE); - if( res!=0 ){ - sqlite3_log(SQLITE_CANTOPEN, - "Failed to ftruncate(%s)", pHma->zName - ); - rc = SQLITE_ERROR; - } - if( rc==SQLITE_OK ){ - rc = serverMapFile(pHma); - } - if( rc==SQLITE_OK ){ - memset(pHma->aMap, 0, HMA_FILE_SIZE); - }else{ - rc = SQLITE_ERROR; - } - for(i=0; rc==SQLITE_OK && ifd, 0, SERVER_READ_LOCK, 1); - } - } - - if( rc!=SQLITE_OK ){ - serverDecrHmaRefcount(pHma); - pHma = 0; + rc = sqlite3PagerRollbackJournal(pNew->pPager, pJ->jfd); } } - }else{ - pHma->nClient++; } } - *ppHma = pHma; + if( rc==SQLITE_OK ){ + pDb->bInit = 1; + }else{ + serverShutdownDatabase(pNew->pDb); + } return rc; } -static u32 *serverPageLockSlot(Server *p, Pgno pgno){ - int iSlot = pgno % HMA_PAGELOCK_SLOTS; - return &p->pHma->aMap[1 + HMA_CLIENT_SLOTS + iSlot]; -} -static u32 *serverClientSlot(Server *p, int iClient){ - return &p->pHma->aMap[1 + iClient]; -} - /* -** Close the "connection" and *-hma file. This deletes the object passed -** as the first argument. +** Close the connection. */ void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd){ - if( p->pHma ){ - ServerHMA *pHma = p->pHma; - serverEnterMutex(); - if( p->iClient>=0 ){ - u32 *pSlot = serverClientSlot(p, p->iClient); - *pSlot = 0; - assert( pHma->aClient[p->iClient]==p ); - pHma->aClient[p->iClient] = 0; - posixLock(pHma->fd, p->iClient+1, SERVER_NO_LOCK, 0); - } - if( dbfd - && pHma->nClient==1 - && SQLITE_OK==sqlite3OsLock(dbfd, SQLITE_LOCK_EXCLUSIVE) - ){ - unlink(pHma->zName); - } - serverDecrHmaRefcount(pHma); - serverLeaveMutex(); + ServerDb *pDb = p->pDb; + + serverEnterMutex(); + pDb->nClient--; + if( pDb->nClient==0 ){ + ServerDb **pp; + serverShutdownDatabase(pDb); + for(pp=&g_server.pDb; *pp!=pDb; pp=&((*pp)->pNext)); + *pp = pDb->pNext; + sqlite3_mutex_free(pDb->mutex); + sqlite3_free(pDb); } + serverLeaveMutex(); + sqlite3_free(p->aLock); sqlite3_free(p); } -static int serverRollbackClient(Server *p, int iBlock){ +/* +** Connect to the system. +*/ +int sqlite3ServerConnect( + Pager *pPager, + Server **ppOut /* OUT: Server handle */ +){ + Server *pNew = 0; + sqlite3_file *dbfd = sqlite3PagerFile(pPager); + i64 aFileId[2]; int rc; - sqlite3_log(SQLITE_NOTICE, "Rolling back failed client %d", iBlock); - - /* Roll back any journal file for client iBlock. */ - rc = sqlite3PagerRollbackJournal(p->pPager, iBlock); - - /* Clear any locks held by client iBlock from the HMA file. */ + rc = sqlite3OsFileControl(dbfd, SQLITE_FCNTL_FILEID, (void*)aFileId); if( rc==SQLITE_OK ){ - int i; - for(i=0; i>HMA_CLIENT_SLOTS)==iBlock+1 ){ - n = n & ((1<pPager = pPager; + pNew->iTransId = -1; + rc = serverFindDatabase(pNew, aFileId); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + }else{ + sqlite3_mutex_enter(pNew->pDb->mutex); + if( pNew->pDb->bInit==0 ){ + rc = serverInitDatabase(pNew); } - if( __sync_val_compare_and_swap(pSlot, v, n)==v ) break; - v = *pSlot; + sqlite3_mutex_leave(pNew->pDb->mutex); } + }else{ + rc = SQLITE_NOMEM_BKPT; } } + *ppOut = pNew; return rc; } - /* -** Open the *-hma file and "connect" to the system. +** Begin a transaction. */ -int sqlite3ServerConnect( - Pager *pPager, - Server **ppOut, - int *piClient -){ - const char *zPath = sqlite3PagerFilename(pPager, 0); +int sqlite3ServerBegin(Server *p){ int rc = SQLITE_OK; - Server *p; - - p = (Server*)sqlite3_malloc(sizeof(Server)); - if( p==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(p, 0, sizeof(Server)); - p->iClient = -1; - p->pPager = pPager; - serverEnterMutex(); - rc = serverOpenHma(pPager, zPath, &p->pHma); + if( p->iTransId<0 ){ + int id; + ServerDb *pDb = p->pDb; + u32 t; - /* File is now mapped. Find a free client slot. */ - if( rc==SQLITE_OK ){ - int i; - Server **aClient = p->pHma->aClient; - int fd = p->pHma->fd; - for(i=0; imutex); - if( rc==SQLITE_OK ){ - if( i>HMA_CLIENT_SLOTS ){ - rc = SQLITE_BUSY; - }else{ - u32 *pSlot = serverClientSlot(p, i); - *piClient = p->iClient = i; - aClient[i] = p; - *pSlot = 1; - } + /* Find a transaction id to use */ + rc = SQLITE_BUSY; + t = pDb->transmask; + for(id=0; idtransmask = t; - serverLeaveMutex(); - } - - if( rc!=SQLITE_OK ){ - sqlite3ServerDisconnect(p, 0); - p = 0; - } - *ppOut = p; - return rc; -} - -static int serverOvercomeLock( - Server *p, /* Server connection */ - int bWrite, /* True for a write-lock */ - int bBlock, /* If true, block for this lock */ - u32 v, /* Value of blocking page locking slot */ - int *pbRetry /* OUT: True if caller should retry lock */ -){ - int rc = SQLITE_OK; - int iBlock = ((int)(v>>HMA_CLIENT_SLOTS))-1; - - if( iBlock<0 || iBlock==p->iClient ){ - for(iBlock=0; iBlockiClient && (v & (1<mutex); - if( 0==p->pHma->aClient[iBlock] ){ - rc = posixLock(p->pHma->fd, iBlock+1, SERVER_WRITE_LOCK, 0); if( rc==SQLITE_OK ){ - rc = serverRollbackClient(p, iBlock); - - /* Release the lock on slot iBlock */ - posixLock(p->pHma->fd, iBlock+1, SERVER_NO_LOCK, 0); - if( rc==SQLITE_OK ){ - *pbRetry = 1; - } - }else if( rc==SQLITE_BUSY ){ - if( bBlock ){ - rc = posixLock(p->pHma->fd, iBlock+1, SERVER_READ_LOCK, 1); - if( rc==SQLITE_OK ){ - posixLock(p->pHma->fd, iBlock+1, SERVER_NO_LOCK, 0); - *pbRetry = 1; - } - } - - if( rc==SQLITE_BUSY ){ - rc = SQLITE_OK; - } + ServerJournal *pJrnl = &pDb->aJrnl[id]; + sqlite3PagerServerJournal(p->pPager, pJrnl->jfd, pJrnl->zJournal); + p->iTransId = id; } } - serverLeaveMutex(); - return rc; } -/* -** Begin a transaction. -*/ -int sqlite3ServerBegin(Server *p){ -#if 1 - int rc = posixLock(p->pHma->fd, p->iClient+1, SERVER_WRITE_LOCK, 1); - if( rc ) return rc; -#endif - return sqlite3ServerLock(p, 1, 0, 1); -} - -/* -** End a transaction (and release all locks). -*/ -int sqlite3ServerEnd(Server *p){ +static void serverReleaseLocks(Server *p){ + ServerDb *pDb = p->pDb; int i; + assert( sqlite3_mutex_held(pDb->mutex) ); + for(i=0; inLock; i++){ - u32 *pSlot = serverPageLockSlot(p, p->aLock[i]); - while( 1 ){ - u32 v = *pSlot; - u32 n = v; - if( (v>>HMA_CLIENT_SLOTS)==p->iClient+1 ){ - n = n & ((1 << HMA_CLIENT_SLOTS)-1); - } - n = n & ~(1 << p->iClient); - if( __sync_val_compare_and_swap(pSlot, v, n)==v ) break; - } - if( p->aLock[i]==0 ){ - struct timeval t2; - i64 nUs; - gettimeofday(&t2, 0); - nUs = (i64)t2.tv_sec * 1000000 + t2.tv_usec - p->iUsWrite; - p->nUsWrite += nUs; - if( (p->nUsWrite / 1000000)!=((p->nUsWrite + nUs)/1000000) ){ - sqlite3_log(SQLITE_WARNING, - "Cumulative WRITER time: %lldms\n", p->nUsWrite/1000 - ); - } + u32 *pSlot = &pDb->aSlot[p->aLock[i] % HMA_PAGELOCK_SLOTS]; + if( slotGetWriter(*pSlot)==p->iTransId ){ + *pSlot -= ((p->iTransId + 1) << HMA_MAX_TRANSACTIONID); } + *pSlot &= ~((u32)1 << p->iTransId); } + p->nLock = 0; -#if 1 - return posixLock(p->pHma->fd, p->iClient+1, SERVER_READ_LOCK, 0); -#endif - return SQLITE_OK; } /* -** Release all write-locks. +** End a transaction (and release all locks). */ -int sqlite3ServerReleaseWriteLocks(Server *p){ +int sqlite3ServerEnd(Server *p){ int rc = SQLITE_OK; + ServerDb *pDb = p->pDb; + sqlite3_mutex_enter(pDb->mutex); + + serverReleaseLocks(p); + pDb->transmask &= ~((u32)1 << p->iTransId); + + sqlite3_mutex_leave(pDb->mutex); + p->iTransId = -1; return rc; } /* -** Return the client id of the client that currently holds the EXCLUSIVE -** or RESERVED lock according to page-locking slot value v. Or -1 if no -** client holds such a lock. +** Release all write-locks. */ -int serverWriteLocker(u32 v){ - return ((int)(v >> HMA_CLIENT_SLOTS)) - 1; +int sqlite3ServerReleaseWriteLocks(Server *p){ + int rc = SQLITE_OK; + return rc; } /* @@ -500,102 +379,56 @@ int serverWriteLocker(u32 v){ */ int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock){ int rc = SQLITE_OK; - int bReserved = 0; - u32 *pSlot = serverPageLockSlot(p, pgno); + ServerDb *pDb = p->pDb; + int iWriter; + int bSkip = 0; + u32 *pSlot; - /* Grow the aLock[] array, if required */ + assert( p->nLock<=p->nAlloc ); if( p->nLock==p->nAlloc ){ - int nNew = p->nAlloc ? p->nAlloc*2 : 128; - u32 *aNew; - aNew = (u32*)sqlite3_realloc(p->aLock, sizeof(u32)*nNew); - if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - p->aLock = aNew; - p->nAlloc = nNew; - } + int nNew = p->nLock ? p->nLock*2 : 256; + u32 *aNew = sqlite3_realloc(p->aLock, nNew*sizeof(u32)); + if( aNew==0 ) return SQLITE_NOMEM_BKPT; + memset(&aNew[p->nLock], 0, sizeof(u32) * (nNew - p->nLock)); + p->nAlloc = nNew; + p->aLock = aNew; } - if( rc==SQLITE_OK ){ - u32 v = *pSlot; - - /* Check if the required lock is already held. If so, exit this function - ** early. Otherwise, add an entry to the aLock[] array to record the fact - ** that the lock may need to be released. */ - if( bWrite ){ - int iLock = ((int)(v>>HMA_CLIENT_SLOTS)) - 1; - if( iLock==p->iClient ) goto server_lock_out; - }else{ - if( v & (1<iClient) ) goto server_lock_out; - } - p->aLock[p->nLock++] = pgno; - while( 1 ){ - u32 n; - int w; - u32 mask = (bWrite ? (((1<iClient)) : 0); - - while( ((w = serverWriteLocker(v))>=0 && w!=p->iClient) || (v & mask) ){ - int bRetry = 0; - - if( w<0 && bWrite && bBlock ){ - /* Attempt a RESERVED lock before anything else */ - n = v | ((p->iClient+1) << HMA_CLIENT_SLOTS); - assert( serverWriteLocker(n)==p->iClient ); - if( __sync_val_compare_and_swap(pSlot, v, n)!=v ){ - v = *pSlot; - continue; - } - v = n; - bReserved = 1; - } - - rc = serverOvercomeLock(p, bWrite, bBlock, v, &bRetry); - if( rc!=SQLITE_OK ) goto server_lock_out; - if( bRetry==0 ){ - /* There is a conflicting lock. Cannot obtain this lock. */ - sqlite3_log(SQLITE_BUSY_DEADLOCK, "Conflict at page %d", (int)pgno); - rc = SQLITE_BUSY_DEADLOCK; - goto server_lock_out; - } - - v = *pSlot; - } - - n = v | (1 << p->iClient); - if( bWrite ){ - n = n | ((p->iClient+1) << HMA_CLIENT_SLOTS); - } - if( __sync_val_compare_and_swap(pSlot, v, n)==v ) break; - v = *pSlot; + assert( p->iTransId>=0 ); + + sqlite3_mutex_enter(pDb->mutex); + pSlot = &pDb->aSlot[pgno % HMA_PAGELOCK_SLOTS]; + iWriter = slotGetWriter(*pSlot); + if( iWriter==p->iTransId || (bWrite==0 && (*pSlot & (1<iTransId))) ){ + bSkip = 1; + }else if( iWriter>=0 ){ + rc = SQLITE_BUSY_DEADLOCK; + }else if( bWrite ){ + if( (slotReaderMask(*pSlot) & ~(1 << p->iTransId))==0 ){ + *pSlot += ((p->iTransId + 1) << HMA_MAX_TRANSACTIONID); + }else{ + rc = SQLITE_BUSY_DEADLOCK; } + }else{ + *pSlot |= (1 << p->iTransId); } -server_lock_out: - if( rc!=SQLITE_OK && bReserved ){ - u32 n; - u32 v; - do{ - v = *pSlot; - assert( serverWriteLocker(v)==p->iClient ); - n = v & ((1<mutex); - if( pgno==0 ){ - struct timeval t1; - gettimeofday(&t1, 0); - p->iUsWrite = ((i64)t1.tv_sec * 1000000) + (i64)t1.tv_usec; + if( bSkip==0 ){ + p->aLock[p->nLock++] = pgno; } - assert( rc!=SQLITE_OK || sqlite3ServerHasLock(p, pgno, bWrite) ); return rc; } int sqlite3ServerHasLock(Server *p, Pgno pgno, int bWrite){ - u32 v = *serverPageLockSlot(p, pgno); - if( bWrite ){ - return (v>>HMA_CLIENT_SLOTS)==(p->iClient+1); - } - return (v & (1 << p->iClient))!=0; + assert( 0 ); + return 0; } #endif /* ifdef SQLITE_SERVER_EDITION */ diff --git a/src/server.h b/src/server.h index c7c4192452..46e67a0d8c 100644 --- a/src/server.h +++ b/src/server.h @@ -19,7 +19,7 @@ typedef struct Server Server; -int sqlite3ServerConnect(Pager *pPager, Server **ppOut, int *piClient); +int sqlite3ServerConnect(Pager *pPager, Server **ppOut); void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 803d8472a1..f8fb5a13d0 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1044,6 +1044,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_JOURNAL_POINTER 28 #define SQLITE_FCNTL_WIN32_GET_HANDLE 29 #define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_FILEID 31 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE diff --git a/test/permutations.test b/test/permutations.test index 173702d95a..53c08cd9c5 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -277,9 +277,7 @@ test_suite "server" -prefix "" -description { All server-edition tests. } -files [ test_set \ - select1.test \ - [glob -nocomplain $::testdir/server*.test] \ - -exclude *server1.test + select1.test server2.test server3.test ] test_suite "fts5-light" -prefix "" -description { diff --git a/test/server2.test b/test/server2.test index adfd76859b..7ff13bc1b9 100644 --- a/test/server2.test +++ b/test/server2.test @@ -17,36 +17,60 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix server2 +db close +foreach f [glob -nocomplain test.db*] { + forcedelete $f +} + #------------------------------------------------------------------------- -# Check that the *-hma file is deleted correctly. +# Check that the *-journal* files are deleted correctly. # +reset_db do_execsql_test 1.0 { CREATE TABLE t1(a, b); } {} + do_test 1.1 { - file exists test.db-hma -} {1} + lsort [glob test.db*] +} {test.db test.db-journal0} + do_test 1.2 { db close - file exists test.db-hma -} {0} -do_test 1.3 { - sqlite3 db test.db - db eval { CREATE TABLE t2(a, b) } - sqlite3 db2 test.db - db2 eval { CREATE TABLE t3(a, b) } - file exists test.db-hma -} {1} + lsort [glob test.db*] +} {test.db} + +sqlite3 db test.db +do_execsql_test 1.3 { + CREATE TABLE t2(a, b); +} {} + +sqlite3 db2 test.db do_test 1.4 { - db2 close - file exists test.db-hma -} {1} -integrity_check 1.5 + db eval { + BEGIN; + INSERT INTO t1 VALUES(1, 2); + } + db2 eval { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + } +} {} + +do_test 1.5 { + db2 eval COMMIT + db eval COMMIT + lsort [glob test.db*] +} {test.db test.db-journal0 test.db-journal1} + do_test 1.6 { db close - file exists test.db-hma -} {0} + lsort [glob test.db*] +} {test.db test.db-journal0 test.db-journal1} +do_test 1.7 { + db2 close + lsort [glob test.db*] +} {test.db} #------------------------------------------------------------------------- # @@ -72,13 +96,13 @@ do_test 2.1 { } {} do_test 2.2 { lsort [glob test.db*] -} {test.db test.db-hma test.db-journal0 test.db-journal1} +} {test.db test.db-journal0 test.db-journal1} do_test 2.3.1 { db eval COMMIT } {} do_test 2.3.2 { db2 eval COMMIT } {} do_execsql_test 2.4 {SELECT * FROM t1, t2} {1 2 3 4} do_test 2.5 { lsort [glob test.db*] -} {test.db test.db-hma test.db-journal0 test.db-journal1} +} {test.db test.db-journal0 test.db-journal1} do_test 2.6 { execsql {BEGIN} @@ -106,7 +130,7 @@ do_execsql_test 3.0 { do_test 3.1 { glob test.db* -} {test.db-journal0 test.db test.db-hma} +} {test.db-journal0 test.db} do_test 3.2 { db close diff --git a/test/server3.test b/test/server3.test index 85d69e28b4..966ca1c454 100644 --- a/test/server3.test +++ b/test/server3.test @@ -21,6 +21,8 @@ set testprefix server3 db close do_multiclient_test tn { + if {$tn==1} continue + do_test $tn.1 { sql1 { CREATE TABLE t1(a, b) } sql2 { CREATE TABLE t2(a, b) }