From: dan Date: Fri, 7 Jul 2017 16:12:45 +0000 (+0000) Subject: Have this branch maintain an in-memory hash-table of old pages for read-only X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=f42e5d891720471b04755e58a64a376c44083501;p=thirdparty%2Fsqlite.git Have this branch maintain an in-memory hash-table of old pages for read-only MVCC clients. There is no way to access it yet. FossilOrigin-Name: a3a9a2e1899cc963315590ef4666972e9d92986843706a551962ed16661a19b2 --- diff --git a/manifest b/manifest index 61b6b17670..7263bbf72f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stserver\sfixes\swith\sthis\sbranch. -D 2017-06-28T20:21:15.793 +C Have\sthis\sbranch\smaintain\san\sin-memory\shash-table\sof\sold\spages\sfor\sread-only\nMVCC\sclients.\sThere\sis\sno\sway\sto\saccess\sit\syet. +D 2017-07-07T16:12:45.878 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6 @@ -389,7 +389,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c d3c3a186ceb27d363f7a5e281ece4eb32b56e173a331e153a761d280034e3212 F src/os_win.c 2a6c73eef01c51a048cc4ddccd57f981afbec18a F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c 3f9c94c6906c624c4db730523f4e91045ffb7bdfa48ead118025c6ae3e176cea +F src/pager.c 0aa31e7e72a6ba74f7480f5116d01a177e5e8f16d8737275abc660720c3b1788 F src/pager.h 81d2c28e92e543cb1ed977f44810d04c34be2a23d7782bb02548d86a4b9ae657 F src/parse.y 0513387ce02fea97897d8caef82d45f347818593f24f1bdc48e0c530a8af122d F src/pcache.c 62835bed959e2914edd26afadfecce29ece0e870 @@ -403,8 +403,8 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 3e518b962d932a997fae373366880fc028c75706 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 4f0adefaa5e9417459b07757e0f6060cac97930a86f0fba9797bab233ced66c0 -F src/server.c 9f1ce178587ae672058923f47de628d97490d3089fae8b9d6e3e6d6c5148320f -F src/server.h 3796b9dcb61c60c4e49e9f579832ca1b4e63c6493dae5314b2970540a7cd22f8 +F src/server.c 4b76241f63e07683fe30db6f6b8a334867b75c4cc5e87a8374b649cf6b8b0daa +F src/server.h 31645080b17836bcb193a3e5ea8e60d7ae3e4cee59e8f9450b17cdbf452a2b1e F src/shell.c e5950029da103c5d378e71d548759459b9a7fc76177a71562c22082c705745ab F src/sqlite.h.in 61cac6e788c961bbaf2f91f55d63a8175809bddb7b4971a894b588321c1a82a6 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 @@ -1585,7 +1585,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dfa9a4d53e0aa7f7911b11af57a4d37a1df2d835cf60eb9e76d28b6b39c2ba58 093b9108ea3532437683547c82f756df2413013a50a2fddad3d928cf8d74f9eb -R 72b821ad7bad9e97d348feb666cadf11 +P 58a0aab8fd5f5dcbec867468b7f4c1b67c3e16bbe3aeea6bd21ae6b04c2b93f1 +R ad98c37bc306ca10fc5f37a74c877e1d U dan -Z d8120536ead40ef5709cd4556448f644 +Z a567fec683db61555798d82aeb253406 diff --git a/manifest.uuid b/manifest.uuid index f2e21e8d12..92f7b0f176 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -58a0aab8fd5f5dcbec867468b7f4c1b67c3e16bbe3aeea6bd21ae6b04c2b93f1 \ No newline at end of file +a3a9a2e1899cc963315590ef4666972e9d92986843706a551962ed16661a19b2 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 297cdeadc2..53b7ba1dda 100644 --- a/src/pager.c +++ b/src/pager.c @@ -708,6 +708,7 @@ struct Pager { #endif #ifdef SQLITE_SERVER_EDITION Server *pServer; + ServerPage *pServerPage; #endif }; @@ -1789,6 +1790,18 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } +#ifdef SQLITE_SERVER_EDITION +static void pagerFreeServerPage(Pager *pPager){ + ServerPage *pPg; + ServerPage *pNext; + for(pPg=pPager->pServerPage; pPg; pPg=pNext){ + pNext = pPg->pNext; + sqlite3_free(pPg); + } + pPager->pServerPage = 0; +} +#endif + /* ** This function is a no-op if the pager is in exclusive mode and not ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN @@ -1819,6 +1832,7 @@ static void pager_unlock(Pager *pPager){ #ifdef SQLITE_SERVER_EDITION if( pagerIsServer(pPager) ){ + pagerFreeServerPage(pPager); sqlite3ServerEnd(pPager->pServer); pPager->eState = PAGER_OPEN; }else @@ -4359,6 +4373,14 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( pPager->eLock==EXCLUSIVE_LOCK ); assert( isOpen(pPager->fd) || pList->pDirty==0 ); +#ifdef SQLITE_SERVER_EDITION + if( pagerIsServer(pPager) ){ + rc = sqlite3ServerPreCommit(pPager->pServer, pPager->pServerPage); + pPager->pServerPage = 0; + if( rc!=SQLITE_OK ) return rc; + } +#endif + /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch ** is taken, as pager_wait_on_lock() is a no-op for temp-files. @@ -4536,6 +4558,8 @@ static int pagerStress(void *p, PgHdr *pPg){ Pager *pPager = (Pager *)p; int rc = SQLITE_OK; + if( pagerIsServer(pPager) ) return SQLITE_OK; + assert( pPg->pPager==pPager ); assert( pPg->flags&PGHDR_DIRTY ); @@ -5909,6 +5933,20 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ char *pData2; i64 iOff = pPager->journalOff; +#ifdef SQLITE_SERVER_EDITION + if( pagerIsServer(pPager) ){ + int nByte = sizeof(ServerPage) + pPager->pageSize; + ServerPage *p = (ServerPage*)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM_BKPT; + memset(p, 0, sizeof(ServerPage)); + p->aData = (u8*)&p[1]; + p->nData = pPager->pageSize; + p->pgno = pPg->pgno; + p->pNext = pPager->pServerPage; + pPager->pServerPage = p; + } +#endif + /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ diff --git a/src/server.c b/src/server.c index c265de5998..5e086207b8 100644 --- a/src/server.c +++ b/src/server.c @@ -13,6 +13,8 @@ #include "sqliteInt.h" +#ifdef SQLITE_SERVER_EDITION + /* ** Page-locking slot format: ** @@ -22,14 +24,23 @@ ** lock is taken, the client sets the bit associated with its ** transaction-id. ** -** The next 8 bits are set to the number of transient-read locks -** currently held on the page. -** ** The next 5 bits are set to 0 if no client currently holds a write ** lock. Or to (transaction-id + 1) if a write lock is held. +** +** The next 8 bits are set to the number of transient-read locks +** currently held on the page. */ +#define HMA_SLOT_RL_BITS 16 /* bits for Read Locks */ +#define HMA_SLOT_WL_BITS 5 /* bits for Write Locks */ +#define HMA_SLOT_TR_BITS 8 /* bits for Transient Reader locks */ + +#define HMA_SLOT_RLWL_BITS (HMA_SLOT_RL_BITS + HMA_SLOT_WL_BITS) + + +#define HMA_SLOT_RL_MASK ((1 << HMA_SLOT_RL_BITS)-1) +#define HMA_SLOT_WL_MASK (((1 << HMA_SLOT_WL_BITS)-1) << HMA_SLOT_RL_BITS) +#define HMA_SLOT_TR_MASK (((1 << HMA_SLOT_TR_BITS)-1) << HMA_SLOT_RLWL_BITS) -#ifdef SQLITE_SERVER_EDITION /* Number of page-locking slots */ #define HMA_PAGELOCK_SLOTS (256*1024) @@ -37,14 +48,23 @@ /* Maximum concurrent read/write transactions */ #define HMA_MAX_TRANSACTIONID 16 + +#define HMA_HASH_SIZE 512 + /* ** The argument to this macro is the value of a locking slot. It returns ** -1 if no client currently holds the write lock, or the transaction-id ** of the locker otherwise. */ -#define slotGetWriter(v) (((int)((v) >> HMA_MAX_TRANSACTIONID) & 0x1f) -1) +#define slotGetWriter(v) ((((int)(v)&HMA_SLOT_WL_MASK) >> HMA_SLOT_RL_BITS) - 1) + +/* +** The argument to this macro is the value of a locking slot. This macro +** returns the current number of slow reader clients reading the page. +*/ +#define slotGetSlowReaders(v) (((v) & HMA_SLOT_TR_MASK) >> HMA_SLOT_RLWL_BITS) -#define slotReaderMask(v) ((v) & ((1 << HMA_MAX_TRANSACTIONID)-1)) +#define slotReaderMask(v) ((v) & HMA_SLOT_RL_MASK) #include "unistd.h" #include "fcntl.h" @@ -82,6 +102,13 @@ struct ServerDb { sqlite3_vfs *pVfs; ServerJournal aJrnl[HMA_MAX_TRANSACTIONID]; u8 *aJrnlFdSpace; + + int iNextCommit; /* Commit id for next pre-commit call */ + Server *pCommit; /* List of connections currently commiting */ + Server *pReader; /* Connections in slower-reader transaction */ + ServerPage *pPgFirst; /* First (oldest) in list of pages */ + ServerPage *pPgLast; /* Last (newest) in list of pages */ + ServerPage *apPg[HMA_HASH_SIZE]; }; /* @@ -92,9 +119,11 @@ struct Server { ServerDb *pDb; /* Database object */ Pager *pPager; /* Associated pager object */ int iTransId; /* Current transaction id (or -1) */ + int iCommitId; /* Current comit id (or 0) */ int nAlloc; /* Allocated size of aLock[] array */ int nLock; /* Number of entries in aLock[] */ u32 *aLock; /* Mapped lock file */ + Server *pNext; /* Next in pCommit or pReader list */ }; #define SERVER_WRITE_LOCK 3 @@ -142,6 +171,7 @@ static int serverFindDatabase(Server *pNew, i64 *aFileId){ p = 0; }else{ p->nClient = 1; + p->iNextCommit = 1; p->aFileId[0] = aFileId[0]; p->aFileId[1] = aFileId[1]; p->pNext = g_server.pDb; @@ -352,14 +382,129 @@ static void serverReleaseLocks(Server *p){ */ int sqlite3ServerEnd(Server *p){ int rc = SQLITE_OK; + Server **pp; ServerDb *pDb = p->pDb; + ServerPage *pFree = 0; + ServerPage *pPg = 0; sqlite3_mutex_enter(pDb->mutex); serverReleaseLocks(p); + + /* Clear the bit in the transaction mask. */ pDb->transmask &= ~((u32)1 << p->iTransId); + /* If this connection is in the committers list, remove it. */ + for(pp=&pDb->pCommit; *pp; pp = &((*pp)->pNext)){ + if( *pp==p ){ + *pp = p->pNext; + break; + } + } + + /* See if it is possible to free any ServerPage records. If so, remove + ** them from the linked list and hash table, but do not call sqlite3_free() + ** on them until the mutex has been released. */ + if( pDb->pPgFirst ){ + Server *pIter; + int iOldest = 0x7FFFFFFF; + for(pIter=pDb->pReader; pIter; pIter=pIter->pNext){ + iOldest = MIN(iOldest, pIter->iCommitId); + } + for(pIter=pDb->pCommit; pIter; pIter=pIter->pNext){ + iOldest = MIN(iOldest, pIter->iCommitId); + } + + pFree = pDb->pPgFirst; + for(pPg=pDb->pPgFirst; pPg && pPg->iCommitIdpNext){ + if( pPg->pHashPrev ){ + pPg->pHashPrev->pHashNext = pPg->pHashNext; + }else{ + int iHash = pPg->pgno % HMA_HASH_SIZE; + assert( pDb->apPg[iHash]==pPg ); + pDb->apPg[iHash] = pPg->pHashNext; + } + if( pPg->pHashNext ){ + pPg->pHashNext->pHashPrev = pPg->pHashPrev; + } + } + if( pPg==0 ){ + pDb->pPgFirst = pDb->pPgLast = 0; + }else{ + pDb->pPgFirst = pPg; + } + } + sqlite3_mutex_leave(pDb->mutex); + + /* Call sqlite3_free() on any pages that were unlinked from the hash + ** table above. */ + while( pFree && pFree!=pPg ){ + ServerPage *pNext = pFree->pNext; + sqlite3_free(pFree); + pFree = pNext; + } + p->iTransId = -1; + p->iCommitId = 0; + return rc; +} + +int sqlite3ServerPreCommit(Server *p, ServerPage *pPg){ + ServerDb *pDb = p->pDb; + int rc = SQLITE_OK; + ServerPage *pIter; + ServerPage *pNext; + + if( pPg==0 ) return SQLITE_OK; + + sqlite3_mutex_enter(pDb->mutex); + + /* Assign a commit id to this transaction */ + assert( p->iCommitId==0 ); + p->iCommitId = pDb->iNextCommit++; + + /* Iterate through all pages. For each: + ** + ** 1. Set the iCommitId field. + ** 2. Add the page to the hash table. + ** 3. Wait until all slow-reader locks have cleared. + */ + for(pIter=pPg; pIter; pIter=pIter->pNext){ + u32 *pSlot = &pDb->aSlot[pIter->pgno % HMA_PAGELOCK_SLOTS]; + int iHash = pIter->pgno % HMA_HASH_SIZE; + pIter->iCommitId = p->iCommitId; + pIter->pHashNext = pDb->apPg[iHash]; + if( pIter->pHashNext ){ + pIter->pHashNext->pHashPrev = pIter; + } + pDb->apPg[iHash] = pIter; + + /* TODO: Something better than this! */ + while( slotGetSlowReaders(*pSlot)>0 ){ + sqlite3_mutex_leave(pDb->mutex); + sqlite3_mutex_enter(pDb->mutex); + } + + /* If pIter is the last element in the list, append the new list to + ** the ServerDb.pPgFirst/pPgLast list at this point. */ + if( pIter->pNext==0 ){ + if( pDb->pPgLast ){ + assert( pDb->pPgFirst ); + pDb->pPgLast->pNext = pPg; + }else{ + assert( pDb->pPgFirst==0 ); + pDb->pPgFirst = pPg; + } + pDb->pPgLast = pIter; + } + } + + /* Add this connection to the list of current committers */ + assert( p->pNext==0 ); + p->pNext = pDb->pCommit; + pDb->pCommit = p; + + sqlite3_mutex_leave(pDb->mutex); return rc; } @@ -384,6 +529,7 @@ int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock){ int bSkip = 0; u32 *pSlot; + assert( p->iTransId>=0 ); assert( p->nLock<=p->nAlloc ); if( p->nLock==p->nAlloc ){ int nNew = p->nLock ? p->nLock*2 : 256; @@ -394,10 +540,14 @@ int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock){ p->aLock = aNew; } - assert( p->iTransId>=0 ); - sqlite3_mutex_enter(pDb->mutex); + pSlot = &pDb->aSlot[pgno % HMA_PAGELOCK_SLOTS]; + assert( slotGetWriter(*pSlot)<0 + || slotReaderMask(*pSlot)==0 + || slotReaderMask(*pSlot)==(1 << slotGetWriter(*pSlot)) + ); + iWriter = slotGetWriter(*pSlot); if( iWriter==p->iTransId || (bWrite==0 && (*pSlot & (1<iTransId))) ){ bSkip = 1; diff --git a/src/server.h b/src/server.h index 46e67a0d8c..ad81001fa2 100644 --- a/src/server.h +++ b/src/server.h @@ -19,12 +19,25 @@ typedef struct Server Server; -int sqlite3ServerConnect(Pager *pPager, Server **ppOut); +typedef struct ServerPage ServerPage; +struct ServerPage { + Pgno pgno; /* Page number for this record */ + int nData; /* Size of aData[] in bytes */ + u8 *aData; + ServerPage *pNext; + + int iCommitId; + ServerPage *pHashNext; + ServerPage *pHashPrev; +}; +int sqlite3ServerConnect(Pager *pPager, Server **ppOut); void sqlite3ServerDisconnect(Server *p, sqlite3_file *dbfd); int sqlite3ServerBegin(Server *p); +int sqlite3ServerPreCommit(Server*, ServerPage*); int sqlite3ServerEnd(Server *p); + int sqlite3ServerReleaseWriteLocks(Server *p); int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock); @@ -32,5 +45,5 @@ int sqlite3ServerLock(Server *p, Pgno pgno, int bWrite, int bBlock); int sqlite3ServerHasLock(Server *p, Pgno pgno, int bWrite); #endif /* SQLITE_SERVER_H */ - #endif /* SQLITE_SERVER_EDITION */ +