From: drh Date: Thu, 16 Jul 2015 18:18:19 +0000 (+0000) Subject: Defer the bulk pcache1 memory allocation until the first page allocation X-Git-Tag: version-3.8.11~30 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=957026ac784f88287d714cadf2566ed9bac1a1b2;p=thirdparty%2Fsqlite.git Defer the bulk pcache1 memory allocation until the first page allocation request. Limit the size of the pcache1 bulk allocation to the cache_size setting. Deallocate the bulk allocation on a sqlite3_db_release_memory() request, if the bulk allocation is completely unused. FossilOrigin-Name: b79a4affe44bd0c8e155cae19f3f62c715684cd6 --- diff --git a/manifest b/manifest index dbbb79b3e9..0ea09a905b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissues\swith\sSQLITE_OMIT_COMPOUND_SELECT\sdefined. -D 2015-07-16T17:29:27.953 +C Defer\sthe\sbulk\spcache1\smemory\sallocation\suntil\sthe\sfirst\spage\sallocation\nrequest.\s\sLimit\sthe\ssize\sof\sthe\spcache1\sbulk\sallocation\sto\sthe\scache_size\nsetting.\s\sDeallocate\sthe\sbulk\sallocation\son\sa\ssqlite3_db_release_memory()\nrequest,\sif\sthe\sbulk\sallocation\sis\scompletely\sunused. +D 2015-07-16T18:18:19.580 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6e8af213d49e6325bf283ebed7662254f8e15bda F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -319,9 +319,9 @@ F src/pager.h 6d435f563b3f7fcae4b84433b76a6ac2730036e2 F src/parse.y 6d60dda8f8d418b6dc034f1fbccd816c459983a8 F src/pcache.c cde06aa50962595e412d497e22fd2e07878ba1f0 F src/pcache.h 9968603796240cdf83da7e7bef76edf90619cea9 -F src/pcache1.c 3f4c87cf913f2de8189026d9a5223ddaf55eaf6b -F src/pragma.c c1f4d012ea9f6b1ce52d341b2cd0ad72d560afd7 -F src/pragma.h b8632d7cdda7b25323fa580e3e558a4f0d4502cc +F src/pcache1.c 6c1e5957087b74e7e7cff871561849cf2fe09654 +F src/pragma.c e52084b37a08a88f258830518461e94627af2621 +F src/pragma.h 631a91c8b0e6ca8f051a1d8a4a0da4150e04620a F src/prepare.c 82e5db1013846a819f198336fed72c44c974e7b1 F src/printf.c 2bc439ff20a4aad0e0ad50a37a67b5eae7d20edc F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -823,7 +823,7 @@ F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 F test/malloc.test 21c213365f2cca95ab2d7dc078dc8525f96065f8 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 -F test/malloc5.test e9a9116f80ab6b7a9ca258876c6f3dedb08cb08b +F test/malloc5.test 21f6552bacb9efe649e6ba10a52f2cedaecce242 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d @@ -900,8 +900,8 @@ F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 5769fc62d8c890a83a503f67d47508dfdc543305 -F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d -F test/pcache2.test ec3ae192f444ee6a0a80d1fd80d99695d884bfb3 +F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b +F test/pcache2.test c70d92547550136ba6f818e6a44fe246d2738604 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test 6a88fd9ca15b804e9c20990773262ca67494058f F test/pragma.test be7195f0aa72bdb8a512133e9640ac40f15b57a2 @@ -1327,7 +1327,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh d1a2da0e15b2ed33d60af35c7e9d483f13a8eb9f F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 40c287d3f929ece67da6e9e7c49885789960accf +F tool/mkpragmatab.tcl 84af2b180484323a2ea22a2279e8bd9e3e1e492e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 87240b09c20042999b41d5fabe091b7111287835 F tool/mksqlite3c.tcl f29898d34f1dcd77ccc4e6fd7dcc2f9ebb54622f @@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2cdd647951ff5dca53576bb8be6dd6310a557571 -R 2352a36372e4c50dccbac9452cf765fb -U mistachkin -Z ddf00ea17d7e53d080e6ac931c2ced93 +P 9c39d4644530ccc532f4ff26464106c6da43269a +R 09036069f4d0dade4c9d3cbed758be18 +U drh +Z 38e6769945da8f33e2e2b46c42a73ce6 diff --git a/manifest.uuid b/manifest.uuid index 1edb0e4abf..f712181dec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c39d4644530ccc532f4ff26464106c6da43269a \ No newline at end of file +b79a4affe44bd0c8e155cae19f3f62c715684cd6 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index a345e6760f..8d74da5c90 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -191,6 +191,7 @@ static SQLITE_WSD struct PCacheGlobal { */ int isInit; /* True if initialized */ int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ @@ -259,6 +260,43 @@ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ } } +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + int szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * pcache1.nInitPage; + }else{ + szBulk = -1024*pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*pCache->nMax ){ + szBulk = pCache->szAlloc*pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + int i; + for(i=0; iszPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->pNext = pCache->pFree; + pCache->pFree = pX; + zBulk += pCache->szAlloc; + } + } + return pCache->pFree!=0; +} + /* ** Malloc function used within this file to allocate space from the buffer ** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no @@ -359,7 +397,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ void *pPg; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - if( pCache->pFree ){ + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ p = pCache->pFree; pCache->pFree = p->pNext; p->pNext = 0; @@ -563,7 +601,8 @@ static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ ** If there are currently more than nMaxPage pages allocated, try ** to recycle pages to reduce the number allocated to nMaxPage. */ -static void pcache1EnforceMaxPage(PGroup *pGroup){ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; assert( sqlite3_mutex_held(pGroup->mutex) ); while( pGroup->nCurrentPage>pGroup->nMaxPage && pGroup->pLruTail ){ PgHdr1 *p = pGroup->pLruTail; @@ -572,6 +611,10 @@ static void pcache1EnforceMaxPage(PGroup *pGroup){ pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; + } } /* @@ -647,6 +690,14 @@ static int pcache1Init(void *NotUsed){ pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } #endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; + } pcache1.grp.mxPinned = 10; pcache1.isInit = 1; return SQLITE_OK; @@ -701,36 +752,6 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; } pcache1LeaveMutex(pGroup); - /* Try to initialize the local bulk pagecache line allocation if using - ** separate caches and if nPage!=0 */ - if( pcache1.separateCache - && sqlite3GlobalConfig.nPage!=0 - && sqlite3GlobalConfig.pPage==0 - ){ - int szBulk; - char *zBulk; - sqlite3BeginBenignMalloc(); - if( sqlite3GlobalConfig.nPage>0 ){ - szBulk = pCache->szAlloc * sqlite3GlobalConfig.nPage; - }else{ - szBulk = -1024*sqlite3GlobalConfig.nPage; - } - zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); - sqlite3EndBenignMalloc(); - if( zBulk ){ - int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; - int i; - for(i=0; ipage.pBuf = zBulk; - pX->page.pExtra = &pX[1]; - pX->isBulkLocal = 1; - pX->pNext = pCache->pFree; - pCache->pFree = pX; - zBulk += pCache->szAlloc; - } - } - } if( pCache->nHash==0 ){ pcache1Destroy((sqlite3_pcache*)pCache); pCache = 0; @@ -753,7 +774,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->nMax = nMax; pCache->n90pct = pCache->nMax*9/10; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); } } @@ -771,7 +792,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pGroup->nMaxPage = savedMaxPage; pcache1LeaveMutex(pGroup); } @@ -1108,7 +1129,7 @@ static void pcache1Destroy(sqlite3_pcache *p){ assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1EnforceMaxPage(pGroup); + pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); diff --git a/src/pragma.c b/src/pragma.c index 1b4213844d..ffa0685727 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -721,6 +721,7 @@ void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); diff --git a/src/pragma.h b/src/pragma.h index bbf141ee2d..9e206dac49 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -86,7 +86,7 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "cache_size", /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, + /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) diff --git a/test/malloc5.test b/test/malloc5.test index 2b8ba74e5b..c9ee5dbc7d 100644 --- a/test/malloc5.test +++ b/test/malloc5.test @@ -41,6 +41,7 @@ ifcapable !memorymanage { sqlite3_soft_heap_limit 0 sqlite3 db test.db +db eval {PRAGMA cache_size=1} do_test malloc5-1.1 { # Simplest possible test. Call sqlite3_release_memory when there is exactly @@ -61,15 +62,14 @@ do_test malloc5-1.2 { # not the case before version 3.6.2). # sqlite3 db2 test.db - execsql { SELECT * FROM sqlite_master } db2 + execsql {PRAGMA cache_size=2; SELECT * FROM sqlite_master } db2 } {} do_test malloc5-1.3 { # Call [sqlite3_release_memory] when there is exactly one unused page # in the cache belonging to db2. # set ::pgalloc [sqlite3_release_memory] - expr $::pgalloc > 0 -} {1} +} {0} # The sizes of memory allocations from system malloc() might vary, # depending on the memory allocator algorithms used. The following @@ -231,6 +231,7 @@ do_test malloc5-4.1 { expr $nMaxBytes > 1000000 } {1} do_test malloc5-4.2 { + db eval {PRAGMA cache_size=1} db cache flush sqlite3_release_memory sqlite3_soft_heap_limit 100000 @@ -302,7 +303,7 @@ do_test malloc5-6.1.1 { sqlite3 db test.db execsql { PRAGMA page_size=1024; - PRAGMA default_cache_size=10; + PRAGMA default_cache_size=2; } execsql { PRAGMA temp_store = memory; @@ -325,24 +326,25 @@ do_test malloc5-6.1.1 { } forcecopy test.db test2.db sqlite3 db2 test2.db + db2 eval {PRAGMA cache_size=2} list \ [expr ([file size test.db]/1024)>20] [expr ([file size test2.db]/1024)>20] } {1 1} do_test malloc5-6.1.2 { list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2] -} {10 10} +} {2 2} do_test malloc5-6.2.1 { execsql {SELECT * FROM abc} db2 execsql {SELECT * FROM abc} db expr [nPage db] + [nPage db2] -} {20} +} {4} do_test malloc5-6.2.2 { # If we now try to reclaim some memory, it should come from the db2 cache. sqlite3_release_memory 3000 expr [nPage db] + [nPage db2] -} {17} +} {4} do_test malloc5-6.2.3 { # Access the db2 cache again, so that all the db2 pages have been used # more recently than all the db pages. Then try to reclaim 3000 bytes. @@ -350,7 +352,7 @@ do_test malloc5-6.2.3 { execsql { SELECT * FROM abc } db2 sqlite3_release_memory 3000 expr [nPage db] + [nPage db2] -} {17} +} {4} do_test malloc5-6.3.1 { # Now open a transaction and update 2 pages in the db2 cache. Then @@ -367,19 +369,19 @@ do_test malloc5-6.3.1 { } db2 execsql { SELECT * FROM abc } db expr [nPage db] + [nPage db2] -} {20} +} {4} do_test malloc5-6.3.2 { # Try to release 7700 bytes. This should release all the # non-dirty pages held by db2. sqlite3_release_memory [expr 7*1132] list [nPage db] [nPage db2] -} {10 3} +} {1 3} do_test malloc5-6.3.3 { # Try to release another 1000 bytes. This should come fromt the db # cache, since all three pages held by db2 are either in-use or diry. sqlite3_release_memory 1000 list [nPage db] [nPage db2] -} {9 3} +} {1 3} do_test malloc5-6.3.4 { # Now release 9900 more (about 9 pages worth). This should expunge # the rest of the db cache. But the db2 cache remains intact, because @@ -390,20 +392,20 @@ do_test malloc5-6.3.4 { sqlite3_release_memory 9900 } list [nPage db] [nPage db2] -} {0 3} +} {1 3} do_test malloc5-6.3.5 { # But if we are really insistent, SQLite will consent to call sync() # if there is no other option. UPDATE: As of 3.6.2, SQLite will not # call sync() in this scenario. So no further memory can be reclaimed. sqlite3_release_memory 1000 list [nPage db] [nPage db2] -} {0 3} +} {1 3} do_test malloc5-6.3.6 { # The referenced page (page 1 of the db2 cache) will not be freed no # matter how much memory we ask for: sqlite3_release_memory 31459 list [nPage db] [nPage db2] -} {0 3} +} {1 3} db2 close diff --git a/test/pcache.test b/test/pcache.test index 0766fce088..5d9d93edb7 100644 --- a/test/pcache.test +++ b/test/pcache.test @@ -73,7 +73,7 @@ do_test pcache-1.4 { do_test pcache-1.5 { sqlite3 db2 test.db - execsql "PRAGMA cache_size=10" db2 + execsql "PRAGMA cache_size; PRAGMA cache_size=10" db2 pcache_stats } {current 11 max 22 min 20 recyclable 1} diff --git a/test/pcache2.test b/test/pcache2.test index 1c15b8b56a..c59dfb25d4 100644 --- a/test/pcache2.test +++ b/test/pcache2.test @@ -36,13 +36,13 @@ do_test pcache2-1.1 { do_test pcache2-1.2 { forcedelete test.db test.db-journal sqlite3 db test.db - db eval {PRAGMA cache_size=10} + db eval {PRAGMA cache_size=10; SELECT 1 FROM sqlite_master;} lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1 } {2} do_test pcache2-1.3 { forcedelete test2.db test2.db-journal sqlite3 db2 test2.db - db2 eval {PRAGMA cache_size=50} + db2 eval {PRAGMA cache_size=50; SELECT 1 FROM sqlite_master;} lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 1 } {4} diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index cb3aed0fff..bbdf9da754 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -170,7 +170,6 @@ set pragma_def { IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: cache_size - FLAG: NeedSchema IF: !defined(SQLITE_OMIT_PAGER_PRAGMAS) NAME: mmap_size