From: dan Date: Mon, 31 Mar 2014 19:57:34 +0000 (+0000) Subject: Add the SQLITE_MAX_WORKER_THREADS compile time option. And the SQLITE_CONFIG_WORKER_T... X-Git-Tag: version-3.8.7~132^2~91 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b3f56fdb69201329c96ef9bb904a5df46fc62c0d;p=thirdparty%2Fsqlite.git Add the SQLITE_MAX_WORKER_THREADS compile time option. And the SQLITE_CONFIG_WORKER_THREADS sqlite3_config() switch. FossilOrigin-Name: 2774710df8cd2bfaca49888c69f1b01c0ddadf9a --- diff --git a/manifest b/manifest index 0b2bd63f50..4a4f37cc87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\svdbesort.c. -D 2014-03-29T10:01:58.802 +C Add\sthe\sSQLITE_MAX_WORKER_THREADS\scompile\stime\soption.\sAnd\sthe\sSQLITE_CONFIG_WORKER_THREADS\ssqlite3_config()\sswitch. +D 2014-03-31T19:57:34.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 +F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db +F src/main.c d3655832585baef4c2356529a5c6ca5ca3bd7c1f F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -219,10 +219,10 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df -F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 +F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3f5190a4e07ca227035334da8d66ebe227071528 +F src/sqliteInt.h 7f42c2792b951db22fa189bbed828a5e3b38789c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b96d62f88c06d4fa980a4a92685d1b130c4c84d3 +F src/threads.c 6992f70cab8d5d8451a6b5641a9256d1749af87b F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2881297f4acdba5908078c5d7f00635288a1ca08 +F src/vdbesort.c b4d6133bada297e118492420346f83cd76c6da31 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a683c05f6250389e84b980b16559e162ba1a27c2 -R d8e63408790a442d33eec2a57272c54d +P 18d1b402f2dbe78f1a1113bb356b710e348365ef +R 8b3347b8372cc17a3226330524ab6da6 U dan -Z 6c5e71d99f167a1ded5f09353b3e0513 +Z 11cb5db8cbfdddf6047ddaf9c26850df diff --git a/manifest.uuid b/manifest.uuid index 1bf7c269ad..096bf0eab2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d1b402f2dbe78f1a1113bb356b710e348365ef \ No newline at end of file +2774710df8cd2bfaca49888c69f1b01c0ddadf9a \ No newline at end of file diff --git a/src/global.c b/src/global.c index 1ee3f6436f..4ed597d292 100644 --- a/src/global.c +++ b/src/global.c @@ -167,6 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ + SQLITE_MAX_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/main.c b/src/main.c index 9e83d4963a..c1eaa6849a 100644 --- a/src/main.c +++ b/src/main.c @@ -515,6 +515,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_WORKER_THREADS: { + int n = va_arg(ap, int); + if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; + if( n>=0 ) sqlite3GlobalConfig.nWorker = n; + break; + } + default: { rc = SQLITE_ERROR; break; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5d2c87552c..78aa9c36e1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1715,6 +1715,16 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_WORKER_THREADS]] +**
SQLITE_CONFIG_WORKER_THREADS +**
^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int. +** It is used to set the number of background worker threads that may be +** launched when sorting large amounts of data. A value of 0 means launch +** no background threads at all. The maximum number of background threads +** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS +** pre-processor option. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1739,6 +1749,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_WORKER_THREADS 24 /* int nWorker */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 41219c710a..5f85e80d82 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -422,6 +422,19 @@ # define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 +# undef SQLITE_MAX_WORKER_THREADS +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif + + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -2685,6 +2698,7 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ + int nWorker; /* Number of worker threads to use */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ diff --git a/src/threads.c b/src/threads.c index cb148b6c6f..96cb17cef7 100644 --- a/src/threads.c +++ b/src/threads.c @@ -27,6 +27,8 @@ */ #include "sqliteInt.h" +#if SQLITE_MAX_WORKER_THREADS>0 + /********************************* Unix Pthreads ****************************/ #if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 @@ -215,3 +217,4 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ #endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ /****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c6927f87a7..84c6fadc94 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -26,14 +26,6 @@ typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; -/* -** Maximum number of threads to use. Setting this value to 1 forces all -** operations to be single-threaded. -*/ -#ifndef SQLITE_MAX_SORTER_THREAD -# define SQLITE_MAX_SORTER_THREAD 4 -#endif - /* ** Candidate values for SorterThread.eWork */ @@ -48,9 +40,10 @@ typedef struct FileWriter FileWriter; ** is configured and passed to vdbeSorterThreadMain() - either directly by ** the main thread or via a background thread. ** -** Exactly SQLITE_MAX_SORTER_THREAD instances of this structure are allocated +** Exactly SorterThread.nThread instances of this structure are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. +** way. SorterThread.nThread is set to the number of worker threads allowed +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SorterThread.bDone ** is set to 0 and the SorterThread.pThread variable set to point to the @@ -59,7 +52,7 @@ typedef struct FileWriter FileWriter; ** exits. SorterThread.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SQLITE_MAX_SORTER_THREAD-1]) +** One object (specifically, VdbeSorter.aThread[SorterThread.nThread-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SorterThread.eWork, @@ -187,7 +180,8 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ - SorterThread aThread[SQLITE_MAX_SORTER_THREAD]; + int nThread; /* Size of aThread[] array */ + SorterThread aThread[1]; }; /* @@ -572,7 +566,7 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ vdbeSorterCompare( pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); @@ -597,21 +591,26 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; + int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); - pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SorterThread); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->pSorter = pSorter; if( pSorter==0 ){ rc = SQLITE_NOMEM; }else{ - pKeyInfo = (KeyInfo*)&pSorter[1]; + pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - for(i=0; inThread = nWorker + 1; + for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; pThread->pKeyInfo = pKeyInfo; pThread->pVfs = db->pVfs; @@ -674,10 +673,11 @@ static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){ /* ** Join all threads. */ +#if SQLITE_MAX_WORKER_THREADS>0 static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pThread ){ void *pRet; @@ -690,6 +690,9 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ } return rc; } +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +#endif /* ** Allocate a new SorterMerger object with space for nIter iterators. @@ -739,7 +742,7 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; vdbeSorterThreadCleanup(db, pThread); } @@ -1246,8 +1249,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ SorterThread *pThread; /* Thread context used to create new PMA */ pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ pThread = &pSorter->aThread[i]; +#if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; assert( pThread->pThread ); @@ -1258,11 +1262,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ rc = SQLITE_PTR_TO_INT(pRet); } } +#endif if( pThread->pThread==0 ) break; } if( rc==SQLITE_OK ){ - int bUseFg = (bFg || i==(SQLITE_MAX_SORTER_THREAD-1)); + int bUseFg = (bFg || i==(pSorter->nThread-1)); assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; @@ -1277,6 +1282,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->aMemory = aMem; } +#if SQLITE_MAX_WORKER_THREADS>0 if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; @@ -1290,7 +1296,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } } rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); - }else{ + }else +#endif + { /* Use the foreground thread for this operation */ u8 *aMem; rc = vdbeSorterRunThread(pThread); @@ -1370,7 +1378,9 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*)( + aNew + ((u8*)pSorter->pRecord - pSorter->aMemory) + ); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1399,7 +1409,7 @@ int sqlite3VdbeSorterWrite( static int vdbeSorterCountPMA(VdbeSorter *pSorter){ int nPMA = 0; int i; - for(i=0; inThread; i++){ nPMA += pSorter->aThread[i].nPMA; } return nPMA; @@ -1446,19 +1456,21 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge ** some of them together so that this is no longer the case. */ - assert( SORTER_MAX_MERGE_COUNT>=SQLITE_MAX_SORTER_THREAD ); if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; - for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/SQLITE_MAX_SORTER_THREAD; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; - if( i<(SQLITE_MAX_SORTER_THREAD-1) ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( i<(pSorter->nThread-1) ){ void *pCtx = (void*)pThread; rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); - }else{ + }else +#endif + { rc = vdbeSorterRunThread(pThread); } } @@ -1475,7 +1487,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ int nIter = 0; /* Number of iterators used */ int i; SorterMerger *pMerger; - for(i=0; inThread; i++){ nIter += pSorter->aThread[i].nPMA; } @@ -1485,7 +1497,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else{ int iIter = 0; int iThread = 0; - for(iThread=0; iThreadnThread; iThread++){ int iPMA; i64 iReadOff = 0; SorterThread *pThread = &pSorter->aThread[iThread];