From: dan Date: Tue, 1 Apr 2014 18:41:51 +0000 (+0000) Subject: When sorting data for a CREATE INDEX statement in single-threaded mode, assume that... X-Git-Tag: version-3.8.7~132^2~88 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=fad9f9a8a6920c795223d84cc3923920fe2269a9;p=thirdparty%2Fsqlite.git When sorting data for a CREATE INDEX statement in single-threaded mode, assume that keys are delivered to the sorter in primary key order. Also fix various comments that had fallen out of date. FossilOrigin-Name: 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 --- diff --git a/manifest b/manifest index 73c52a7c23..d0ef35d91d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. -D 2014-04-01T15:38:44.990 +C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate. +D 2014-04-01T18:41:51.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 +F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b +F src/vdbe.c eed2230017ca8ac5ca8fc221b38be036bba48499 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 +F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 +F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 +F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 -R 574aa31e9a2482d16cbf0b6e42831c35 +P 643c86a056168e39fcb7f39b8a72731f1eb246db +R c2ac1c0b48f44ef7c812322485a83f86 U dan -Z 53e6ec40073ce4b7a6ae1cb9efd4c01b +Z 23ba0058a55a91ed976e2b1f4e943278 diff --git a/manifest.uuid b/manifest.uuid index 957ecbe9a9..823d08f629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file +821d1ac4504243fa13b9e3c0d56361ad9fb80d78 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 10077e5018..26c2922ebd 100644 --- a/src/build.c +++ b/src/build.c @@ -2669,7 +2669,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) sqlite3KeyInfoRef(pKey), P4_KEYINFO); /* Open the table. Loop through all rows of the table, inserting index diff --git a/src/vdbe.c b/src/vdbe.c index de02d1f638..21da5007ab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3390,11 +3390,15 @@ case OP_OpenEphemeral: { break; } -/* Opcode: SorterOpen P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 P3 P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. */ case OP_SorterOpen: { VdbeCursor *pCx; @@ -3406,7 +3410,7 @@ case OP_SorterOpen: { pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); - rc = sqlite3VdbeSorterInit(db, pCx); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 043b56da59..03a303ca1b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -437,7 +437,7 @@ void sqlite3VdbeFrameDelete(VdbeFrame*); int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); -int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); diff --git a/src/vdbesort.c b/src/vdbesort.c index 746618cc25..81b2ff5c2b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -489,47 +489,27 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Argument pKeyInfo supplies the collation functions -** used by the comparison. If an error occurs, return an SQLite error code. -** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive -** value, depending on whether key1 is smaller, equal to or larger than key2. +** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. ** -** If the bOmitRowid argument is non-zero, assume both keys end in a rowid -** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid -** is true and key1 contains even a single NULL value, it is considered to -** be less than key2. Even if key2 also contains NULL values. +** Before returning, object (pThread->pUnpacked) is populated with the +** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it +** is assumed that the (pThread->pUnpacked) structure already contains the +** unpacked key to use as key2. ** -** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace -** has been allocated and contains an unpacked record that is used as key2. +** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. */ -static void vdbeSorterCompare( +static int vdbeSorterCompare( SorterThread *pThread, /* Thread context (for pKeyInfo) */ - int nIgnore, /* Ignore the last nIgnore fields */ const void *pKey1, int nKey1, /* Left side of comparison */ - const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ ){ - KeyInfo *pKeyInfo = pThread->pKeyInfo; UnpackedRecord *r2 = pThread->pUnpacked; - int i; - if( pKey2 ){ - sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); - } - - if( nIgnore ){ - r2->nField = pKeyInfo->nField - nIgnore; - assert( r2->nField>0 ); - for(i=0; inField; i++){ - if( r2->aMem[i].flags & MEM_Null ){ - *pRes = -1; - return; - } - } - assert( r2->default_rc==0 ); + sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2); } - - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } /* @@ -568,8 +548,8 @@ static int vdbeSorterDoCompare( }else{ int res; assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ - vdbeSorterCompare( - pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + res = vdbeSorterCompare( + pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -585,7 +565,7 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. */ -int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ int i; /* Used to iterate through aThread[] */ int mxCache; /* Cache size */ @@ -608,6 +588,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; + if( nField && nWorker==0 ) pKeyInfo->nField = nField; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nThread = nWorker + 1; @@ -806,7 +787,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pThread, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); + res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; @@ -1075,8 +1056,8 @@ static int vdbeSorterNext( }else if( pIter2->pFile==0 ){ iRes = -1; }else{ - vdbeSorterCompare(pThread, 0, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + iRes = vdbeSorterCompare(pThread, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey ); } @@ -1596,6 +1577,9 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** passed as the first argument currently points to. For the purposes of ** the comparison, ignore the rowid field at the end of each record. ** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Evn if pVal also contains NULL values. +** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter @@ -1608,10 +1592,23 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterThread *pMain = &pSorter->aThread[0]; + UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + assert( r2->nField>=pKeyInfo->nField-nIgnore ); + r2->nField = pKeyInfo->nField-nIgnore; + pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; inField; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2, 0); return SQLITE_OK; } diff --git a/test/sort2.test b/test/sort2.test index 2a77b61ed1..f8bfb0fe51 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -33,6 +33,20 @@ do_execsql_test 1 { 20000 100 20000 100 20000 100 20000 100 20000 100 } +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; +} + +do_execsql_test 2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); +} + db close sqlite3_shutdown sqlite3_config_worker_threads 0