When sorting data for a CREATE INDEX statement in single-threaded mode, assume that...

FossilOrigin-Name: 821d1ac4504243fa13b9e3c0d56361ad9fb80d78

diff --git a/manifest b/manifest
index 73c52a7c237422eed2a65e8c71ce55168277452c..d0ef35d91df0ac3c81e510fff43d63afee8b62a0 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test.
-D 2014-04-01T15:38:44.990
+C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate.
+D 2014-04-01T18:41:51.759
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d
 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba
 F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4
-F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0
+F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1
 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a
@@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115
 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c
 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf
 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179
-F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b
+F src/vdbe.c eed2230017ca8ac5ca8fc221b38be036bba48499
 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94
-F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7
+F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d
 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4
 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea
 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa
 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447
-F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1
+F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c
 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
@@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d
 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f
 F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24
 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a
-F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574
+F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8
 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
 F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8
@@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425
-R 574aa31e9a2482d16cbf0b6e42831c35
+P 643c86a056168e39fcb7f39b8a72731f1eb246db
+R c2ac1c0b48f44ef7c812322485a83f86
 U dan
-Z 53e6ec40073ce4b7a6ae1cb9efd4c01b
+Z 23ba0058a55a91ed976e2b1f4e943278

diff --git a/manifest.uuid b/manifest.uuid
index 957ecbe9a900deeb24b873de559bef47ebe97b0a..823d08f6292699beca1b98692b236919b511aeef 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-643c86a056168e39fcb7f39b8a72731f1eb246db
\ No newline at end of file
+821d1ac4504243fa13b9e3c0d56361ad9fb80d78
\ No newline at end of file

diff --git a/src/build.c b/src/build.c
index 10077e50187b87b3503e385310434aeaae628458..26c2922ebdb37ea01a0f531daed5564c0569e8e0 100644 (file)
--- a/src/build.c
+++ b/src/build.c
@@ -2669,7 +2669,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
 
   /* Open the sorter cursor if we are to use one. */
   iSorter = pParse->nTab++;
-  sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)
+  sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*)
                     sqlite3KeyInfoRef(pKey), P4_KEYINFO);
 
   /* Open the table. Loop through all rows of the table, inserting index

diff --git a/src/vdbe.c b/src/vdbe.c
index de02d1f6385da73fff83e672e3a136c2b79f4710..21da5007ab5c3061c472519ee62afc598b47b237 100644 (file)
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3390,11 +3390,15 @@ case OP_OpenEphemeral: {
   break;
 }
 
-/* Opcode: SorterOpen P1 P2 * P4 *
+/* Opcode: SorterOpen P1 P2 P3 P4 *
 **
 ** This opcode works like OP_OpenEphemeral except that it opens
 ** a transient index that is specifically designed to sort large
 ** tables using an external merge-sort algorithm.
+**
+** If argument P3 is non-zero, then it indicates that the sorter may
+** assume that a stable sort considering the first P3 fields of each
+** key is sufficient to produce the required results.
 */
 case OP_SorterOpen: {
   VdbeCursor *pCx;
@@ -3406,7 +3410,7 @@ case OP_SorterOpen: {
   pCx->pKeyInfo = pOp->p4.pKeyInfo;
   assert( pCx->pKeyInfo->db==db );
   assert( pCx->pKeyInfo->enc==ENC(db) );
-  rc = sqlite3VdbeSorterInit(db, pCx);
+  rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx);
   break;
 }
 

diff --git a/src/vdbeInt.h b/src/vdbeInt.h
index 043b56da59b0fa36dea74617b4cae83bec2d348c..03a303ca1b086599bf66fc9813111e3b8b2c3e53 100644 (file)
--- a/src/vdbeInt.h
+++ b/src/vdbeInt.h
@@ -437,7 +437,7 @@ void sqlite3VdbeFrameDelete(VdbeFrame*);
 int sqlite3VdbeFrameRestore(VdbeFrame *);
 int sqlite3VdbeTransferError(Vdbe *p);
 
-int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *);
+int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *);
 void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *);
 void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *);
 int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *);

diff --git a/src/vdbesort.c b/src/vdbesort.c
index 746618cc257585117ff0cb49d4b3fbe4255e35b4..81b2ff5c2bda0c38b4db726be24768216bc609d4 100644 (file)
--- a/src/vdbesort.c
+++ b/src/vdbesort.c
@@ -489,47 +489,27 @@ static int vdbeSorterIterInit(
 
 /*
 ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, 
-** size nKey2 bytes).  Argument pKeyInfo supplies the collation functions
-** used by the comparison. If an error occurs, return an SQLite error code.
-** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive
-** value, depending on whether key1 is smaller, equal to or larger than key2.
+** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences
+** used by the comparison. Return the result of the comparison.
 **
-** If the bOmitRowid argument is non-zero, assume both keys end in a rowid
-** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid
-** is true and key1 contains even a single NULL value, it is considered to
-** be less than key2. Even if key2 also contains NULL values.
+** Before returning, object (pThread->pUnpacked) is populated with the
+** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it 
+** is assumed that the (pThread->pUnpacked) structure already contains the 
+** unpacked key to use as key2.
 **
-** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace
-** has been allocated and contains an unpacked record that is used as key2.
+** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set
+** to SQLITE_NOMEM.
 */
-static void vdbeSorterCompare(
+static int vdbeSorterCompare(
   SorterThread *pThread,          /* Thread context (for pKeyInfo) */
-  int nIgnore,                    /* Ignore the last nIgnore fields */
   const void *pKey1, int nKey1,   /* Left side of comparison */
-  const void *pKey2, int nKey2,   /* Right side of comparison */
-  int *pRes                       /* OUT: Result of comparison */
+  const void *pKey2, int nKey2    /* Right side of comparison */
 ){
-  KeyInfo *pKeyInfo = pThread->pKeyInfo;
   UnpackedRecord *r2 = pThread->pUnpacked;
-  int i;
-
   if( pKey2 ){
-    sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2);
-  }
-
-  if( nIgnore ){
-    r2->nField = pKeyInfo->nField - nIgnore;
-    assert( r2->nField>0 );
-    for(i=0; i<r2->nField; i++){
-      if( r2->aMem[i].flags & MEM_Null ){
-        *pRes = -1;
-        return;
-      }
-    }
-    assert( r2->default_rc==0 );
+    sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2);
   }
-
-  *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0);
+  return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0);
 }
 
 /*
@@ -568,8 +548,8 @@ static int vdbeSorterDoCompare(
   }else{
     int res;
     assert( pThread->pUnpacked!=0 );  /* allocated in vdbeSorterThreadMain() */
-    vdbeSorterCompare(
-        pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res
+    res = vdbeSorterCompare(
+        pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey
     );
     if( res<=0 ){
       iRes = i1;
@@ -585,7 +565,7 @@ static int vdbeSorterDoCompare(
 /*
 ** Initialize the temporary index cursor just opened as a sorter cursor.
 */
-int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
+int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){
   int pgsz;                       /* Page size of main database */
   int i;                          /* Used to iterate through aThread[] */
   int mxCache;                    /* Cache size */
@@ -608,6 +588,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){
     pKeyInfo = (KeyInfo*)((u8*)pSorter + sz);
     memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo);
     pKeyInfo->db = 0;
+    if( nField && nWorker==0 ) pKeyInfo->nField = nField;
     pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
 
     pSorter->nThread = nWorker + 1;
@@ -806,7 +787,7 @@ static void vdbeSorterMerge(
 
   while( p1 && p2 ){
     int res;
-    vdbeSorterCompare(pThread, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res);
+    res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal);
     if( res<=0 ){
       *pp = p1;
       pp = &p1->u.pNext;
@@ -1075,8 +1056,8 @@ static int vdbeSorterNext(
       }else if( pIter2->pFile==0 ){
         iRes = -1;
       }else{
-        vdbeSorterCompare(pThread, 0,
-            pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes
+        iRes = vdbeSorterCompare(pThread, 
+            pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey
         );
       }
 
@@ -1596,6 +1577,9 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){
 ** passed as the first argument currently points to. For the purposes of
 ** the comparison, ignore the rowid field at the end of each record.
 **
+** If the sorter cursor key contains any NULL values, consider it to be
+** less than pVal. Evn if pVal also contains NULL values.
+**
 ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM).
 ** Otherwise, set *pRes to a negative, zero or positive value if the
 ** key in pVal is smaller than, equal to or larger than the current sorter
@@ -1608,10 +1592,23 @@ int sqlite3VdbeSorterCompare(
   int *pRes                       /* OUT: Result of comparison */
 ){
   VdbeSorter *pSorter = pCsr->pSorter;
-  SorterThread *pMain = &pSorter->aThread[0];
+  UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked;
+  KeyInfo *pKeyInfo = pCsr->pKeyInfo;
+  int i;
   void *pKey; int nKey;           /* Sorter key to compare pVal with */
 
+  assert( r2->nField>=pKeyInfo->nField-nIgnore );
+  r2->nField = pKeyInfo->nField-nIgnore;
+
   pKey = vdbeSorterRowkey(pSorter, &nKey);
-  vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes);
+  sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2);
+  for(i=0; i<r2->nField; i++){
+    if( r2->aMem[i].flags & MEM_Null ){
+      *pRes = -1;
+      return SQLITE_OK;
+    }
+  }
+
+  *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2, 0);
   return SQLITE_OK;
 }

diff --git a/test/sort2.test b/test/sort2.test
index 2a77b61ed1923a0772fdc8f59cdf30252ef8243f..f8bfb0fe51256920849bd62aad2cdd0f7923228e 100644 (file)
--- a/test/sort2.test
+++ b/test/sort2.test
@@ -33,6 +33,20 @@ do_execsql_test 1 {
   20000 100 20000 100 20000 100 20000 100 20000 100
 }
 
+do_execsql_test 2.1 {
+  CREATE TABLE t1(a, b);
+  WITH r(x,y) AS (
+    SELECT 1, randomblob(100)
+    UNION ALL
+    SELECT x+1, randomblob(100) FROM r
+    LIMIT 10000
+  ) INSERT INTO t1 SELECT * FROM r;
+}
+
+do_execsql_test 2.2 {
+  CREATE UNIQUE INDEX i1 ON t1(b, a);
+}
+
 db close
 sqlite3_shutdown
 sqlite3_config_worker_threads 0