Change the recursive common table expression algorithm to use a queue instead

of a pair of tables.  Runs about 25% faster on the sudoku solver query.
The OP_SwapCursors opcode is no longer required.  The current implementation
uses just a fifo, but the plan is to change it into a queue that will support
ORDER BY and LIMIT in a recursive query.

FossilOrigin-Name: b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97

diff --git a/manifest b/manifest
index 9793a62dd6aacfef146eb220fce0a42819f62dfb..955bc3f8a90e8c8310ec88fcfd54f893ba9da462 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Remove\sthe\sundocumented\srequirement\sfor\sapplications\sthat\suse\san\sSQLITE_ENABLE_SQLLOG\sbuild\sto\sdefine\sa\ssqlite3_init_sqllog()\sfunction.
-D 2014-01-21T15:04:47.807
+C Change\sthe\srecursive\scommon\stable\sexpression\salgorithm\sto\suse\sa\squeue\sinstead\nof\sa\spair\sof\stables.\s\sRuns\sabout\s25%\sfaster\son\sthe\ssudoku\ssolver\squery.\s\nThe\sOP_SwapCursors\sopcode\sis\sno\slonger\srequired.\s\sThe\scurrent\simplementation\nuses\sjust\sa\sfifo,\sbut\sthe\splan\sis\sto\schange\sit\sinto\sa\squeue\sthat\swill\ssupport\s\nORDER\sBY\sand\sLIMIT\sin\sa\srecursive\squery.
+D 2014-01-21T22:25:45.721
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -219,8 +219,8 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269
 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c a27ac21844df3123b7c1e89d79cd7034d4eb0e8e
-F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48
+F src/select.c f7b1558aae71d4f6ff48ad91122185d065730ba6
+F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca
 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
@@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb
 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269
 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c
 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179
-F src/vdbe.c 98d96d04d9a2bef78ca850be1053dc91d031338a
+F src/vdbe.c dede894c2990329f8bc5a70da7de44ce8c3c6bf5
 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26
 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56
 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad
@@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
-F src/where.c 56f85486bc8d0cb57fc15e5db2a58d1dfa1114cf
+F src/where.c d908f4e9e45b567e87a890959ebef01187fab46f
 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -1152,7 +1152,10 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 7d9e22187daaa3160b875a1df17b924969bf718e
-R 72db5d0300da6235ddfc19ebd0af857f
-U dan
-Z 21763f37f81635b2fe9a520f9fd3a652
+P 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae
+R ead1532b7d5d88209851a3d527991f22
+T *branch * cte-via-queue
+T *sym-cte-via-queue *
+T -sym-trunk *
+U drh
+Z 386bc5602f3047b62d7a78b288244cbd

diff --git a/manifest.uuid b/manifest.uuid
index 89ddacefffc483e3920551622419220327744b4b..76656dfd03780dc56a5d28a4afa56fceaa40487f 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5e43bf013253921e4dfbe71de11ee7ed4b3e7eae
\ No newline at end of file
+b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97
\ No newline at end of file

diff --git a/src/select.c b/src/select.c
index dd58ed22e637a51305865b2307549a6047826699..e6dd0e7a0a93c3aa900a7aeda95f20906018ca37 100644 (file)
--- a/src/select.c
+++ b/src/select.c
@@ -1792,20 +1792,21 @@ static int multiSelect(
 
 #ifndef SQLITE_OMIT_CTE
   if( p->selFlags & SF_Recursive ){
-    SrcList *pSrc = p->pSrc;
-    int nCol = p->pEList->nExpr;
-    int addrNext;
-    int addrSwap;
-    int iCont, iBreak;
-    int tmp1;                     /* Intermediate table */
-    int tmp2;                     /* Next intermediate table */
-    int tmp3 = 0;                 /* To ensure unique results if UNION */
-    int eDest = SRT_Table;
-    SelectDest tmp2dest;
-    int i;
+    SrcList *pSrc = p->pSrc;      /* The FROM clause of the recursive query */
+    int nCol = p->pEList->nExpr;  /* Number of columns in the CTE */
+    int addrTop;                  /* Top of the loop */
+    int addrCont, addrBreak;      /* CONTINUE and BREAK addresses */
+    int iCurrent;                 /* The Current table */
+    int regCurrent;               /* Register holding Current table */
+    int iQueue;                   /* The Queue table */
+    int iDistinct;                /* To ensure unique results if UNION */
+    int eDest;                    /* How to write to Queue */
+    SelectDest destQueue;         /* SelectDest targetting the Queue table */
+    int i;                        /* Loop counter */
 
     /* Check that there is no ORDER BY or LIMIT clause. Neither of these 
-    ** are supported on recursive queries.  */
+    ** are currently supported on recursive queries.
+    */
     assert( p->pOffset==0 || p->pLimit );
     if( p->pOrderBy || p->pLimit ){
       sqlite3ErrorMsg(pParse, "%s in a recursive query",
@@ -1817,56 +1818,66 @@ static int multiSelect(
     if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){
       goto multi_select_end;
     }
-    iBreak = sqlite3VdbeMakeLabel(v);
-    iCont = sqlite3VdbeMakeLabel(v);
+    addrBreak = sqlite3VdbeMakeLabel(v);
+    addrCont = sqlite3VdbeMakeLabel(v);
 
+    /* Locate the cursor number of the Current table */
     for(i=0; ALWAYS(i<pSrc->nSrc); i++){
       if( pSrc->a[i].isRecursive ){
-        tmp1 = pSrc->a[i].iCursor;
+        iCurrent = pSrc->a[i].iCursor;
         break;
       }
     }
 
-    tmp2 = pParse->nTab++;
+    /* Allocate cursors for Queue and Distinct.  The cursor number for
+    ** the Distinct table must be exactly one greater than Queue in order
+    ** for the SRT_DistTable destination to work. */
+    iQueue = pParse->nTab++;
     if( p->op==TK_UNION ){
       eDest = SRT_DistTable;
-      tmp3 = pParse->nTab++;
+      iDistinct = pParse->nTab++;
+    }else{
+      eDest = SRT_Table;
+      iDistinct = 0;
     }
-    sqlite3SelectDestInit(&tmp2dest, eDest, tmp2);
-
-    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol);
-    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol);
-    if( tmp3 ){
-      p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0);
+    sqlite3SelectDestInit(&destQueue, eDest, iQueue);
+
+    /* Allocate cursors for Current, Queue, and iDistinct. */
+    regCurrent = ++pParse->nMem;
+    sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol);
+    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol);
+    if( iDistinct ){
+      p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0);
       p->selFlags |= SF_UsesEphemeral;
     }
 
-    /* Store the results of the initial SELECT in tmp2. */
-    rc = sqlite3Select(pParse, pPrior, &tmp2dest);
+    /* Store the results of the initial SELECT in Queue. */
+    rc = sqlite3Select(pParse, pPrior, &destQueue);
     if( rc ) goto multi_select_end;
 
-    /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return 
-    ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this
-    ** point, the recursive query has finished - jump to address iBreak.  */
-    addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2);
-    sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak);
-    addrNext = sqlite3VdbeCurrentAddr(v);
-    selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr,
-        0, 0, &dest, iCont, iBreak);
-    sqlite3VdbeResolveLabel(v, iCont);
-    sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext);
-
-    /* Execute the recursive SELECT. Store the results in tmp2. While this
-    ** SELECT is running, the contents of tmp1 are read by recursive 
-    ** references to the current CTE.  */
+    /* Find the next row in the Queue and output that row */
+    addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak);
+    selectInnerLoop(pParse, p, p->pEList, iQueue, p->pEList->nExpr,
+        0, 0, &dest, addrCont, addrBreak);
+    sqlite3VdbeResolveLabel(v, addrCont);
+
+    /* Transfer the next row in Queue over to Current */
+    sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */
+    sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent);
+    sqlite3VdbeAddOp1(v, OP_Delete, iQueue);
+
+    /* Execute the recursive SELECT taking the single row in Current as
+    ** the value for the CTE. Store the results in the Queue.
+    */
     p->pPrior = 0;
-    rc = sqlite3Select(pParse, p, &tmp2dest);
+    rc = sqlite3Select(pParse, p, &destQueue);
     assert( p->pPrior==0 );
     p->pPrior = pPrior;
     if( rc ) goto multi_select_end;
 
-    sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap);
-    sqlite3VdbeResolveLabel(v, iBreak);
+    /* Keep running the loop until the Queue is empty */
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop);
+    sqlite3VdbeResolveLabel(v, addrBreak);
   }else
 #endif
 

diff --git a/src/shell.c b/src/shell.c
index b5ce90208c35c2bee702bc433a4034d984097f3a..1c4c4ad3e305ce8a5a9a6eb71657d336787b3911 100644 (file)
--- a/src/shell.c
+++ b/src/shell.c
@@ -1177,7 +1177,7 @@ static int str_in_array(const char *zStr, const char **azArray){
 **
 **     * For each "Goto", if the jump destination is earlier in the program
 **       and ends on one of:
-**          Yield  SeekGt  SeekLt  RowSetRead
+**          Yield  SeekGt  SeekLt  RowSetRead  Rewind
 **       then indent all opcodes between the earlier instruction
 **       and "Goto" by 2 spaces.
 */
@@ -1189,7 +1189,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){
   int iOp;                        /* Index of operation in p->aiIndent[] */
 
   const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 };
-  const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", 0 };
+  const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 };
   const char *azGoto[] = { "Goto", 0 };
 
   /* Try to figure out if this is really an EXPLAIN statement. If this
@@ -1226,7 +1226,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){
       for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
     }
     if( str_in_array(zOp, azGoto) && p2op<p->nIndent && abYield[p2op] ){
-      for(i=p2op; i<iOp; i++) p->aiIndent[i] += 2;
+      for(i=p2op+1; i<iOp; i++) p->aiIndent[i] += 2;
     }
   }
 

diff --git a/src/vdbe.c b/src/vdbe.c
index 86aae3c65d3ce876c2687778f68121462544490c..10a70750e30107c4102b487a6d8b8f2ff431c4f4 100644 (file)
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -3369,33 +3369,6 @@ case OP_OpenEphemeral: {
   break;
 }
 
-#ifndef SQLITE_OMIT_CTE
-/* Opcode: SwapCursors P1 P2 * * *
-**
-** Parameters P1 and P2 are both cursors opened by the OpenEphemeral
-** opcode. This opcode deletes the contents of epheremal table P1,
-** then renames P2 to P1 and P1 to P2. In other words, following this
-** opcode cursor P2 is open on an empty table and P1 is open on the
-** table that was initially accessed by P2.
-*/
-case OP_SwapCursors: {
-  Mem tmp;
-  VdbeCursor *pTmp;
-
-  tmp = p->aMem[p->nMem - pOp->p1];
-  p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2];
-  p->aMem[p->nMem - pOp->p2] = tmp;
-
-  pTmp = p->apCsr[pOp->p1];
-  p->apCsr[pOp->p1] = p->apCsr[pOp->p2];
-  p->apCsr[pOp->p2] = pTmp;
-
-  assert( pTmp->isTable );
-  rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT, 0);
-  break;
-}
-#endif /* ifndef SQLITE_OMIT_CTE */
-
 /* Opcode: SorterOpen P1 * * P4 *
 **
 ** This opcode works like OP_OpenEphemeral except that it opens
@@ -4393,7 +4366,6 @@ case OP_NullRow: {
   pC->nullRow = 1;
   pC->rowidIsValid = 0;
   pC->cacheStatus = CACHE_STALE;
-  assert( pC->pCursor || pC->pVtabCursor );
   if( pC->pCursor ){
     sqlite3BtreeClearCursor(pC->pCursor);
   }

diff --git a/src/where.c b/src/where.c
index 86376c495575ef57e7d2c083c9161dcffcdc81e8..f2d72760d31c6a5a5cb31f9cb0662c77f845f176 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -3410,10 +3410,14 @@ static Bitmask codeOneLoopStart(
     static const u8 aStep[] = { OP_Next, OP_Prev };
     static const u8 aStart[] = { OP_Rewind, OP_Last };
     assert( bRev==0 || bRev==1 );
-    pLevel->op = aStep[bRev];
-    pLevel->p1 = iCur;
-    pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
-    pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
+    if( pTabItem->isRecursive ){
+      pLevel->op = OP_Noop;
+    }else{
+      pLevel->op = aStep[bRev];
+      pLevel->p1 = iCur;
+      pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
+      pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
+    }
   }
 
   /* Insert code to test every subexpression that can be completely