Experiments with the optimization of ORDER BY and GROUP BY clauses.

FossilOrigin-Name: b150902579d708b454efd5f8750e26a816f7f1a6

diff --git a/manifest b/manifest
index bb216398015bcdf1b95254dbcef4df41446035d6..6f347f2fefa43b0b02298d22bf93d94cc20dde37 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c.
-D 2014-03-17T15:06:57.359
+C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses.
+D 2014-03-18T15:30:27.660
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
-F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259
-F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138
+F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2
+F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8
-R f35820126ad6a1c360258186873a2e79
+P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36
+R 8abedb680a805ab9b5e400a1ac18119b
+T *branch * orderby-planning
+T *sym-orderby-planning *
+T -sym-trunk *
 U drh
-Z b6382160ccdf74a44bc9653763bb82ef
+Z 6c19349f1c38f37cb3b2bffb6095d2a0

diff --git a/manifest.uuid b/manifest.uuid
index 3e3e1d6ac78fb706c6ef5a07419340371dc2b19c..26699b8468ebaeee91f9299680592b94e145a5e2 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-ecd9d3f9453be0bb8e312d8027fd1a9e55882f36
\ No newline at end of file
+b150902579d708b454efd5f8750e26a816f7f1a6
\ No newline at end of file

diff --git a/src/where.c b/src/where.c
index 6cd9c167a19f43565386765d8f60414ef98e3274..bed58ac75238bcc447724618ec6719a1ca356bff 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -4506,8 +4506,8 @@ static int whereLoopAddVirtual(
       pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr;
       pIdxInfo->needToFreeIdxStr = 0;
       pNew->u.vtab.idxStr = pIdxInfo->idxStr;
-      pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0)
-                                     && pIdxInfo->orderByConsumed);
+      pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
+                                      pIdxInfo->nOrderBy : 0);
       pNew->rSetup = 0;
       pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
       pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
@@ -4668,11 +4668,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
 /*
 ** Examine a WherePath (with the addition of the extra WhereLoop of the 5th
 ** parameters) to see if it outputs rows in the requested ORDER BY
-** (or GROUP BY) without requiring a separate sort operation.  Return:
+** (or GROUP BY) without requiring a separate sort operation.  Return N:
 ** 
-**    0:  ORDER BY is not satisfied.  Sorting required
-**    1:  ORDER BY is satisfied.      Omit sorting
-**   -1:  Unknown at this time
+**   N>0:   N terms of the ORDER BY clause are satisfied
+**   N==0:  No terms of the ORDER BY clause are satisfied
+**   N<0:   Unknown yet how many terms of ORDER BY might be satisfied.   
 **
 ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as
 ** strict.  With GROUP BY and DISTINCT the only requirement is that
@@ -4682,7 +4682,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
 ** the pOrderBy terms can be matched in any order.  With ORDER BY, the 
 ** pOrderBy terms must be matched in strict left-to-right order.
 */
-static int wherePathSatisfiesOrderBy(
+static i8 wherePathSatisfiesOrderBy(
   WhereInfo *pWInfo,    /* The WHERE clause */
   ExprList *pOrderBy,   /* ORDER BY or GROUP BY or DISTINCT clause to check */
   WherePath *pPath,     /* The WherePath to check */
@@ -4915,7 +4915,7 @@ static int wherePathSatisfiesOrderBy(
       }
     }
   } /* End the loop over all WhereLoops from outer-most down to inner-most */
-  if( obSat==obDone ) return 1;
+  if( obSat==obDone ) return nOrderBy;
   if( !isOrderDistinct ) return 0;
   return -1;
 }
@@ -5001,12 +5001,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
   ** to sqlite3WhereBegin() was concerned about sorting */
   rSortCost = 0;
   if( pWInfo->pOrderBy==0 || nRowEst==0 ){
-    aFrom[0].isOrderedValid = 1;
+    aFrom[0].isOrdered = 0;
   }else{
     /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the
     ** number of output rows. The 48 is the expected size of a row to sort. 
     ** FIXME:  compute a better estimate of the 48 multiplier based on the
     ** result set expressions. */
+    aFrom[0].isOrdered = -1;
     rSortCost = nRowEst + estLog(nRowEst);
     WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost));
   }
@@ -5020,8 +5021,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){
         Bitmask maskNew;
         Bitmask revMask = 0;
-        u8 isOrderedValid = pFrom->isOrderedValid;
-        u8 isOrdered = pFrom->isOrdered;
+        i8 isOrdered = pFrom->isOrdered;
         if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue;
         if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue;
         /* At this point, pWLoop is a candidate to be the next loop. 
@@ -5030,21 +5030,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         rCost = sqlite3LogEstAdd(rCost, pFrom->rCost);
         nOut = pFrom->nRow + pWLoop->nOut;
         maskNew = pFrom->maskLoop | pWLoop->maskSelf;
-        if( !isOrderedValid ){
-          switch( wherePathSatisfiesOrderBy(pWInfo,
+        if( isOrdered<0 ){
+          isOrdered = wherePathSatisfiesOrderBy(pWInfo,
                        pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags,
-                       iLoop, pWLoop, &revMask) ){
-            case 1:  /* Yes.  pFrom+pWLoop does satisfy the ORDER BY clause */
-              isOrdered = 1;
-              isOrderedValid = 1;
-              break;
-            case 0:  /* No.  pFrom+pWLoop will require a separate sort */
-              isOrdered = 0;
-              isOrderedValid = 1;
-              rCost = sqlite3LogEstAdd(rCost, rSortCost);
-              break;
-            default: /* Cannot tell yet.  Try again on the next iteration */
-              break;
+                       iLoop, pWLoop, &revMask);
+          if( isOrdered==0 ){
+            rCost = sqlite3LogEstAdd(rCost, rSortCost);
           }
         }else{
           revMask = pFrom->revLoop;
@@ -5052,7 +5043,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         /* Check to see if pWLoop should be added to the mxChoice best so far */
         for(jj=0, pTo=aTo; jj<nTo; jj++, pTo++){
           if( pTo->maskLoop==maskNew
-           && pTo->isOrderedValid==isOrderedValid
+           && ((pTo->isOrdered^isOrdered)&80)==0
            && ((pTo->rCost<=rCost && pTo->nRow<=nOut) ||
                 (pTo->rCost>=rCost && pTo->nRow>=nOut))
           ){
@@ -5066,7 +5057,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
             if( sqlite3WhereTrace&0x4 ){
               sqlite3DebugPrintf("Skip   %s cost=%-3d,%3d order=%c\n",
                   wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
-                  isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?');
+                  isOrdered>=0 ? isOrdered+'0' : '?');
             }
 #endif
             continue;
@@ -5084,7 +5075,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
           if( sqlite3WhereTrace&0x4 ){
             sqlite3DebugPrintf("New    %s cost=%-3d,%3d order=%c\n",
                 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
-                isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?');
+                isOrdered>=0 ? isOrdered+'0' : '?');
           }
 #endif
         }else{
@@ -5094,10 +5085,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
               sqlite3DebugPrintf(
                   "Skip   %s cost=%-3d,%3d order=%c",
                   wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
-                  isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?');
+                  isOrdered>=0 ? isOrdered+'0' : '?');
               sqlite3DebugPrintf("   vs %s cost=%-3d,%d order=%c\n",
                   wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-                  pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?');
+                  pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
             }
 #endif
             testcase( pTo->rCost==rCost );
@@ -5110,10 +5101,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
             sqlite3DebugPrintf(
                 "Update %s cost=%-3d,%3d order=%c",
                 wherePathName(pFrom, iLoop, pWLoop), rCost, nOut,
-                isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?');
+                isOrdered>=0 ? isOrdered+'0' : '?');
             sqlite3DebugPrintf("  was %s cost=%-3d,%3d order=%c\n",
                 wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-                pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?');
+                pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?');
           }
 #endif
         }
@@ -5122,7 +5113,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
         pTo->revLoop = revMask;
         pTo->nRow = nOut;
         pTo->rCost = rCost;
-        pTo->isOrderedValid = isOrderedValid;
         pTo->isOrdered = isOrdered;
         memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop);
         pTo->aLoop[iLoop] = pWLoop;
@@ -5147,8 +5137,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
       for(ii=0, pTo=aTo; ii<nTo; ii++, pTo++){
         sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c",
            wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow,
-           pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?');
-        if( pTo->isOrderedValid && pTo->isOrdered ){
+           pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?');
+        if( pTo->isOrdered>0 ){
           sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop);
         }else{
           sqlite3DebugPrintf("\n");
@@ -5191,9 +5181,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){
     Bitmask notUsed;
     int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom,
                  WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], &notUsed);
-    if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
+    if( rc==pWInfo->pResultSet->nExpr ){
+      pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
+    }
   }
-  if( pFrom->isOrdered ){
+  if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){
     if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){
       pWInfo->eDistinct = WHERE_DISTINCT_ORDERED;
     }else{
@@ -5386,7 +5378,7 @@ WhereInfo *sqlite3WhereBegin(
   Parse *pParse,        /* The parser context */
   SrcList *pTabList,    /* FROM clause: A list of all tables to be scanned */
   Expr *pWhere,         /* The WHERE clause */
-  ExprList *pOrderBy,   /* An ORDER BY clause, or NULL */
+  ExprList *pOrderBy,   /* An ORDER BY (or GROUP BY) clause, or NULL */
   ExprList *pResultSet, /* Result set of the query */
   u16 wctrlFlags,       /* One of the WHERE_* flags defined in sqliteInt.h */
   int iIdxCur           /* If WHERE_ONETABLE_ONLY is set, index cursor number */
@@ -5408,6 +5400,10 @@ WhereInfo *sqlite3WhereBegin(
   /* Variable initialization */
   db = pParse->db;
   memset(&sWLB, 0, sizeof(sWLB));
+
+  /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
+  testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
+  if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
   sWLB.pOrderBy = pOrderBy;
 
   /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via

diff --git a/src/whereInt.h b/src/whereInt.h
index 23f929c4610030022c5e37e688b1602c7ae27a27..233b331d4282cc68e763ebfe3bd9e5bbb59d90d4 100644 (file)
--- a/src/whereInt.h
+++ b/src/whereInt.h
@@ -121,7 +121,7 @@ struct WhereLoop {
     struct {               /* Information for virtual tables */
       int idxNum;            /* Index number */
       u8 needFree;           /* True if sqlite3_free(idxStr) is needed */
-      u8 isOrdered;          /* True if satisfies ORDER BY */
+      i8 isOrdered;          /* True if satisfies ORDER BY */
       u16 omitMask;          /* Terms that may be omitted */
       char *idxStr;          /* Index identifier string */
     } vtab;
@@ -183,8 +183,7 @@ struct WherePath {
   Bitmask revLoop;      /* aLoop[]s that should be reversed for ORDER BY */
   LogEst nRow;          /* Estimated number of rows generated by this path */
   LogEst rCost;         /* Total cost of this path */
-  u8 isOrdered;         /* True if this path satisfies ORDER BY */
-  u8 isOrderedValid;    /* True if the isOrdered field is valid */
+  i8 isOrdered;         /* No. of ORDER BY terms satisfied. -1 for unknown */
   WhereLoop **aLoop;    /* Array of WhereLoop objects implementing this path */
 };