From: dan <Dan Kennedy>
Date: Tue, 9 Mar 2021 16:06:25 +0000 (+0000)
Subject: Experimental optimization for distinct aggregates (e.g. "SELECT count(DISTINCT <expr... 
X-Git-Tag: version-3.36.0~289^2~6
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4fcb30b7d7147cddb05adab2191ae0f1a8268b8a;p=thirdparty%2Fsqlite.git

Experimental optimization for distinct aggregates (e.g. "SELECT count(DISTINCT <expr) FROM ...").

FossilOrigin-Name: eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22
---

diff --git a/manifest b/manifest
index 85fc9ec2fd..9be753e63b 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\stest\scases\sfor\sALTER\sTABLE.
-D 2021-03-08T17:22:01.712
+C Experimental\soptimization\sfor\sdistinct\saggregates\s(e.g.\s"SELECT\scount(DISTINCT\s<expr)\sFROM\s...").
+D 2021-03-09T16:06:25.224
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -542,12 +542,12 @@ F src/printf.c 2b03a80d7c11bb422115dca175a18bf430e9c9dbaa0eee63b758f0c022f8f34f
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 688070848f0a0c41bcc545a4b4b052921d9abc29ba3102985d3d6f7595d9637c
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c fb1851222c3ec1e0e1b4c8197a76a4595008956220ddebd7a0a898ad15705acc
+F src/select.c b51fbd795e5de19bd8448816eb37b098ef679f49b2572a5d570db73aaa2494a6
 F src/shell.c.in af18a2e980aabe739a8188266464866fe7947b100674e07480e7ba3e37595947
 F src/sqlite.h.in 3426a080ea1f222a73e3bd91e7eacbd30570a0117c03d42c6dde606f33e5e318
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e
-F src/sqliteInt.h 7c4679b3b068149f497fe50203a4b04be6b17b8aba581cdbc75da45329cad915
+F src/sqliteInt.h 75d9504d22cc08a355a51e7027aa955c3423121ce7768a65331a4082aafd66ed
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -629,7 +629,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 69e770e96fd56cc21608992bf2c6f1f3dc5cf2572d0495c6a643b06c3a679f14
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c d42d6c80ea363ef689a462e65eefcfe87deab924c50de5baa37ecb6af7d7ddaa
-F src/where.c 10d06b16670a1d2a992d52a9f08e49426d38a08fb0a7ae5f7f62fd023d560e1e
+F src/where.c 6fcbfab409b4a60dd92abd4c5f3fca44f5df3f8ea24f689d2ea620f05da89d96
 F src/whereInt.h 446e5e8018f83358ef917cf32d8e6a86dc8430113d0b17e720f1839d3faa44c4
 F src/wherecode.c e57a8690311a75d06e723e8d379f9831de04aba300e07174d236e32a7f9c7a13
 F src/whereexpr.c 53452fe2fb07be2f4cb17f55cc721416fae0092c00717f106faf289c990b6494
@@ -1910,7 +1910,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f13e0d12f137cd6b4e83e92bd36652431d8416e6eed4797313e061f2e390d39b
-R f9de7ca9f057059fa9533f91ff672e89
+P 9c9ba36e859e330c50ed40ede4b93eeb0a5c3337240465d953a7be9115a81390
+R 05e155a5881f070d054d078349eab572
+T *branch * distinct-agg-opt
+T *sym-distinct-agg-opt *
+T -sym-trunk *
 U dan
-Z e860451ea7454ca8ef2d0c3618d92e27
+Z a27a5de08cdd8303a3cf21c478413fb4
diff --git a/manifest.uuid b/manifest.uuid
index 8abd56e9f1..24e374a9bb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9c9ba36e859e330c50ed40ede4b93eeb0a5c3337240465d953a7be9115a81390
\ No newline at end of file
+eb919611fd2f255e4ad1fe7db633363793169f6cf99c650eaefa48c022eb5d22
\ No newline at end of file
diff --git a/src/select.c b/src/select.c
index 2b73b2f4d5..6eba9cb548 100644
--- a/src/select.c
+++ b/src/select.c
@@ -740,6 +740,7 @@ static void codeOffset(
   }
 }
 
+#if 0
 /*
 ** Add code that will check to make sure the N registers starting at iMem
 ** form a distinct entry.  iTab is a sorting index that holds previously
@@ -767,6 +768,78 @@ static void codeDistinct(
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   sqlite3ReleaseTempReg(pParse, r1);
 }
+#endif
+
+static void codeDistinct2(
+  Parse *pParse,     /* Parsing and code generating context */
+  int eTnctType,     /* WHERE_DISTINCT_* value */
+  int iTab,          /* A sorting index used to test for distinctness */
+  int iTabAddr,      /* Address of OP_OpenEphemeral instruction for iTab */
+  int addrRepeat,    /* Jump to here if not distinct */
+  ExprList *pEList,  /* Expression for each element */
+  int regElem        /* First element */
+){
+  int nResultCol = pEList->nExpr;
+  Vdbe *v = pParse->pVdbe;
+
+  switch( eTnctType ){
+    case WHERE_DISTINCT_ORDERED: {
+      int i;
+      VdbeOp *pOp;            /* No longer required OpenEphemeral instr. */
+      int iJump;              /* Jump destination */
+      int regPrev;            /* Previous row content */
+
+      /* Allocate space for the previous row */
+      regPrev = pParse->nMem+1;
+      pParse->nMem += nResultCol;
+
+      /* Change the OP_OpenEphemeral coded earlier to an OP_Null
+      ** sets the MEM_Cleared bit on the first register of the
+      ** previous value.  This will cause the OP_Ne below to always
+      ** fail on the first iteration of the loop even if the first
+      ** row is all NULLs.  */
+      sqlite3VdbeChangeToNoop(v, iTabAddr);
+      pOp = sqlite3VdbeGetOp(v, iTabAddr);
+      pOp->opcode = OP_Null;
+      pOp->p1 = 1;
+      pOp->p2 = regPrev;
+      pOp = 0;  /* Ensure pOp is not used after sqlite3VdbeAddOp() */
+
+      iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
+      for(i=0; i<nResultCol; i++){
+        CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[i].pExpr);
+        if( i<nResultCol-1 ){
+          sqlite3VdbeAddOp3(v, OP_Ne, regElem+i, iJump, regPrev+i);
+          VdbeCoverage(v);
+        }else{
+          sqlite3VdbeAddOp3(v, OP_Eq, regElem+i, addrRepeat, regPrev+i);
+          VdbeCoverage(v);
+         }
+        sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
+        sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
+      }
+      assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
+      sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1);
+      break;
+    }
+
+    case WHERE_DISTINCT_UNIQUE: {
+      sqlite3VdbeChangeToNoop(v, iTabAddr);
+      break;
+    }
+
+    default: {
+      int r1 = sqlite3GetTempReg(pParse);
+      sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol);
+      VdbeCoverage(v);
+      sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1);
+      sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol);
+      sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
+      sqlite3ReleaseTempReg(pParse, r1);
+      break;
+    }
+  }
+}
 
 #ifdef SQLITE_ENABLE_SORTER_REFERENCES
 /*
@@ -1013,59 +1086,9 @@ static void selectInnerLoop(
   ** part of the result.
   */
   if( hasDistinct ){
-    switch( pDistinct->eTnctType ){
-      case WHERE_DISTINCT_ORDERED: {
-        VdbeOp *pOp;            /* No longer required OpenEphemeral instr. */
-        int iJump;              /* Jump destination */
-        int regPrev;            /* Previous row content */
-
-        /* Allocate space for the previous row */
-        regPrev = pParse->nMem+1;
-        pParse->nMem += nResultCol;
-
-        /* Change the OP_OpenEphemeral coded earlier to an OP_Null
-        ** sets the MEM_Cleared bit on the first register of the
-        ** previous value.  This will cause the OP_Ne below to always
-        ** fail on the first iteration of the loop even if the first
-        ** row is all NULLs.
-        */
-        sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct);
-        pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct);
-        pOp->opcode = OP_Null;
-        pOp->p1 = 1;
-        pOp->p2 = regPrev;
-        pOp = 0;  /* Ensure pOp is not used after sqlite3VdbeAddOp() */
-
-        iJump = sqlite3VdbeCurrentAddr(v) + nResultCol;
-        for(i=0; i<nResultCol; i++){
-          CollSeq *pColl = sqlite3ExprCollSeq(pParse, p->pEList->a[i].pExpr);
-          if( i<nResultCol-1 ){
-            sqlite3VdbeAddOp3(v, OP_Ne, regResult+i, iJump, regPrev+i);
-            VdbeCoverage(v);
-          }else{
-            sqlite3VdbeAddOp3(v, OP_Eq, regResult+i, iContinue, regPrev+i);
-            VdbeCoverage(v);
-           }
-          sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ);
-          sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
-        }
-        assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed );
-        sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1);
-        break;
-      }
-
-      case WHERE_DISTINCT_UNIQUE: {
-        sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct);
-        break;
-      }
-
-      default: {
-        assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED );
-        codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol,
-                     regResult);
-        break;
-      }
-    }
+    assert( nResultCol==p->pEList->nExpr );
+    codeDistinct2(pParse, pDistinct->eTnctType, pDistinct->tabTnct, 
+        pDistinct->addrTnct, iContinue, p->pEList, regResult);
     if( pSort==0 ){
       codeOffset(v, p->iOffset, iContinue);
     }
@@ -5635,8 +5658,8 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
         pFunc->iDistinct = -1;
       }else{
         KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0);
-        sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0,
-                          (char*)pKeyInfo, P4_KEYINFO);
+        pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, 
+            pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO);
       }
     }
   }
@@ -5668,7 +5691,12 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
 ** registers if register regAcc contains 0. The caller will take care
 ** of setting and clearing regAcc.
 */
-static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){
+static void updateAccumulator(
+  Parse *pParse, 
+  int regAcc, 
+  AggInfo *pAggInfo,
+  int eDistinctType
+){
   Vdbe *v = pParse->pVdbe;
   int i;
   int regHit = 0;
@@ -5714,13 +5742,14 @@ static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){
       nArg = 0;
       regAgg = 0;
     }
-    if( pF->iDistinct>=0 ){
+    if( pF->iDistinct>=0 && pList ){
       if( addrNext==0 ){ 
         addrNext = sqlite3VdbeMakeLabel(pParse);
       }
       testcase( nArg==0 );  /* Error condition */
       testcase( nArg>1 );   /* Also an error */
-      codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg);
+      codeDistinct2(pParse, eDistinctType, pF->iDistinct, pF->iDistAddr, 
+          addrNext, pList, regAgg);
     }
     if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){
       CollSeq *pColl = 0;
@@ -6905,7 +6934,7 @@ int sqlite3Select(
       ** the current row
       */
       sqlite3VdbeJumpHere(v, addr1);
-      updateAccumulator(pParse, iUseFlag, pAggInfo);
+      updateAccumulator(pParse, iUseFlag, pAggInfo, WHERE_DISTINCT_UNORDERED);
       sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag);
       VdbeComment((v, "indicate data in accumulator"));
 
@@ -7025,6 +7054,8 @@ int sqlite3Select(
         explainSimpleCount(pParse, pTab, pBest);
       }else{
         int regAcc = 0;           /* "populate accumulators" flag */
+        ExprList *pDistinct = 0;
+        u16 distFlag = 0;
 
         /* If there are accumulator registers but no min() or max() functions
         ** without FILTER clauses, allocate register regAcc. Register regAcc
@@ -7048,6 +7079,9 @@ int sqlite3Select(
             regAcc = ++pParse->nMem;
             sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc);
           }
+        }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){
+          pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
+          distFlag = pDistinct ? WHERE_WANT_DISTINCT : 0;
         }
 
         /* This case runs if the aggregate has no GROUP BY clause.  The
@@ -7067,12 +7101,14 @@ int sqlite3Select(
 
         SELECTTRACE(1,pParse,p,("WhereBegin\n"));
         pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy,
-                                   0, minMaxFlag, 0);
+                                   pDistinct, minMaxFlag|distFlag, 0);
         if( pWInfo==0 ){
           goto select_end;
         }
         SELECTTRACE(1,pParse,p,("WhereBegin returns\n"));
-        updateAccumulator(pParse, regAcc, pAggInfo);
+        updateAccumulator(
+            pParse, regAcc, pAggInfo, sqlite3WhereIsDistinct(pWInfo)
+        );
         if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc);
         if( minMaxFlag ){
           sqlite3WhereMinMaxOptEarlyOut(v, pWInfo);
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 913de54ddc..7b451b5573 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2598,6 +2598,7 @@ struct AggInfo {
     FuncDef *pFunc;          /* The aggregate function implementation */
     int iMem;                /* Memory location that acts as accumulator */
     int iDistinct;           /* Ephemeral table used to enforce DISTINCT */
+    int iDistAddr;           /* Addres of OP_OpenEphemeral */
   } *aFunc;
   int nFunc;              /* Number of entries in aFunc[] */
   u32 selId;              /* Select to which this AggInfo belongs */
diff --git a/src/where.c b/src/where.c
index 75b3ceff87..8e44e25765 100644
--- a/src/where.c
+++ b/src/where.c
@@ -492,7 +492,7 @@ static int findIndexCol(
   for(i=0; i<pList->nExpr; i++){
     Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr);
     if( ALWAYS(p!=0)
-     && p->op==TK_COLUMN
+     && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN)
      && p->iColumn==pIdx->aiColumn[iCol]
      && p->iTable==iBase
     ){
@@ -557,7 +557,8 @@ static int isDistinctRedundant(
   for(i=0; i<pDistinct->nExpr; i++){
     Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr);
     if( NEVER(p==0) ) continue;
-    if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1;
+    if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue;
+    if( p->iTable==iBase && p->iColumn<0 ) return 1;
   }
 
   /* Loop through all indices on the table, checking each to see if it makes
@@ -3822,7 +3823,7 @@ static i8 wherePathSatisfiesOrderBy(
       if( MASKBIT(i) & obSat ) continue;
       pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr);
       if( NEVER(pOBExpr==0) ) continue;
-      if( pOBExpr->op!=TK_COLUMN ) continue;
+      if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
       if( pOBExpr->iTable!=iCur ) continue;
       pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn,
                        ~ready, eqOpMask, 0);
@@ -3951,7 +3952,7 @@ static i8 wherePathSatisfiesOrderBy(
           if( NEVER(pOBExpr==0) ) continue;
           if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0;
           if( iColumn>=XN_ROWID ){
-            if( pOBExpr->op!=TK_COLUMN ) continue;
+            if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue;
             if( pOBExpr->iTable!=iCur ) continue;
             if( pOBExpr->iColumn!=iColumn ) continue;
           }else{