From: drh <drh@noemail.net>
Date: Fri, 10 May 2013 15:16:30 +0000 (+0000)
Subject: Update the NGQP so that it can produce plans that include automatic indices.
X-Git-Tag: version-3.8.0~130^2~80
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=eb04de3224c0f6e5160d8c641bf4dd2c83bb63e8;p=thirdparty%2Fsqlite.git

Update the NGQP so that it can produce plans that include automatic indices.

FossilOrigin-Name: 586b55d8d7722de1c0530b3b832bae0511e6d05c
---

diff --git a/manifest b/manifest
index f81212a36c..180d5b5b51 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Factor\sout\scommon\soperations\sinto\swhereLoopAddAll().\s\sAdd\sstubs\sfor\nmissing\sfeatures.
-D 2013-05-10T03:30:49.076
+C Update\sthe\sNGQP\sso\sthat\sit\scan\sproduce\splans\sthat\sinclude\sautomatic\sindices.
+D 2013-05-10T15:16:30.669
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c 31160bacc36e1fa5818caf091d5559d55eae7a5d
+F src/where.c f96b2602fd2da89873ed5a210608c58bf80d7c0e
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 5ed31c8279a4f465b982df5dc20cefc286928b93
-R 6a6b39040b040f90ce4281cc062a3caa
+P 0278e420614e02fb2d8474ed99b0904275882dfe
+R 69dec67361636bd352a6f82f3ef9f1cb
 U drh
-Z ca90367b6425b581381608ce8a76b4fa
+Z ef95f556fc74affef6b6df28ead69ff3
diff --git a/manifest.uuid b/manifest.uuid
index a2c17feaba..0143fae8fc 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0278e420614e02fb2d8474ed99b0904275882dfe
\ No newline at end of file
+586b55d8d7722de1c0530b3b832bae0511e6d05c
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 368081feae..40653d7c51 100644
--- a/src/where.c
+++ b/src/where.c
@@ -289,6 +289,7 @@ struct WhereLoopBuilder {
   SrcList *pTabList;        /* FROM clause */
   ExprList *pOrderBy;       /* ORDER BY clause */
   WhereLoop *pNew;          /* Template WhereLoop */
+  int mxTerm;               /* Maximum number of aTerm[] entries on pNew */
 };
 
 /*
@@ -5241,6 +5242,7 @@ static int whereLoopAddBtreeIndex(
   WhereLoop savedLoop;            /* Saved original content of pNew[] */
   int iCol;                       /* Index of the column in the table */
   int rc = SQLITE_OK;             /* Return code */
+  double rLogSize;                /* Logarithm of table size */
 
   db = pBuilder->db;
   pNew = pBuilder->pNew;
@@ -5262,10 +5264,12 @@ static int whereLoopAddBtreeIndex(
                         opMask, iCol>=0 ? pProbe : 0);
   savedLoop = *pNew;
   pNew->rSetup = (double)0;
+  rLogSize = estLog(pProbe->aiRowEst[0]);
   for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){
     int nIn = 1;
     pNew->u.btree.nEq = savedLoop.u.btree.nEq;
     pNew->nTerm = savedLoop.nTerm;
+    if( pNew->nTerm>=pBuilder->mxTerm ) break; /* Repeated column in index */
     pNew->aTerm[pNew->nTerm++] = pTerm;
     pNew->prereq = (savedLoop.prereq | pTerm->prereqRight) & ~pNew->maskSelf;
     if( pTerm->eOperator & WO_IN ){
@@ -5291,7 +5295,16 @@ static int whereLoopAddBtreeIndex(
       pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT;
       pNew->nOut = savedLoop.nOut/3;
     }
-    pNew->rRun = pNew->nOut + estLog(pProbe->aiRowEst[0])*nIn;
+    pNew->rRun = rLogSize*nIn;  /* Cost for nIn binary searches */
+    if( pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK) ){
+      pNew->rRun += pNew->nOut;  /* Unit step cost to reach each row */
+    }else{
+      /* Each row involves a step of the index, then a binary search of
+      ** the main table */
+      pNew->rRun += pNew->nOut*(1 + rLogSize);
+    }
+    /* TBD: Adjust nOut and rRun for STAT3 range values */
+    /* TBD: Adjust nOut for additional constraints */
     rc = whereLoopInsert(pBuilder->pWInfo, pNew);
     if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
      && pNew->u.btree.nEq<pProbe->nColumn
@@ -5318,6 +5331,8 @@ static int whereLoopAddBtree(
   struct SrcList_item *pSrc;  /* The FROM clause btree term to add */
   WhereLoop *pNew;            /* Template WhereLoop object */
   int rc = SQLITE_OK;         /* Return code */
+  double rSize;               /* number of rows in the table */
+  double rLogSize;            /* Logarithm of the number of rows in the table */
 
   pNew = pBuilder->pNew;
   pSrc = pBuilder->pTabList->a + pNew->iTab;
@@ -5347,6 +5362,33 @@ static int whereLoopAddBtree(
     }
     pProbe = &sPk;
   }
+  rSize = (double)pSrc->pTab->nRowEst;
+  rLogSize = estLog(rSize);
+
+  /* Automatic indexes */
+  if( (pBuilder->pParse->db->flags & SQLITE_AutoIndex)!=0 
+   && !pSrc->viaCoroutine
+   && !pSrc->notIndexed
+   && !pSrc->isCorrelated
+  ){
+    /* Generate auto-index WhereLoops */
+    WhereClause *pWC = pBuilder->pWC;
+    WhereTerm *pTerm;
+    WhereTerm *pWCEnd = pWC->a + pWC->nTerm;
+    for(pTerm=pWC->a; rc==SQLITE_OK && pTerm<pWCEnd; pTerm++){
+      if( termCanDriveIndex(pTerm, pSrc, 0) ){
+        pNew->u.btree.nEq = 1;
+        pNew->nTerm = 1;
+        pNew->aTerm[0] = pTerm;
+        pNew->rSetup = 2*rLogSize*pSrc->pTab->nRowEst;
+        pNew->nOut = (double)10;
+        pNew->rRun = rLogSize + pNew->nOut;
+        pNew->wsFlags = WHERE_TEMP_INDEX;
+        pNew->prereq = mExtra | pTerm->prereqRight;
+        rc = whereLoopInsert(pBuilder->pWInfo, pNew);
+      }
+    }
+  }
 
   /* Insert a full table scan */
   pNew->u.btree.nEq = 0;
@@ -5355,12 +5397,11 @@ static int whereLoopAddBtree(
   pNew->prereq = mExtra;
   pNew->u.btree.pIndex = 0;
   pNew->wsFlags = 0;
-  pNew->rRun = (double)pSrc->pTab->nRowEst;
-  pNew->nOut = (double)pSrc->pTab->nRowEst;
+  pNew->nOut = rSize;
+  pNew->rRun = rSize + rLogSize;
+  /* TBD: Reduce nOut using constraints */
   rc = whereLoopInsert(pBuilder->pWInfo, pNew);
 
-  /* TBD: Insert automatic index opportunities */
-
   /* Loop over all indices
   */
   for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext){
@@ -5478,6 +5519,7 @@ static int whereLoopAddVirtual(
     if( rc ) goto whereLoopAddVtab_exit;
     pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
     pNew->prereq = 0;
+    assert( pIdxInfo->nConstraint<=pBuilder->mxTerm );
     for(i=0; i<pIdxInfo->nConstraint; i++) pNew->aTerm[i] = 0;
     mxTerm = -1;
     for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
@@ -5555,7 +5597,13 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){
   /* Loop over the tables in the join, from left to right */
   pBuilder->pNew = pNew = sqlite3DbMallocZero(db, sizeof(WhereLoop));
   if( pNew==0 ) return SQLITE_NOMEM;
-  pNew->aTerm = sqlite3DbMallocZero(db, (pWC->nTerm+1)*sizeof(pNew->aTerm[0]));
+  pBuilder->mxTerm = pWC->nTerm+1;
+  while( pWC->pOuter ){
+    pWC = pWC->pOuter;
+    pBuilder->mxTerm += pWC->nTerm;
+  }
+  pWC = pBuilder->pWC;
+  pNew->aTerm = sqlite3DbMallocZero(db,pBuilder->mxTerm*sizeof(pNew->aTerm[0]));
   if( pNew->aTerm==0 ){
     rc = SQLITE_NOMEM;
     goto whereLoopAddAll_end;
@@ -6331,6 +6379,17 @@ WhereInfo *sqlite3WhereBegin(
     pWInfo->a[0].plan.wsFlags &= ~WHERE_IDX_ONLY;
   }
 
+#if 0
+  /* Scaffolding:  Check the new query plan against the old.  Report any
+  ** discrepencies */
+  for(ii=0; ii<nTabList; ii++){
+    if( pWInfo->a[ii].iFrom!=pWInfo->a[ii].pWLoop->iTab ){
+      sqlite3DebugPrintf("(QP-Mismatch)");
+      break;
+    }
+  }
+#endif
+
   /* Open all tables in the pTabList and any indices selected for
   ** searching those tables.
   */