Add the ability to use indices for constraints of the form "x IS NOT NULL"

when sqlite_stat2 is available and most entries for column x are NULL.

FossilOrigin-Name: 5d5bddd290e71a7b03bcc23ff29881c23233cbff

diff --git a/manifest b/manifest
index f65504e81977ac4166a46239ad8fa1c3f1e2ce43..2e9f7b38fcc2c8042bbc4985e87de338a83c42dc 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
 -----BEGIN PGP SIGNED MESSAGE-----
 Hash: SHA1
 
-C Adjustments\sto\sthe\sresult\srow\sestimator\sfor\sthe\sIN\soperator\sso\sthat\sit\sgives\nthe\ssame\sestimates\sas\sthe\sequivalent\sOR\soperator.\s\sTest\scases\sfor\sthe\ssame.
-D 2011-01-21T18:18:13.960
+C Add\sthe\sability\sto\suse\sindices\sfor\sconstraints\sof\sthe\sform\s"x\sIS\sNOT\sNULL"\nwhen\ssqlite_stat2\sis\savailable\sand\smost\sentries\sfor\scolumn\sx\sare\sNULL.
+D 2011-01-22T00:10:45.721
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in de6498556d536ae60bb8bb10e8c1ba011448658c
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -243,7 +243,7 @@ F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30
 F src/wal.c dbca424f71678f663a286ab2a98f947af1d412a7
 F src/wal.h c1aac6593a0b02b15dc625987e619edeab39292e
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
-F src/where.c 2de6723cfb051bcfcfd3d3ca1ac04bb1388ba530
+F src/where.c 99a9ea77114b649d68d01127331119f6785a80f1
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 51756962d522e474338e9b2ebb26e7364d4aa125
@@ -900,14 +900,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P fd3977a27ae68e694df12a4713e55515c1e87c5d
-R cea6312924a8fb4373e961fbaf9716e5
+P c82cb9c028b3ba5463ae50c30196dbf157a7a305
+R 8c710a35ac2f95522b4422902520d5c8
 U drh
-Z d2cdc178cbf264c31de567c61a7d5758
+Z 5feaab9c960a4232f37e5b9d507f4c5a
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1.4.6 (GNU/Linux)
 
-iD8DBQFNOc3poxKgR168RlERAkIwAKCEe6e9BZEE6g3M5kOLzfgzYu8BvQCghsyD
-JkbODaFMx8NcwWU/YYsOcuo=
-=cn1U
+iD8DBQFNOiCJoxKgR168RlERAvjuAKCFxe3Zz4WQnNCqaR5BtD/txHvS9QCePp1G
+iZQ2yz7nxUFtZ+UwOppTLQo=
+=91DY
 -----END PGP SIGNATURE-----

diff --git a/manifest.uuid b/manifest.uuid
index 67ab42aca6f40fe440fa9ec49013c40c6bdf96b3..372d8e6a1e28b7f55d6b21f52b1c217a2a33106e 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c82cb9c028b3ba5463ae50c30196dbf157a7a305
\ No newline at end of file
+5d5bddd290e71a7b03bcc23ff29881c23233cbff
\ No newline at end of file

diff --git a/src/where.c b/src/where.c
index cb0b4638f3222cc37b815e3b2f0bcbc583525c96..8a7e258bd9f35a208f9aaa0887f9e26ff8c7560d 100644 (file)
--- a/src/where.c
+++ b/src/where.c
@@ -117,7 +117,7 @@ struct WhereTerm {
 #define TERM_ORINFO     0x10   /* Need to free the WhereTerm.u.pOrInfo object */
 #define TERM_ANDINFO    0x20   /* Need to free the WhereTerm.u.pAndInfo obj */
 #define TERM_OR_OK      0x40   /* Used during OR-clause processing */
-#define TERM_NOHELP     0x80   /* This term does not reduce the search space */
+#define TERM_VNULL      0x80   /* Manufactured x>NULL or x<=NULL term */
 
 /*
 ** An instance of the following structure holds all information about a
@@ -211,6 +211,7 @@ struct WhereCost {
 #define WO_ISNULL 0x080
 #define WO_OR     0x100       /* Two or more OR-connected terms */
 #define WO_AND    0x200       /* Two or more AND-connected terms */
+#define WO_NOOP   0x800       /* This term does not restrict search space */
 
 #define WO_ALL    0xfff       /* Mask of all possible WO_* values */
 #define WO_SINGLE 0x0ff       /* Mask of all non-compound WO_* values */
@@ -1061,8 +1062,7 @@ static void exprAnalyzeOrTerm(
       }else{
         sqlite3ExprListDelete(db, pList);
       }
-      pTerm->wtFlags |= TERM_NOHELP;
-      pTerm->eOperator = 0;  /* case 1 trumps case 2 */
+      pTerm->eOperator = WO_NOOP;  /* case 1 trumps case 2 */
     }
   }
 }
@@ -1326,6 +1326,42 @@ static void exprAnalyze(
   }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+#ifdef SQLITE_ENABLE_STAT2
+  /* When sqlite_stat2 histogram data is available an operator of the
+  ** form "x IS NOT NULL" can sometimes be evaluated more efficiently
+  ** as "x>NULL" if x is not an INTEGER PRIMARY KEY.  So construct a
+  ** virtual term of that form.
+  **
+  ** Note that the virtual term must be tagged with TERM_VNULL.  This
+  ** TERM_VNULL tag will suppress the not-null check at the beginning
+  ** of the loop.  Without the TERM_VNULL flag, the not-null check at
+  ** the start of the loop will prevent any results from being returned.
+  */
+  if( pExpr->op==TK_NOTNULL && pExpr->pLeft->iColumn>=0 ){
+    Expr *pNewExpr;
+    Expr *pLeft = pExpr->pLeft;
+    int idxNew;
+    WhereTerm *pNewTerm;
+
+    pNewExpr = sqlite3PExpr(pParse, TK_GT,
+                            sqlite3ExprDup(db, pLeft, 0),
+                            sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0);
+
+    idxNew = whereClauseInsert(pWC, pNewExpr,
+                              TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL);
+    testcase( idxNew==0 );
+    pNewTerm = &pWC->a[idxNew];
+    pNewTerm->leftCursor = pLeft->iTable;
+    pNewTerm->u.leftColumn = pLeft->iColumn;
+    pNewTerm->eOperator = WO_GT;
+    pNewTerm->iParent = idxTerm;
+    pTerm = &pWC->a[idxTerm];
+    pTerm->nChild = 1;
+    pTerm->wtFlags |= TERM_COPIED;
+    pNewTerm->prereqAll = pTerm->prereqAll;
+  }
+#endif /* SQLITE_ENABLE_STAT2 */
+
   /* Prevent ON clause terms of a LEFT JOIN from being used to drive
   ** an index for tables to the left of the join.
   */
@@ -2461,11 +2497,9 @@ range_est_fallback:
   UNUSED_PARAMETER(nEq);
 #endif
   assert( pLower || pUpper );
-  if( pLower && pUpper ){
-    *piEst = 11;
-  }else{
-    *piEst = 33;
-  }
+  *piEst = 100;
+  if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *piEst /= 3;
+  if( pUpper ) *piEst /= 3;
   return rc;
 }
 
@@ -2936,7 +2970,7 @@ static void bestBtreeIndex(
 
       thisTab = getMask(pWC->pMaskSet, iCur);
       for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){
-        if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_NOHELP) ) continue;
+        if( pTerm->wtFlags & TERM_VIRTUAL ) continue;
         if( (pTerm->prereqAll & notValid)!=thisTab ) continue;
         if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){
           if( nSkipEq ){
@@ -2958,7 +2992,7 @@ static void bestBtreeIndex(
             ** set size by a factor of 3 */
             nRow /= 3;
           }
-        }else{
+        }else if( pTerm->eOperator!=WO_NOOP ){
           /* Any other expression lowers the output row count by half */
           nRow /= 2;
         }
@@ -3796,7 +3830,9 @@ static Bitmask codeOneLoopStart(
     if( pRangeStart ){
       Expr *pRight = pRangeStart->pExpr->pRight;
       sqlite3ExprCode(pParse, pRight, regBase+nEq);
-      sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+      if( (pRangeStart->wtFlags & TERM_VNULL)==0 ){
+        sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+      }
       if( zStartAff ){
         if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_NONE){
           /* Since the comparison is to be performed with no conversions
@@ -3835,7 +3871,9 @@ static Bitmask codeOneLoopStart(
       Expr *pRight = pRangeEnd->pExpr->pRight;
       sqlite3ExprCacheRemove(pParse, regBase+nEq, 1);
       sqlite3ExprCode(pParse, pRight, regBase+nEq);
-      sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+      if( (pRangeEnd->wtFlags & TERM_VNULL)==0 ){
+        sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+      }
       if( zEndAff ){
         if( sqlite3CompareAffinity(pRight, zEndAff[nEq])==SQLITE_AFF_NONE){
           /* Since the comparison is to be performed with no conversions