From: drh <>
Date: Wed, 2 Jun 2021 12:44:26 +0000 (+0000)
Subject: This is an attempt to address the OP_SeekScan performance issue identified by
X-Git-Tag: version-3.36.0~43^2~7
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b034a241403a32eaa11897c2f28e83d9e1fc275e;p=thirdparty%2Fsqlite.git

This is an attempt to address the OP_SeekScan performance issue identified by
[forum:/forumpost/b4fcb8a598|forum post b4fcb8a598].

FossilOrigin-Name: d58efb3f21a4150b8136197e43837c7b646c42d492a7c24ddc7591dba415a2c8
---

diff --git a/manifest b/manifest
index ddc20c9616..9732d15ae4 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sissue\sin\s'mkopcodeh.tcl'\stool\sthat\sprevented\sits\stermination\sin\srare\scircumstances.
-D 2021-06-01T21:07:49.044
+C This\sis\san\sattempt\sto\saddress\sthe\sOP_SeekScan\sperformance\sissue\sidentified\sby\n[forum:/forumpost/b4fcb8a598|forum\spost\sb4fcb8a598].
+D 2021-06-02T12:44:26.088
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -631,9 +631,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c c8ec20a1ca161d5635a4f19c2a4efec2e006e19a8a61f272bf6bce1c80ab7436
 F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
 F src/walker.c 7342becedf3f8a26f9817f08436bdf8b56ad69af83705f6b9320a0ad3092c2ac
-F src/where.c db781a6fb79cd68155a4c33638d4a2885e45ad57f8a8e99e95778e259c2aee8f
+F src/where.c 2a6ae34252a15de5e6e1102d037ec899bd22c020d4f3e3df30f7afc1d9d3e349
 F src/whereInt.h 9248161dd004f625ce5d3841ca9b99fed3fc8d61522cf76340fc5217dbe1375b
-F src/wherecode.c 9f1f65d11437b25cd0a1497a170514c785f19ce6ad9d3e6fc73719cb5a49012f
+F src/wherecode.c d0963d5a2aaae8c18f459a7e72dc6c7ca897f3a4ddb40fd56a12384b725e9e0a
 F src/whereexpr.c 5a9c9f5d2dac4bcdcaae3035034b4667523f731df228e0bb1d4efc669efa9da5
 F src/window.c a6d624d83b2d5b3cfb82bb437a2fbae759c928d47dc9ad1338a9419269181bb2
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1918,7 +1918,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 587a3044468a40707c714d013cb766d8a4d9eb13bb20871846a0e8c34bea8cf4
-R 28996daffb0e59933544ba8f6a6b21ff
-U mistachkin
-Z 45822110f0419a0d61a2348a085e7f10
+P a8d921136f8ab132279984a77d0eeaf355342cdb0a98d7e1e59c8e6c8ed9459c
+R c9d14953d90f0bc10aff723dd24fc8cd
+T *branch * seekscan-improvements
+T *sym-seekscan-improvements *
+T -sym-trunk *
+U drh
+Z 37cec0fe3735838a40d15474ed3bffee
diff --git a/manifest.uuid b/manifest.uuid
index 4fa1d90d6b..0ba9b1497b 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a8d921136f8ab132279984a77d0eeaf355342cdb0a98d7e1e59c8e6c8ed9459c
\ No newline at end of file
+d58efb3f21a4150b8136197e43837c7b646c42d492a7c24ddc7591dba415a2c8
\ No newline at end of file
diff --git a/src/where.c b/src/where.c
index 9483abdc5b..9ccdd7e669 100644
--- a/src/where.c
+++ b/src/where.c
@@ -2597,7 +2597,7 @@ static int whereLoopAddBtreeIndex(
         nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
       }
       if( pProbe->hasStat1 && rLogSize>=10 ){
-        LogEst M, logK, safetyMargin;
+        LogEst M, logK, x;
         /* Let:
         **   N = the total number of rows in the table
         **   K = the number of entries on the RHS of the IN operator
@@ -2620,16 +2620,25 @@ static int whereLoopAddBtreeIndex(
         */
         M = pProbe->aiRowLogEst[saved_nEq];
         logK = estLog(nIn);
-        safetyMargin = 10;  /* TUNING: extra weight for indexed IN */
-        if( M + logK + safetyMargin < nIn + rLogSize ){
+        /* TUNING      v-----  10 to bias toward indexed IN */
+        x = M + logK + 10 - (nIn + rLogSize);
+        if( x>=0 ){
           WHERETRACE(0x40,
-            ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n",
-             saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+            ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
+             "prefers indexed lookup\n",
+             saved_nEq, M, logK, nIn, rLogSize, x));
+        }else if( x < -nInMul ){
+          WHERETRACE(0x40,
+            ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
+             " nInMul=%d) prefers skip-scan\n",
+             saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
           pNew->wsFlags |= WHERE_IN_SEEKSCAN;
         }else{
           WHERETRACE(0x40,
-            ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n",
-             saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
+            ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
+             " nInMul=%d) prefers normal scan\n",
+             saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
+          continue;
         }
       }
       pNew->wsFlags |= WHERE_COLUMN_IN;
diff --git a/src/wherecode.c b/src/wherecode.c
index 84fa1a1410..42315e59a0 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -1827,7 +1827,7 @@ Bitmask sqlite3WhereCodeOneLoopStart(
 
       op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev];
       assert( op!=0 );
-      if( (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 && op==OP_SeekGE ){
+      if( (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 && ALWAYS(op==OP_SeekGE) ){
         assert( regBignull==0 );
         /* TUNING:  The OP_SeekScan opcode seeks to reduce the number
         ** of expensive seek operations by replacing a single seek with