From: drh <>
Date: Mon, 7 Jul 2025 16:28:59 +0000 (+0000)
Subject: Rework the fix to the problem described by
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fheads%2Fbranch-3.50;p=thirdparty%2Fsqlite.git

Rework the fix to the problem described by
forum post b9647a113b so that it
provides a more complete fix that covers cases that were not resolved by
the original fix, and so that it does not cause performance regressions.

FossilOrigin-Name: 4084ac20f425bb992e567cff9cc6e304504c5198a41a9ce1304920c6411b7989
---

diff --git a/manifest b/manifest
index 57df75ade7..3e0414da50 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Work\saround\sthe\sEmscripten\s4.10\sregression\sdescribed\sin\s[https://github.com/emscripten-core/emscripten/issues/24656\s|\sEmscripten\sticket\s#24656].
-D 2025-07-07T14:42:30.360
+C Rework\sthe\sfix\sto\sthe\sproblem\sdescribed\sby\nforum\spost\sb9647a113b\sso\sthat\sit\nprovides\sa\smore\scomplete\sfix\sthat\scovers\scases\sthat\swere\snot\sresolved\sby\nthe\soriginal\sfix,\sand\sso\sthat\sit\sdoes\snot\scause\sperformance\sregressions.
+D 2025-07-07T16:28:59.924
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -729,7 +729,7 @@ F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea
 F src/btree.c da98489a981c347cc3a3982ea2810bbb583511a73cc34762547f30dbb4cda7f0
 F src/btree.h 18e5e7b2124c23426a283523e5f31a4bff029131b795bb82391f9d2f3136fc50
 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886
-F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f
+F src/build.c 67159d31bfc565e5f23a7be8159325bae599bddd19fc584ac2511b947cf341d3
 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9
@@ -790,7 +790,7 @@ F src/shell.c.in ba53a52dafb167ac6320703da741386c34fbcabe8c078a188bb9f89808e3ef8
 F src/sqlite.h.in 9ae373d11e1b11ac9c81c508523ae37f1619e739858280078ee9fb4e1e62d3ed
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e
-F src/sqliteInt.h e29d71b90bde0b2e06c813bd40d2265d1292a2ad7062ef7b2a1b9207dc9ecd98
+F src/sqliteInt.h 08a7dcb8db162875b3dd5229eb0bf75691150ac54fea1ff3670391bd6ef40546
 F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b
 F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -869,9 +869,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c 45a3b496248a0b36d91ce34da3278d54f8fa20e9d3fbd36d45a42051d1118137
+F src/where.c 4b9f73b1633b4dcddd82abc69aa6384dbef528289e8daa29521c3112b82fd1b9
 F src/whereInt.h ecdbfb5551cf394f04ec7f0bc7ad963146d80eee3071405ac29aa84950128b8e
-F src/wherecode.c 65670d1ef85ef54a4db3826d63be8b646c9ac280962166b645950901ed1bda29
+F src/wherecode.c 66684a11713667f532a69214ac552acee7e0825e2f1b71abaebe05511d190fb4
 F src/whereexpr.c 1c60db88b6e8472f4864b98014d1b2d9d16bdd42d5d181ec515acbcdeddc18db
 F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1577,7 +1577,7 @@ F test/round1.test 29c3c9039936ed024d672f003c4d35ee11c14c0acb75c5f7d6188ff16190c
 F test/rowallock.test 3f88ec6819489d0b2341c7a7528ae17c053ab7cc
 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81
 F test/rowid.test d27191b5ce794c05bf61081e8b2c546a1844c1641321dcaf7fb785234256cc8e
-F test/rowvalue.test 9c873b2f6e7ce72b24ef133f93515c07a6a7dac4846a344ebc2af7b8bfdf5147
+F test/rowvalue.test 8a3f0fea3a3cbbfc7cb9885b76185a774cd8d891e0c133e289567c755d39eb9f
 F test/rowvalue2.test 060d238b7e5639a7c5630cb5e63e311b44efef2b
 F test/rowvalue3.test 103e9a224ca0548dd0d67e439f39c5dd16de4200221a333927372408c025324c
 F test/rowvalue4.test bac9326d1e886656650f67c0ec484eb5f452244a8209c6af508e9a862ace08ed
@@ -2209,9 +2209,9 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 602f8a639c912d1c17bd9eba2f4e8834eb5a205ee525e5d94894468c759f14f2
-Q +c385475b250f3364507a95c5832137098a9bb7c7fc12ab3bb116e1fad7bb7645
-R 1974857cafc51e6b34fe99842f9a7f50
-U stephan
-Z c2543d871d74348aace3acdc63565967
+P db96ff7bbcef4a5b26a86d7ea6a5b2e4acb3262c5e01419f9bed6d63c78db477
+Q +28db0d152d90fb5e62d03ea5caceabe8901be98522aef3dc2b54564fbc35355d
+R 5c0a152c07e5d0a82d8ff7f0a2410d22
+U drh
+Z a8b343896e64b0414987e321dc658d47
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 92d1e20377..060b24e8d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-db96ff7bbcef4a5b26a86d7ea6a5b2e4acb3262c5e01419f9bed6d63c78db477
+4084ac20f425bb992e567cff9cc6e304504c5198a41a9ce1304920c6411b7989
diff --git a/src/build.c b/src/build.c
index 5bd3aac3ca..27d7b499df 100644
--- a/src/build.c
+++ b/src/build.c
@@ -4219,7 +4219,6 @@ void sqlite3CreateIndex(
       assert( j<=0x7fff );
       if( j<0 ){
         j = pTab->iPKey;
-        pIndex->bIdxRowid = 1;
       }else{
         if( pTab->aCol[j].notNull==0 ){
           pIndex->uniqNotNull = 0;
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 5cc134a9dd..dee1d07fe5 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -2792,7 +2792,6 @@ struct Index {
   unsigned hasStat1:1;     /* aiRowLogEst values come from sqlite_stat1 */
   unsigned bNoQuery:1;     /* Do not use this index to optimize queries */
   unsigned bAscKeyBug:1;   /* True if the bba7b69f9849b5bf bug applies */
-  unsigned bIdxRowid:1;    /* One or more of the index keys is the ROWID */
   unsigned bHasVCol:1;     /* Index references one or more VIRTUAL columns */
   unsigned bHasExpr:1;     /* Index contains an expression, either a literal
                            ** expression, or a reference to a VIRTUAL column */
diff --git a/src/where.c b/src/where.c
index 9561a75ead..d1312952d4 100644
--- a/src/where.c
+++ b/src/where.c
@@ -3233,6 +3233,7 @@ static int whereLoopAddBtreeIndex(
       if( ExprUseXSelect(pExpr) ){
         /* "x IN (SELECT ...)":  TUNING: the SELECT returns 25 rows */
         int i;
+        int bRedundant = 0;
         nIn = 46;  assert( 46==sqlite3LogEst(25) );
 
         /* The expression may actually be of the form (x, y) IN (SELECT...).
@@ -3241,7 +3242,20 @@ static int whereLoopAddBtreeIndex(
         ** for each such term. The following loop checks that pTerm is the
         ** first such term in use, and sets nIn back to 0 if it is not. */
         for(i=0; i<pNew->nLTerm-1; i++){
-          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0;
+          if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ){
+            nIn = 0;
+            if( pNew->aLTerm[i]->u.x.iField == pTerm->u.x.iField ){
+              /* Detect when two or more columns of an index match the same 
+              ** column of a vector IN operater, and avoid adding the column
+              ** to the WhereLoop more than once.  See tag-20250707-01
+              ** in test/rowvalue.test */
+              bRedundant = 1;
+            }
+          }
+        }
+        if( bRedundant ){
+          pNew->nLTerm--;
+          continue;
         }
       }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){
         /* "x IN (value, value, ...)" */
@@ -3473,7 +3487,7 @@ static int whereLoopAddBtreeIndex(
     if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0
      && pNew->u.btree.nEq<pProbe->nColumn
      && (pNew->u.btree.nEq<pProbe->nKeyCol ||
-          (pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY && !pProbe->bIdxRowid))
+          pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY)
     ){
       if( pNew->u.btree.nEq>3 ){
         sqlite3ProgressCheck(pParse);
diff --git a/src/wherecode.c b/src/wherecode.c
index 8e3e56cb18..823b4884ef 100644
--- a/src/wherecode.c
+++ b/src/wherecode.c
@@ -598,7 +598,9 @@ static Expr *removeUnindexableInClauseTerms(
           int iField;
           assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 );
           iField = pLoop->aLTerm[i]->u.x.iField - 1;
-          if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */
+          if( NEVER(pOrigRhs->a[iField].pExpr==0) ){
+            continue; /* Duplicate PK column */
+          }
           pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr);
           pOrigRhs->a[iField].pExpr = 0;
           if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1;
@@ -695,7 +697,7 @@ static SQLITE_NOINLINE void codeINTerm(
       return;
     }
   }
-  for(i=iEq;i<pLoop->nLTerm; i++){
+  for(i=iEq; i<pLoop->nLTerm; i++){
     assert( pLoop->aLTerm[i]!=0 );
     if( pLoop->aLTerm[i]->pExpr==pX ) nEq++;
   }
@@ -704,22 +706,13 @@ static SQLITE_NOINLINE void codeINTerm(
   if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){
     eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab);
   }else{
-    Expr *pExpr = pTerm->pExpr;
-    if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){
-      sqlite3 *db = pParse->db;
-      pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX);
-      if( !db->mallocFailed ){
-        aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq);
-        eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab);
-        pExpr->iTable = iTab;
-      }
-      sqlite3ExprDelete(db, pX);
-    }else{
-      int n = sqlite3ExprVectorSize(pX->pLeft);
-      aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n));
-      eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab);
+    sqlite3 *db = pParse->db;
+    Expr *pXMod = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX);
+    if( !db->mallocFailed ){
+      aiMap = (int*)sqlite3DbMallocZero(db, sizeof(int)*nEq);
+      eType = sqlite3FindInIndex(pParse, pXMod, IN_INDEX_LOOP, 0, aiMap, &iTab);
     }
-    pX = pExpr;
+    sqlite3ExprDelete(db, pXMod);
   }
 
   if( eType==IN_INDEX_INDEX_DESC ){
@@ -749,7 +742,7 @@ static SQLITE_NOINLINE void codeINTerm(
   if( pIn ){
     int iMap = 0;               /* Index in aiMap[] */
     pIn += i;
-    for(i=iEq;i<pLoop->nLTerm; i++){
+    for(i=iEq; i<pLoop->nLTerm; i++){
       if( pLoop->aLTerm[i]->pExpr==pX ){
         int iOut = iTarget + i - iEq;
         if( eType==IN_INDEX_ROWID ){
diff --git a/test/rowvalue.test b/test/rowvalue.test
index e2688e9030..387780c453 100644
--- a/test/rowvalue.test
+++ b/test/rowvalue.test
@@ -788,6 +788,9 @@ do_execsql_test 33.3 {
 # INTEGER PRIMARY KEY, and the columns that UNIQUE constraint are
 # used in a rowvalue-IN operator constraint.
 #
+# 2025-07-07 Discovered that the original fix was incomplete and
+# new tests added.  See tag-20250707-01 in the code.
+#
 reset_db
 do_execsql_test 34.1 {
   CREATE TABLE items (
@@ -804,5 +807,39 @@ do_execsql_test 34.1 {
     WHERE (Id, Item) IN (SELECT Id, Item FROM items);
   SELECT Id, Item, test FROM items ORDER BY id;
 } {1 2 ok 2 2 ok 3 3 ok 4 5 ok}
+db null NULL
+do_execsql_test 34.2 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d);
+  CREATE INDEX idx ON t1(b,a);
+  INSERT INTO t1(a,b) VALUES (1, 22);
+  SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1);
+} {1 22 NULL NULL}
+do_execsql_test 34.3 {
+  DROP TABLE t1;
+  CREATE TABLE t1(a, b, c, d);
+  CREATE INDEX idx ON t1(b,a,a);
+  INSERT INTO t1(a,b) VALUES (1, 22);
+  SELECT * FROM t1 INDEXED BY idx WHERE (b,a) IN (SELECT b,a FROM t1);
+} {1 22 NULL NULL}
+do_execsql_test 34.4 {
+  DROP TABLE t1;
+  CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT);
+  CREATE INDEX t1a ON t1(a,id);  -- index includes PRIMARY KEY
+  CREATE TABLE t2(id INTEGER PRIMARY KEY);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<100)
+    INSERT INTO t1(id,a) SELECT n, 777 FROM c;
+  INSERT INTO t2 SELECT id FROM t1;
+  SELECT *
+    FROM t1 JOIN t2 USING(id)
+   WHERE t1.a=777 AND t2.id>999
+   ORDER BY t1.id;
+} {}
+do_execsql_test 34.5 {
+  EXPLAIN QUERY PLAN
+  SELECT *
+    FROM t1 JOIN t2 USING(id)
+   WHERE t1.a=777 AND t2.id>999
+   ORDER BY t1.id;
+} {/SEARCH t1 USING COVERING INDEX t1a .a=. AND id>../}
 
 finish_test