From: drh <>
Date: Sun, 26 Mar 2023 16:36:27 +0000 (+0000)
Subject: Improvements to register allocation, especially in the ANALYZE command.
X-Git-Tag: version-3.42.0~210
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=aa9192e6aa4da7d6c685bfbbefe0b4f6e55c5b2d;p=thirdparty%2Fsqlite.git

Improvements to register allocation, especially in the ANALYZE command.
New assert() statements added to help verify that memory allocation is
correct, and to help fuzzer find lingering errors.

FossilOrigin-Name: 6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b
---

diff --git a/manifest b/manifest
index e84dd218bd..4dc1905e93 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Disable\sfactoring\sof\sconstant\svalues\sduring\sANALYZE.\s\sThis\sis\sa\stemporary\nfix\sfor\s[forum:/forumpost/07de5f6216|forum\spost\s07de5f6216].\s\sThe\sregister\nallocation\slogic\sin\sANALYZE\sneeds\sto\sbe\scompletely\srefactored,\sbut\sthat\swill\ntake\slonger.\s\sThis\scheck-in\swill\sserve\sto\sresolve\sthe\sissue\suntil\sa\sbetter\nfix\scan\sbe\sdevised.
-D 2023-03-26T11:54:51.582
+C Improvements\sto\sregister\sallocation,\sespecially\sin\sthe\sANALYZE\scommand.\nNew\sassert()\sstatements\sadded\sto\shelp\sverify\sthat\smemory\sallocation\sis\ncorrect,\sand\sto\shelp\sfuzzer\sfind\slingering\serrors.
+D 2023-03-26T16:36:27.642
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -558,7 +558,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786
 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a
 F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2
 F src/alter.c 3ca2f449c890f8b86ec9e06f0c4fccf0648941c3308a16904cb2852227db83f7
-F src/analyze.c 3449514fb387fbc15d2cead1fb6856d0556bf7491ab5088d7937bf1d48567ef3
+F src/analyze.c 8230fc1f3ddcf5492d258f28afcd36026c0de2f35cbbee22a41eed4baaf26148
 F src/attach.c cc9d00d30da916ff656038211410ccf04ed784b7564639b9b61d1839ed69fd39
 F src/auth.c f4fa91b6a90bbc8e0d0f738aa284551739c9543a367071f55574681e0f24f8cf
 F src/backup.c a2891172438e385fdbe97c11c9745676bec54f518d4447090af97189fd8e52d7
@@ -575,7 +575,7 @@ F src/date.c f21815ca7172ce073db3163ac54c8d9f2841077165c1a6123b4d1c376a0c7ec7
 F src/dbpage.c d47549716549311f79dc39fe5c8fb19390a6eb2c960f8e37c89a9c4de0c1052e
 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef
 F src/delete.c 201fe0763c52783d205c8c13cdd9d55c1bd5cb21c1f036753f99103b43284b90
-F src/expr.c c938990570effc02e065b176702d627a0237ec796263c142868f8007b0b703c0
+F src/expr.c 4cc4595f1c589b76f33f884f9c6b5f85ea80d205f1fc5ef9fb3617bf03920f0b
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
 F src/func.c d187be57a886ddf4e6b7ef584a494361899be3df5eee6d4a747b68ff4aff4122
@@ -618,7 +618,7 @@ F src/parse.y 424e49ed8fc6c907920db9be5a13a75ed43811e1ea8dd21b0fa9ef97f083dc6b
 F src/pcache.c 842410539b544e12d5fccfcf29890782f46a58f227a77bc0bd76243799662c0c
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c dee95e3cd2b61e6512dc814c5ab76d5eb36f0bfc9441dbb4260fccc0d12bbddc
-F src/pragma.c c0c6e0735436e0e9e8d490f82959ddc9475d438e9f808a51953a4d43e30cea51
+F src/pragma.c 20103b19d91551a673e5a35bd41a830b3d1bf2929174825cf6148430562c4533
 F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c ce87a08cfddd45a147150db34190b1986f2d4a0e0828858cb6bd908c78fb02e3
 F src/printf.c 7eac1a9896a80697e03e08963e210830532ae2ff610e16c193e95af007ca5623
@@ -630,7 +630,7 @@ F src/shell.c.in d80b53a9ec0a43ac0157fc54d667cb49fb6dd2229558135530feab2fc04b909
 F src/sqlite.h.in f01033703156615566bb329144d736a37fc35a278049db91127782a9c799b938
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4
-F src/sqliteInt.h ee87a8982f59897ea0594df986bb3ebbe971b16fa0248c9c4940ae18beda7deb
+F src/sqliteInt.h 8dab239c6de1d9b9e8732bbc4ad454f142b4b2781867cab0786597a391489ed4
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -2051,8 +2051,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ffe23af73fcb324df988a00be343654ce7078b7208647c4eb779d666b8297e7c
-R 23223a4c6ca9776e461c6a724ecec668
+P c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466
+R 83f2cb26cda25754a99646b5b49c48f5
 U drh
-Z 8a53fd62f6653c359d7e1076b2f287a9
+Z e8ea09f6c5d7f65fd4a1a44c0da01723
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index dbafff546d..cde71f3477 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-c3967d1259f1df969d303394986960bd098e174dcd337e374c9c3c39e0efa466
\ No newline at end of file
+6f8b97f31a4c8552312b4c98432ea356ae54c06d9cc929969f50c3c88360cd7b
\ No newline at end of file
diff --git a/src/analyze.c b/src/analyze.c
index 086a58494f..b6b59a6b79 100644
--- a/src/analyze.c
+++ b/src/analyze.c
@@ -198,7 +198,6 @@ static void openStatTable(
   assert( sqlite3BtreeHoldsAllMutexes(db) );
   assert( sqlite3VdbeDb(v)==db );
   pDb = &db->aDb[iDb];
-  pParse->okConstFactor = 0;
 
   /* Create new statistic tables if they do not exist, or clear them
   ** if they do already exist.
@@ -1002,7 +1001,8 @@ static void analyzeOneTable(
   Table *pStat1 = 0;
 #endif
 
-  pParse->nMem = MAX(pParse->nMem, iMem);
+  sqlite3TouchRegister(pParse, iMem);
+  assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) );
   v = sqlite3GetVdbe(pParse);
   if( v==0 || NEVER(pTab==0) ){
     return;
@@ -1108,7 +1108,7 @@ static void analyzeOneTable(
     ** the regPrev array and a trailing rowid (the rowid slot is required
     ** when building a record to insert into the sample column of 
     ** the sqlite_stat4 table.  */
-    pParse->nMem = MAX(pParse->nMem, regPrev+nColTest);
+    sqlite3TouchRegister(pParse, regPrev+nColTest);
 
     /* Open a read-only cursor on the index being analyzed. */
     assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) );
@@ -1296,24 +1296,19 @@ static void analyzeOneTable(
         }
 
         /* Allocate space to compute results for the largest index */
-        pParse->nMem = MAX(pParse->nMem, regCol+mxCol);
+        sqlite3TouchRegister(pParse, regCol+mxCol);
         doOnce = 0;
 #ifdef SQLITE_DEBUG
-        /* Verify that setting pParse->nTempReg to zero below really
-        ** is needed in some cases, in order to excise all temporary
-        ** registers from the middle of the STAT4 buffer.  
+        /* Verify that the call to sqlite3ClearTempRegCache() below
+        ** really is needed.
         ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25)
         */
-        if( pParse->nTempReg>0 ){
-          int kk;
-          for(kk=0; kk<pParse->nTempReg; kk++){
-            int regT = pParse->aTempReg[kk];
-            testcase( regT>=regCol && regT<regCol+mxCol );
-          }
-        }
+        testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) );
 #endif
-        pParse->nTempReg = 0;  /* tag-20230325-1 */
+        sqlite3ClearTempRegCache(pParse);  /* tag-20230325-1 */
+        assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) );
       }
+      assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) );
 
       addrNext = sqlite3VdbeCurrentAddr(v);
       callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid);
@@ -1394,6 +1389,7 @@ static void analyzeDatabase(Parse *pParse, int iDb){
   for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){
     Table *pTab = (Table*)sqliteHashData(k);
     analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab);
+    iMem = sqlite3FirstAvailableRegister(pParse, iMem);
   }
   loadAnalysis(pParse, iDb);
 }
diff --git a/src/expr.c b/src/expr.c
index cd09c0d711..564dfc7fbb 100644
--- a/src/expr.c
+++ b/src/expr.c
@@ -6635,6 +6635,35 @@ void sqlite3ClearTempRegCache(Parse *pParse){
   pParse->nRangeReg = 0;
 }
 
+/*
+** Make sure sufficient registers have been allocated so that
+** iReg is a valid register number.
+*/
+void sqlite3TouchRegister(Parse *pParse, int iReg){
+  if( pParse->nMem<iReg ) pParse->nMem = iReg;
+}
+
+/*
+** Return the latest reusable register in the set of all registers.
+** The value returned is no less than iMin.  If any register iMin or
+** greater is in permanent use, then return one more than that last
+** permanent register.
+*/
+int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){
+  const ExprList *pList = pParse->pConstExpr;
+  if( pList ){
+    int i;
+    for(i=0; i<pList->nExpr; i++){
+      if( pList->a[i].u.iConstExprReg>=iMin ){
+        iMin = pList->a[i].u.iConstExprReg + 1;
+      }
+    }
+  }
+  pParse->nTempReg = 0;
+  pParse->nRangeReg = 0;
+  return iMin;
+}
+
 /*
 ** Validate that no temporary register falls within the range of
 ** iFirst..iLast, inclusive.  This routine is only call from within assert()
@@ -6654,6 +6683,14 @@ int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){
       return 0;
     }
   }
+  if( pParse->pConstExpr ){
+    ExprList *pList = pParse->pConstExpr;
+    for(i=0; i<pList->nExpr; i++){
+      int iReg = pList->a[i].u.iConstExprReg;
+      if( iReg==0 ) continue;
+      if( iReg>=iFirst && iReg<=iLast ) return 0;
+    }
+  }
   return 1;
 }
 #endif /* SQLITE_DEBUG */
diff --git a/src/pragma.c b/src/pragma.c
index cd8d51c735..4686adfad5 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1524,7 +1524,7 @@ void sqlite3Pragma(
       zDb = db->aDb[iDb].zDbSName;
       sqlite3CodeVerifySchema(pParse, iDb);
       sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
-      if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow;
+      sqlite3TouchRegister(pParse, pTab->nCol+regRow);
       sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead);
       sqlite3VdbeLoadString(v, regResult, pTab->zName);
       assert( IsOrdinaryTable(pTab) );
@@ -1565,7 +1565,7 @@ void sqlite3Pragma(
         ** regRow..regRow+n. If any of the child key values are NULL, this 
         ** row cannot cause an FK violation. Jump directly to addrOk in 
         ** this case. */
-        if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol;
+        sqlite3TouchRegister(pParse, regRow + pFK->nCol);
         for(j=0; j<pFK->nCol; j++){
           int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom;
           sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j);
@@ -1729,7 +1729,7 @@ void sqlite3Pragma(
       aRoot[0] = cnt;
 
       /* Make sure sufficient number of registers have been allocated */
-      pParse->nMem = MAX( pParse->nMem, 8+mxIdx );
+      sqlite3TouchRegister(pParse, 8+mxIdx);
       sqlite3ClearTempRegCache(pParse);
 
       /* Do the b-tree integrity checks */
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 8cb0198a29..d6560c51bf 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4649,6 +4649,8 @@ void sqlite3ReleaseTempReg(Parse*,int);
 int sqlite3GetTempRange(Parse*,int);
 void sqlite3ReleaseTempRange(Parse*,int,int);
 void sqlite3ClearTempRegCache(Parse*);
+void sqlite3TouchRegister(Parse*,int);
+int sqlite3FirstAvailableRegister(Parse*,int);
 #ifdef SQLITE_DEBUG
 int sqlite3NoTempsInRange(Parse*,int,int);
 #endif