From: dan <Dan Kennedy>
Date: Sat, 2 Mar 2024 20:39:17 +0000 (+0000)
Subject: Attempt to use less memory when handling a large VALUES clause attached to an INSERT... 
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=40df47d15167ad6b9fac05d136b91b5a65d020e5;p=thirdparty%2Fsqlite.git

Attempt to use less memory when handling a large VALUES clause attached to an INSERT statement. This branch is buggy.

FossilOrigin-Name: 6d4f1ae2fcabdaca4e2c12224a8807c6fe2c62353dff435a0c0030fa58312df6
---

diff --git a/manifest b/manifest
index 28b15c7a43..35afee7129 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stwo\s-Werror=lto-type-mismatch\swarnings\sreported\sin\s[forum:ef62b57bd5|forum\spost\sef62b57bd5].
-D 2024-02-29T10:55:02.008
+C Attempt\sto\suse\sless\smemory\swhen\shandling\sa\slarge\sVALUES\sclause\sattached\sto\san\sINSERT\sstatement.\sThis\sbranch\sis\sbuggy.
+D 2024-03-02T20:39:17.581
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -709,7 +709,7 @@ F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6
 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
-F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276
+F src/insert.c 8b8637c86ac4e9684b6b2e34a585453f22511b5ea73861fc3e03ec093c6d8e61
 F src/json.c 3b4e2778d95d923d6d77e8a5efd51a6265017b466782d597303f5f094fcd68af
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
@@ -739,7 +739,7 @@ F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c ff60e98138d2499082ac6230f01ac508aba545315debccfca2fd6042f5f10fcd
 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a
-F src/parse.y 6209f01e8e7495379571454744fa82a5cfc2e7eeb89e46dee3f410d73ea6252d
+F src/parse.y b922f345edda2d6e55e4436cd98bb06a24d90400ee1401fb7400f02cfa438f78
 F src/pcache.c 040b165f30622a21b7a9a77c6f2e4877a32fb7f22d4c7f0d2a6fa6833a156a75
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 602acb23c471bb8d557a6f0083cc2be641d6cafcafa19e481eba7ef4c9ca0f00
@@ -755,7 +755,7 @@ F src/shell.c.in 2ec564ed3ff0147036be313efeb47b3dbfb8753d5eb5ea0e90636427c6b3a36
 F src/sqlite.h.in 19a2db3995a699bd7f6dfb423856242bfceb7ec849a93c91d241d19fc28d9f0f
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 58b7295a748ca5ed9e211510205b4a66a24c68f864225b81e19d4cf6038b40a1
+F src/sqliteInt.h c9b4ad5eafa2172bb60d716095129e21c698e908275987c9d22ba220536176ab
 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -824,7 +824,7 @@ F src/vdbe.c 523a88b3df328810fbcbb407738c352dd9d5163b7af4c953e6e9887a4b582859
 F src/vdbe.h c2d78d15112c3fc5ab87f5e8e0b75d2db1c624409de2e858c3d1aafb1650bb4f
 F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
 F src/vdbeapi.c 8f57d60c89da0b60e6d4e272358c511f6bae4e24330bdb11f8b42f986d1bf21b
-F src/vdbeaux.c 66161ba75c4a2d86aab124fe3cbcc2a752d8e4b1fc8c1fc6e00625db4798168c
+F src/vdbeaux.c c5704adf1737e4dd7fc52de26f263e4ee7733380fe857269f9633dca1e8eb2cc
 F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
 F src/vdbemem.c 213bf303826c0ef702e3a2a69dab2309d84b8381b822c6787885859fd7cd4c4e
 F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@@ -1920,6 +1920,7 @@ F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7
 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c
 F test/vacuum6.test b137b04bf3392d3f5c3b8fda0ce85a6775a70ca112f6559f74ff52dc9ce042fd
 F test/vacuummem.test 4b30f5b95a9ff86e9d5c20741e50a898b2dc10b0962a3211571eb165357003fb
+F test/values.test f341107204aa71bb71612c2cc95f57e08192398813edfb39b4be64f312816e22
 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
 F test/view.test d4c4281e1679245829db35597817282f60dc513fc39cc5439078f009bd118487
@@ -2176,9 +2177,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d51c699ae413215d534f954c129691ef5d95d540dc5d4304cbba576c976d931c 29f94610dc8319709b8d3726a60e2a7a33ffbf4998c681f772da26c9701cc0b0
-R 728f9a968510fa9cb95ea7ce7eec45f1
-T +closed 29f94610dc8319709b8d3726a60e2a7a33ffbf4998c681f772da26c9701cc0b0 Closed\sby\sintegrate-merge.
-U stephan
-Z 37dee8fa6539319388e07ab4c515c697
+P 803481f25020f3c25941f1e7d1a8071937820dea951e8798198b0b0fa3fb48ce
+R a126c2ec3e1ab3b5af590c55f1bacd30
+T *branch * exp-values-clause
+T *sym-exp-values-clause *
+T -sym-trunk *
+U dan
+Z 9fda6a79767fb24afbbb26d990c827a5
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 2f2c96f612..5e4d107b6a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-803481f25020f3c25941f1e7d1a8071937820dea951e8798198b0b0fa3fb48ce
\ No newline at end of file
+6d4f1ae2fcabdaca4e2c12224a8807c6fe2c62353dff435a0c0030fa58312df6
\ No newline at end of file
diff --git a/src/insert.c b/src/insert.c
index 1c31ca2338..6fc20ecd4b 100644
--- a/src/insert.c
+++ b/src/insert.c
@@ -577,6 +577,69 @@ void sqlite3AutoincrementEnd(Parse *pParse){
 # define autoIncStep(A,B,C)
 #endif /* SQLITE_OMIT_AUTOINCREMENT */
 
+typedef struct MultiValues MultiValues;
+struct MultiValues {
+  Select *pSelect;
+  SelectDest dest;
+  int addrTop;
+  int regYield;
+};
+
+static int isConstantRow(ExprList *pRow){
+  int ii;
+  for(ii=0; ii<pRow->nExpr; ii++){
+    if( !sqlite3ExprIsConstantOrFunction(pRow->a[ii].pExpr, 0) ) return 0;
+  }
+  return 1;
+}
+
+void sqlite3MultiValuesStart(Parse *pParse, Select *pSel){
+  Vdbe *v = sqlite3GetVdbe(pParse);
+  MultiValues *pVal = 0;
+
+  assert( pParse->pValues==0 );
+  assert( pParse->zValuesToken!=0 );
+  if( isConstantRow(pSel->pEList) ){
+    pVal = (MultiValues*)sqlite3DbMallocZero(pParse->db, sizeof(*pVal));
+    pVal->pSelect = pSel;
+    pVal->regYield = ++pParse->nMem;
+    pVal->addrTop = sqlite3VdbeCurrentAddr(v) + 1;
+    sqlite3VdbeAddOp3(v, OP_InitCoroutine, pVal->regYield, 0, pVal->addrTop);
+    sqlite3SelectDestInit(&pVal->dest, SRT_Coroutine, pVal->regYield);
+    sqlite3Select(pParse, pSel, &pVal->dest);
+    pParse->pValues = pVal;
+    pSel->selFlags = pSel->selFlags & (~SF_Values);
+    sqlite3ParserAddCleanup(pParse, sqlite3DbFree, pVal);
+  }
+}
+
+Select *sqlite3InsertMultiValues(Parse *pParse, Select *pSel, ExprList *pRow){
+  MultiValues *pVal = pParse->pValues;
+  Select *pRight;
+
+  pRight = sqlite3SelectNew(pParse,pRow, 0,0,0,0,0, SF_Values|SF_MultiValue, 0);
+
+  if( pRight ){
+    if( pVal==0 || pVal->pSelect!=pSel || !isConstantRow(pRight->pEList) ){
+      pSel->selFlags &= ~SF_MultiValue;
+      pRight->op = TK_ALL;
+      pRight->pPrior = pSel;
+      pSel = pRight;
+    }else{
+      if( pRight->pEList->nExpr!=pSel->pEList->nExpr ){
+        sqlite3SelectWrongNumTermsError(pParse, pRight);
+      }else{
+        int explain = pParse->explain;
+        pParse->explain = 255;
+        sqlite3Select(pParse, pRight, &pVal->dest);
+        pParse->explain = explain;
+      }
+      sqlite3SelectDelete(pParse->db, pRight);
+    }
+  }
+
+  return pSel;
+}
 
 /* Forward declaration */
 static int xferOptimization(
@@ -915,16 +978,25 @@ void sqlite3Insert(
     ** Generate a co-routine to run the SELECT. */
     int regYield;       /* Register holding co-routine entry-point */
     int addrTop;        /* Top of the co-routine */
-    int rc;             /* Result code */
-
-    regYield = ++pParse->nMem;
-    addrTop = sqlite3VdbeCurrentAddr(v) + 1;
-    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
-    sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
-    dest.iSdst = bIdListInOrder ? regData : 0;
-    dest.nSdst = pTab->nCol;
-    rc = sqlite3Select(pParse, pSelect, &dest);
-    regFromSelect = dest.iSdst;
+    int rc = SQLITE_OK; /* Result code */
+
+    if( pParse->pValues && pParse->pValues->pSelect==pSelect ){
+      MultiValues *pVal = pParse->pValues;
+      /* todo: surely we can use either regYield or dest.iSDParm... */
+      dest.iSDParm = regYield = pVal->regYield;
+      addrTop = pVal->addrTop;
+      regFromSelect = pVal->dest.iSdst;
+    }else{
+      regYield = ++pParse->nMem;
+      addrTop = sqlite3VdbeCurrentAddr(v) + 1;
+      sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield);
+      dest.iSdst = bIdListInOrder ? regData : 0;
+      dest.nSdst = pTab->nCol;
+      rc = sqlite3Select(pParse, pSelect, &dest);
+      regFromSelect = dest.iSdst;
+    }
+
     assert( db->pParse==pParse );
     if( rc || pParse->nErr ) goto insert_cleanup;
     assert( db->mallocFailed==0 );
diff --git a/src/parse.y b/src/parse.y
index 37c9fa8bc9..326f08e97d 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -626,20 +626,14 @@ oneselect(A) ::= values(A).
 
 %type values {Select*}
 %destructor values {sqlite3SelectDelete(pParse->db, $$);}
-values(A) ::= VALUES LP nexprlist(X) RP. {
+values(A) ::= VALUES(T) LP nexprlist(X) RP. {
   A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0);
+  if( T.z==pParse->zValuesToken && yyLookahead==TK_COMMA ){
+    sqlite3MultiValuesStart(pParse, A);
+  }
 }
 values(A) ::= values(A) COMMA LP nexprlist(Y) RP. {
-  Select *pRight, *pLeft = A;
-  pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values|SF_MultiValue,0);
-  if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue;
-  if( pRight ){
-    pRight->op = TK_ALL;
-    pRight->pPrior = pLeft;
-    A = pRight;
-  }else{
-    A = pLeft;
-  }
+  A = sqlite3InsertMultiValues(pParse,A,Y);
 }
 
 // The "distinct" nonterminal is true (1) if the DISTINCT keyword is
@@ -1005,13 +999,38 @@ setlist(A) ::= LP idlist(X) RP EQ expr(Y). {
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) select(S)
-        upsert(U). {
-  sqlite3Insert(pParse, X, S, F, R, U);
+cmd ::= insert_head(H) insert_tail(T). {
+  sqlite3Insert(pParse, H.x, T.s, H.f, H.r, T.u);
 }
-cmd ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F) DEFAULT VALUES returning.
-{
-  sqlite3Insert(pParse, X, 0, F, R, 0);
+
+%destructor insert_tail {
+  sqlite3SelectDelete(pParse->db, $$.s);
+  sqlite3UpsertDelete(pParse->db, $$.u);
+}
+%destructor insert_head {
+  sqlite3SrcListDelete(pParse->db, $$.x);
+  sqlite3IdListDelete(pParse->db, $$.f);
+}
+
+%type insert_tail { struct insert_tail_arg { Select *s; Upsert *u; } }
+
+%type insert_head { struct insert_head_arg { int r; SrcList *x; IdList *f; } }
+
+insert_head(A) ::= with insert_cmd(R) INTO xfullname(X) idlist_opt(F). {
+  A.r = R;
+  A.x = X;
+  A.f = F;
+  if( yyLookahead==TK_VALUES ) pParse->zValuesToken = yyLookaheadToken.z;
+}
+
+insert_tail(A) ::= DEFAULT VALUES returning. {
+  A.s = 0;
+  A.u = 0;
+}
+
+insert_tail(A) ::= select(S) upsert(U). {
+  A.s = S;
+  A.u = U;
 }
 
 %type upsert {Upsert*}
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index f5920748ba..60ff698545 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1349,6 +1349,7 @@ typedef struct Walker Walker;
 typedef struct WhereInfo WhereInfo;
 typedef struct Window Window;
 typedef struct With With;
+typedef struct MultiValues MultiValues;
 
 
 /*
@@ -3907,6 +3908,8 @@ struct Parse {
 #ifndef SQLITE_OMIT_ALTERTABLE
   RenameToken *pRename;     /* Tokens subject to renaming by ALTER TABLE */
 #endif
+  MultiValues *pValues;
+  const char *zValuesToken;
 };
 
 /* Allowed values for Parse.eParseMode
@@ -5769,4 +5772,7 @@ sqlite3_uint64 sqlite3Hwtime(void);
 # define IS_STMT_SCANSTATUS(db) 0
 #endif
 
+Select *sqlite3InsertMultiValues(Parse*, Select*, ExprList*);
+void sqlite3MultiValuesStart(Parse*, Select*);
+
 #endif /* SQLITEINT_H */
diff --git a/src/vdbeaux.c b/src/vdbeaux.c
index 209d02a049..97ffb0add1 100644
--- a/src/vdbeaux.c
+++ b/src/vdbeaux.c
@@ -523,6 +523,7 @@ int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
   ** But omit them (for performance) during production builds */
   if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) )
 #endif
+  if( pParse->explain!=255 )
   {
     char *zMsg;
     Vdbe *v;
diff --git a/test/values.test b/test/values.test
new file mode 100644
index 0000000000..64ed2ca102
--- /dev/null
+++ b/test/values.test
@@ -0,0 +1,52 @@
+# 2024 March 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix values
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, c);
+}
+
+do_execsql_test 1.1.1 {
+  INSERT INTO x1 VALUES(1, 1, 1), (2, 2, 2) UNION ALL SELECT 3, 3, 3;
+}
+
+do_execsql_test 1.1.2 {
+  SELECT * FROM x1;
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+}
+
+set a 2
+set b 3
+set c 4
+do_execsql_test 1.2.1 {
+  DELETE FROM x1;
+  INSERT INTO x1 VALUES(1, 1, 1), ($a, 2, 2), (3, $b, 3), (4, 4, $c);
+}
+
+do_execsql_test 1.2.2 {
+  SELECT * FROM x1;
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}
+
+finish_test