From: drh <> Date: Tue, 10 Mar 2026 18:13:13 +0000 (+0000) Subject: Revamp the QRF insert-mode optimization of [659ff6ab55802507]. The X-Git-Tag: major-release~88 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=abc82c49c2428b8407b974686c130f77c7367ab9;p=thirdparty%2Fsqlite.git Revamp the QRF insert-mode optimization of [659ff6ab55802507]. The control value is now nMultiInsert. Multiple rows are added to each INSERT statement until the number of bytes in that statement exceeds the nMultiInsert value. The CLI uses a default value of 3000, which provides a good balance between speed and prepare-statement size. The output from ".dump" now loads nearly 2x faster in some cases. FossilOrigin-Name: 55b0ce9f93d68b17a18423f0f751b778573127743a2bac67abfea074ac32d41a --- diff --git a/ext/qrf/qrf.c b/ext/qrf/qrf.c index 6e6c379e7b..2ed23d728f 100644 --- a/ext/qrf/qrf.c +++ b/ext/qrf/qrf.c @@ -66,7 +66,7 @@ struct Qrf { int iIndent; /* Current slot */ int *aiIndent; /* Indentation for each opcode */ } sExpln; - unsigned int nTuple; /* Tuples issued under QRF_STYLE_Insert */ + unsigned int nIns; /* Bytes used for current INSERT stmt */ } u; sqlite3_int64 nRow; /* Number of rows handled so far */ int *actualWidth; /* Actual width of each column */ @@ -2564,12 +2564,13 @@ static void qrfOneSimpleRow(Qrf *p){ break; } case QRF_STYLE_Insert: { - int mxTuple = p->spec.iVersion>=2 ? p->spec.mxTuple : 1; - if( p->u.nTuple==0 || p->u.nTuple>=mxTuple ){ - if( p->u.nTuple ){ + int mxIns = p->spec.iVersion>=2 ? p->spec.nMultiInsert : 0; + int szStart = sqlite3_str_length(p->pOut); + if( p->u.nIns==0 || p->u.nIns>=mxIns ){ + if( p->u.nIns ){ sqlite3_str_append(p->pOut, ";\n", 2); + p->u.nIns = 0; } - p->u.nTuple = 0; if( qrf_need_quote(p->spec.zTableName) ){ sqlite3_str_appendf(p->pOut,"INSERT INTO \"%w\"",p->spec.zTableName); }else{ @@ -2596,10 +2597,10 @@ static void qrfOneSimpleRow(Qrf *p){ if( i>0 ) sqlite3_str_append(p->pOut, ",", 1); qrfRenderValue(p, p->pOut, i); } - p->u.nTuple++; - if( p->u.nTuple>=mxTuple ){ + p->u.nIns += sqlite3_str_length(p->pOut) + 2 - szStart; + if( p->u.nIns>=mxIns ){ sqlite3_str_append(p->pOut, ");\n", 3); - p->u.nTuple = 0; + p->u.nIns = 0; }else{ sqlite3_str_append(p->pOut, ")", 1); } @@ -2771,7 +2772,7 @@ qrf_reinit: if( p->spec.zTableName==0 || p->spec.zTableName[0]==0 ){ p->spec.zTableName = "tab"; } - p->u.nTuple = 0; + p->u.nIns = 0; break; } case QRF_STYLE_Line: { @@ -2885,7 +2886,7 @@ static void qrfFinalize(Qrf *p){ break; } case QRF_STYLE_Insert: { - if( p->u.nTuple ){ + if( p->u.nIns ){ sqlite3_str_append(p->pOut, ";\n", 2); } break; diff --git a/ext/qrf/qrf.h b/ext/qrf/qrf.h index e93e8947c2..fbea33a21f 100644 --- a/ext/qrf/qrf.h +++ b/ext/qrf/qrf.h @@ -57,7 +57,7 @@ struct sqlite3_qrf_spec { void *pWriteArg; /* First argument to the xWrite callback */ char **pzOutput; /* Storage location for output string */ /* Fields below are only available if iVersion>=2 */ - unsigned int mxTuple; /* Tuples per INSERT in QRF_STYLE_Insert */ + unsigned int nMultiInsert; /* Add rows to one INSERT until size exceeds */ /* Additional fields may be added in the future */ }; diff --git a/manifest b/manifest index 8648c8cdf5..70f6b3490a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bug\sfix\sto\sthe\s".dump"\scommand\sin\sthe\sCLI:\s\sTemporarily\sdisable\scharacter\nlimit\sto\sQRF\sso\sas\snot\sto\struncate\svalues\sin\sthe\sgenerated\soutput. -D 2026-03-10T15:43:57.174 +C Revamp\sthe\sQRF\sinsert-mode\soptimization\sof\s[659ff6ab55802507].\s\sThe\ncontrol\svalue\sis\snow\snMultiInsert.\s\sMultiple\srows\sare\sadded\sto\seach\sINSERT\nstatement\suntil\sthe\snumber\sof\sbytes\sin\sthat\sstatement\sexceeds\sthe\nnMultiInsert\svalue.\s\sThe\sCLI\suses\sa\sdefault\svalue\sof\s3000,\swhich\sprovides\na\sgood\sbalance\sbetween\sspeed\sand\sprepare-statement\ssize.\s\sThe\soutput\nfrom\s".dump"\snow\sloads\snearly\s2x\sfaster\sin\ssome\scases. +D 2026-03-10T18:13:13.382 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -420,8 +420,8 @@ F ext/misc/zipfile.c c8ee04e1b349270b5df401ad732f5d7c387146e69b33c02fa90322760cc F ext/misc/zorder.c bddff2e1b9661a90c95c2a9a9c7ecd8908afab5763256294dd12d609d4664eee F ext/qrf/README.md e6e0ce2700acf6fd06312b42726a8f08ca240f30e1b122bff87c71c602046352 F ext/qrf/dev-notes.md e68a6d91ce4c7eb296ef2daadc2bb79c95c317ad15b9fafe40850c67b29c2430 -F ext/qrf/qrf.c d9f2e9fe7f34e700cd4245d39b1272bb62fc360ba95aa3de264bc6521ad8dfa0 -F ext/qrf/qrf.h bdd1711b954c2a3714f22d22693e3d76eb8de3db5efa6afa00c55ed1d7f61028 +F ext/qrf/qrf.c d7ed1d2e2460637f970b796e9f04774cc96dbd6c7a85cfff4c78746c2f6c51fe +F ext/qrf/qrf.h 2b27cb8079131ac3af2134e9e2ee9621b41d97bcb546fead41b9e1a12a3567a7 F ext/rbu/rbu.c 801450b24eaf14440d8fd20385aacc751d5c9d6123398df41b1b5aa804bf4ce8 F ext/rbu/rbu1.test 25870dd7db7eb5597e2b4d6e29e7a7e095abf332660f67d89959552ce8f8f255 F ext/rbu/rbu10.test 7c22caa32c2ff26983ca8320779a31495a6555737684af7aba3daaf762ef3363 @@ -733,7 +733,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 928ff887f2a7c64275182060d94d06fdddbe32226c569781cf7e7edc6f58d7fd F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c ffe199f025a0dd74670d2a77232bdea364a4d7b36f32c64a6572d39ba6a11576 -F src/shell.c.in 339f8e92a8531375deb8dc649d60948149a297c55623e63f544f22d0f2700145 +F src/shell.c.in dc19fd2fddbfec60f52852edbb4a4e3723e23e00c63b2ea289800fa4d37b7a6e F src/sqlite.h.in 4d657846d68a58b028f0c4c331b9d3b4a79306f25c3b0d04fb56060343f73d85 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479 F src/sqlite3ext.h 1b7a0ee438bb5c2896d0609c537e917d8057b3340f6ad004d2de44f03e3d3cca @@ -741,7 +741,7 @@ F src/sqliteInt.h 9716721fb57e32938a1d30a84560ce7633c63860a2209e188c87afad15d4b4 F src/sqliteLimit.h 904a3f520362c7065c18165aaabd504fb13cc1b76cb411f38bd41ac219e4af1e F src/status.c 7565d63a79aa2f326339a24a0461a60096d0bd2bce711fefb50b5c89335f3592 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 -F src/tclsqlite.c bebd32b937ec1542234cc63228720ecee7be1a6cd682fb6b55378d7ae01b8bc0 +F src/tclsqlite.c 71230da35d81b7234c5e4dc3747ca2302390884b051052956deb3dbf680a6ed4 F src/tclsqlite.h 614b3780a62522bc9f8f2b9fb22689e8009958e7aa77e572d0f3149050af348a F src/test1.c 3e3b013f59ffcb57dce00c90d55907072d71d4e970cb0a590cb261efe11bae9c F src/test2.c 62f0830958f9075692c29c6de51b495ae8969e1bef85f239ffcd9ba5fb44a5ff @@ -1055,7 +1055,7 @@ F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 691c9e850b0d0b56b66e7e235453198cb4cf0760e324b7403d3c5abbeab0a014 F test/distinct2.test 072f33e1348b5cae4156e7ca4c124d21053f77d96d5d960a1ba21806416074ab F test/distinctagg.test 40d7169ae5846caaf62c6e307d2ca3c333daf9b6f7cde888956a339a97afe85f -F test/dotcmd01.sql 0388a778912ed08436ae5c80e03389d8bd347fa724611193257a18c69692019d +F test/dotcmd01.sql dd5b0475df723fdb9f91031a7b9684abb279f6a686178f0d8e3b5f1da93c27c2 F test/e_blobbytes.test 4c01dfe4f12087b92b20705a3fdfded45dc4ed16d5a211fed4e1d2786ba68a52 F test/e_blobclose.test 692fc02a058476c2222a63d97e3f3b2b809c1842e5525ded7f854d540ac2e075 F test/e_blobopen.test 29f6055ee453b8e679fe9570c4d3acfedbef821622c5dad16875148c5952ef50 @@ -1445,7 +1445,7 @@ F test/mmap4.test 2e2b4e32555b58da15176e6fe750f17c9dcf7f93 F test/mmapcorrupt.test 470fb44fe92e99c1d23701d156f8c17865f5b027063c9119dcfdb842791f4465 F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/mmapwarm.test 2272005969cd17a910077bd5082f70bc1fefad9a875afec7fc9af483898ecaf3 -F test/modeA.sql 24417bc672b9e3dff3243b6678595ecbf77ef30827b5e24f9faaec9edc30b650 +F test/modeA.sql fc64f646b0a1d0806af122fad6db2c89de63c51106655c09d44a652052a14d05 F test/multiplex.test d74c034e52805f6de8cc5432cef8c9eb774bb64ec29b83a22effc8ca4dac1f08 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test fac575e0b1b852025575a6a8357701d80933e98b5d2fe6d35ddaa68f92f6a1f7 @@ -1515,7 +1515,7 @@ F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224c F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/pushdown.test 46a626ef1c0ca79b85296ff2e078b9da20a50e9b804b38f441590c3987580ddd -F test/qrf01.test d4bf258413592b8beb43d838f89c9f57882fc74efef556d2a201b7372ec543a7 +F test/qrf01.test 09caa00e6b4deea5fcd8958b4062233fcef5ef1b354312ec956ec4bc3a09572a F test/qrf02.test 39b4afdc000bedccdafc0aecf17638df67a67aaa2d2942865ae6abcc48ba0e92 F test/qrf03.test e7efe46d204671726b4707585126cd78d107368de4a7d0c7b8d5157cdd8624ed F test/qrf04.test 0894692c998d2401dcc33449c02051b503ecce0c94217be54fb007c82d2d1379 @@ -2192,8 +2192,8 @@ F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee F tool/warnings.sh d924598cf2f55a4ecbc2aeb055c10bd5f48114793e7ba25f9585435da29e7e98 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/winmain.c 00c8fb88e365c9017db14c73d3c78af62194d9644feaf60e220ab0f411f3604c -P 659ff6ab5580250707908af003ecd093bd5313f03f21f0efd000a7aff6638b3c -R c7258df5c286f69cf6ad0fcffe82369d +P f6787e4b50af50663d8fa00ebe0694440354a7ad0509837a6b249e090e6cffca +R d860bc66dd6bb7267fd1bfd3a3708814 U drh -Z ec10a6a2386acbe38f4be26bb414e44b +Z d618cb096ff3c243141ddcd38eae46e6 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 41e0474e1a..ceeffaa003 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6787e4b50af50663d8fa00ebe0694440354a7ad0509837a6b249e090e6cffca +55b0ce9f93d68b17a18423f0f751b778573127743a2bac67abfea074ac32d41a diff --git a/src/shell.c.in b/src/shell.c.in index 2dee2cf92b..c007856b2b 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1375,6 +1375,9 @@ static const ModeInfo aModeInfo[] = { #ifndef DFLT_TITLE_LIMIT # define DFLT_TITLE_LIMIT 20 #endif +#ifndef DFLT_MULTI_INSERT +# define DFLT_MULTI_INSERT 3000 +#endif /* ** Limit input nesting via .read or any other input redirect. @@ -1677,6 +1680,7 @@ static void modeChange(ShellState *p, unsigned char eMode){ p->mode.spec.nLineLimit = DFLT_LINE_LIMIT; p->mode.spec.bTextJsonb = QRF_Yes; p->mode.spec.nTitleLimit = DFLT_TITLE_LIMIT; + p->mode.spec.nMultiInsert = DFLT_MULTI_INSERT; p->mode.mFlags = mFlags; } } @@ -7921,9 +7925,8 @@ static int modeTitleDsply(ShellState *p, int bAll){ ** can also be "off" to mean "0,0,0" or "on" to ** mean "5,300,20". ** --list List available modes -** --mxtuple N In "insert" mode, try to put as many as N -** comma-separated tuples after the VALUES in each -** INSERT statement. +** --multiinsert N In "insert" mode, put multiple rows on a single +** INSERT statement until the size exceeds N bytes. ** --null STRING Render SQL NULL values as the given string ** --once Setting changes to the right are reverted after ** the next SQL command. @@ -8040,7 +8043,7 @@ static int dotCmdMode(ShellState *p){ } chng = 1; }else if( 0<=(k=pickStr(z,0, - "-charlimit","-linelimit","-titlelimit","-mxtuple","")) ){ + "-charlimit","-linelimit","-titlelimit","-multiinsert","")) ){ int w; /* 0 1 2 3 */ if( i+1>=nArg ){ dotCmdError(p, i, "missing argument", 0); @@ -8048,10 +8051,10 @@ static int dotCmdMode(ShellState *p){ } w = integerValue(azArg[++i]); switch( k ){ - case 0: p->mode.spec.nCharLimit = w; break; - case 1: p->mode.spec.nLineLimit = w; break; - case 2: p->mode.spec.nTitleLimit = w; break; - default: p->mode.spec.mxTuple = w; break; + case 0: p->mode.spec.nCharLimit = w; break; + case 1: p->mode.spec.nLineLimit = w; break; + case 2: p->mode.spec.nTitleLimit = w; break; + default: p->mode.spec.nMultiInsert = w; break; } chng = 1; }else if( 0<=(k=pickStr(z,0,"-tablename","-rowsep","-colsep","-null","")) ){ @@ -8395,9 +8398,10 @@ static int dotCmdMode(ShellState *p){ } } if( bAll - || (p->mode.spec.mxTuple>1 && p->mode.spec.eStyle==QRF_STYLE_Insert) + || (p->mode.spec.nMultiInsert && p->mode.spec.eStyle==QRF_STYLE_Insert) ){ - sqlite3_str_appendf(pDesc, " --mxtuple %u", p->mode.spec.mxTuple); + sqlite3_str_appendf(pDesc, " --multiinsert %u", + p->mode.spec.nMultiInsert); } zSetting = aModeStr[pI->eNull]; if( bAll || (zSetting && cli_strcmp(zSetting,p->mode.spec.zNull)!=0) ){ diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 7f51096fb7..3e2db54a4e 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2071,7 +2071,7 @@ static void DbHookCmd( ** -linelimit NUMBER Max lines for any cell ** -charlimit NUMBER Content truncated to this size ** -titlelimit NUMBER Max width of column titles -** -mxtuple NUMBER Max tuples per INSERT +** -multiinsert NUMBER Multi-row INSERT byte size ** -align LIST-OF-ALIGNMENT Alignment of columns ** -widths LIST-OF-NUMBERS Widths for individual columns ** -columnsep TEXT Column separator text @@ -2100,7 +2100,7 @@ static void DbHookCmd( ** -linelimit nLineLimit ** -charlimit nCharLimit ** -titlelimit nTitleLimit -** -mxtuple mxTuple +** -multiinsert nMultiInsert ** -align nAlign, aAlign ** -widths nWidth, aWidth ** -columnsep zColumnSep @@ -2303,12 +2303,12 @@ static int dbQrf(SqliteDb *pDb, int objc, Tcl_Obj *const*objv){ if( v<0 ) v = 0; qrf.nCharLimit = v; i++; - }else if( strcmp(zArg,"-mxtuple")==0 ){ + }else if( strcmp(zArg,"-multiinsert")==0 ){ int v = 0; rc = Tcl_GetIntFromObj(pDb->interp, objv[i+1], &v); if( rc ) goto format_failed; if( v<0 ) v = 0; - qrf.mxTuple = v; + qrf.nMultiInsert = v; i++; }else if( strcmp(zArg,"-align")==0 ){ Tcl_Size n = 0; diff --git a/test/dotcmd01.sql b/test/dotcmd01.sql index 751d326592..75595d6479 100644 --- a/test/dotcmd01.sql +++ b/test/dotcmd01.sql @@ -40,8 +40,8 @@ CREATE TABLE t1(a,b,c); CREATE INDEX t1a ON t1(a); CREATE INDEX t1b ON t1(b); ANALYZE sqlite_schema; -INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'); -INSERT INTO sqlite_stat1 VALUES('t1','t1a','300 30'); +INSERT INTO sqlite_stat1 VALUES('t1','t1b','300 10'), + ('t1','t1a','300 30'); ANALYZE sqlite_schema; END diff --git a/test/modeA.sql b/test/modeA.sql index a031995b56..48f71c0c92 100644 --- a/test/modeA.sql +++ b/test/modeA.sql @@ -87,7 +87,7 @@ END .testcase 140 .mode -v .check <