From: dan Date: Thu, 25 Sep 2025 19:32:29 +0000 (+0000) Subject: Include writes to temp files by sort operations in the SQLITE_DBSTATUS_SPILL values. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=51e3a140c97b5c50a2b389e96529ccb7fbe43a9b;p=thirdparty%2Fsqlite.git Include writes to temp files by sort operations in the SQLITE_DBSTATUS_SPILL values. FossilOrigin-Name: 4c1118ba40172af61d9304dec7a9c7344566e6a5963a317880ad232bebb40176 --- diff --git a/manifest b/manifest index 07fa32d267..eda41b88c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Include\sI/O\sconsumed\sby\sephermeral\scursors\sin\sthe\sSQLITE_DBSTATUS_SPILL\nvalues.\s\sStill\sneed\sto\sadd\ssorter. -D 2025-09-18T18:56:55.622 +C Include\swrites\sto\stemp\sfiles\sby\ssort\soperations\sin\sthe\sSQLITE_DBSTATUS_SPILL\svalues. +D 2025-09-25T19:32:29.107 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea @@ -778,7 +778,7 @@ F src/test_intarray.c 3fcf8ca7bb5c8776ea83f6aa9b66f8df0d1f37a99207b0097c8486f9c1 F src/test_intarray.h 6c3534641108cd1bea517a8e117dcba237081310a29a4c35bd2190caa8972293 F src/test_journal.c a0b9709b2f12b1ec819eea8a1176f283bca6d688a6d4a502bd6fd79786f4e287 F src/test_loadext.c 337056bae59f80b9eb00ba82088b39d0f4fe6dfd -F src/test_malloc.c f1901a7a32fadff1978d6aca8e3bd2c6cc1e41b790ff19d369cc0367116a1828 +F src/test_malloc.c 50966e2967750115c4d06b6943c64da552577e4f3d060e6953efed8d66aff2c4 F src/test_md5.c 811a45330c9391933360f998156a8907ee29909c828ab83ac05d329942cbea8f F src/test_multiplex.c 82f0aa8eee629b6949782cfab8782ed35a9b56dc80d12877af52147f304d22b8 F src/test_multiplex.h f0ff5b6f4462bfd46dac165d6375b9530d08089b7bcbe75e88e0926110db5363 @@ -816,7 +816,7 @@ F src/vdbeapi.c f9a4881a9674fec3fa13da35044a1484d3c4b95f9ec891cc8ffb02ef2b7a41df F src/vdbeaux.c a0718e0ff1e336f7d9b4df5a7fe38fc6eb9b3779871e3768d060fa88a76b8d14 F src/vdbeblob.c b3f0640db9642fbdc88bd6ebcc83d6009514cafc98f062f675f2c8d505d82692 F src/vdbemem.c e67d9c6484d868c879d20c70d00bf4a9058082f1d4058607ca15d50eb3aebc21 -F src/vdbesort.c cb6f472e83ca12c46aa7de0ac0a9d11458b357986f2617a1c90dfb19a542ecbe +F src/vdbesort.c b69220f4ea9ffea5fdef34d968c60305444eea909252a81933b54c296d9cca70 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c fc46b9cbd759dc013f0b3724549cc0d71379183c667df3a5988f7e2f1bd485f3 F src/vtab.c 828221bdbeaaa6d62126ee6d07fd4ec0d09dcaea846f87ad01944d8b7e548859 @@ -1044,7 +1044,7 @@ F test/dblwidth-a.sql eb4141518610e52f931a55a984310075e98dc31eee5a28ae806b1e3537 F test/dbpage.test 63fab1eb026bada121107e53436fa749bbf83281dc9dea17af422f7a5c0f289f F test/dbpagefault.test ea39de2ca86041a9c6df1135645180a76d0a8da93ac159e2fafe38e39636530b F test/dbstatus.test 4a4221a883025ffd39696b3d1b3910b928fb097d77e671351acb35f3aed42759 -F test/dbstatus2.test f5fe0afed3fa45e57cfa70d1147606c20d2ba23feac78e9a172f2fe8ab5b78ef +F test/dbstatus2.test b9c73a59b6f5abd8de6086550114a3ee3228ef13888756e8f074d9b9264ddbaa F test/decimal.test ef731887b43ee32ef86e1c8fddb61a40789f988332c029c601dcf2c319277e9e F test/default.test c7124864cded213a3f118bc7e2e26f34b7c36dfa26cf6945cc8b7f5db1191277 F test/delete.test 2686e1c98d552ef37d79ad55b17b93fe96fad9737786917ce3839767f734c48f @@ -2175,8 +2175,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4e63ee20f80e486ecc4aa4523afa43f994c267e638f8f084f0d6df369aa74a1d -R 23ddf07861ed9bfd32891e000fdd4062 -U drh -Z aaf208a8aa761385255c814551acbbd3 +P 118905b26a96825433f79b47fd3e7c2bdb137d84292fb9419e4436a961a9833a +R 12194ee4659fc7e61523e3e60d402d56 +U dan +Z 54bf731d38cd3b8cac77a45a947299fd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c8916bb36..d66b6b2ff4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -118905b26a96825433f79b47fd3e7c2bdb137d84292fb9419e4436a961a9833a +4c1118ba40172af61d9304dec7a9c7344566e6a5963a317880ad232bebb40176 diff --git a/src/test_malloc.c b/src/test_malloc.c index 1c19d896f2..1d1f9d04f8 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1368,6 +1368,7 @@ static int SQLITE_TCLAPI test_db_status( { "DEFERRED_FKS", SQLITE_DBSTATUS_DEFERRED_FKS }, { "CACHE_USED_SHARED", SQLITE_DBSTATUS_CACHE_USED_SHARED }, { "CACHE_SPILL", SQLITE_DBSTATUS_CACHE_SPILL }, + { "SPILL", SQLITE_DBSTATUS_SPILL }, }; Tcl_Obj *pResult; if( objc!=4 ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index 39661eb4c4..73f3301116 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -302,6 +302,7 @@ struct SortSubtask { SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ + u64 nSpill; /* Total bytes written by this task */ }; @@ -422,6 +423,7 @@ struct PmaWriter { int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ sqlite3_file *pFd; /* File handle to write to */ + u64 nPmaSpill; /* Total number of bytes written */ }; /* @@ -1280,6 +1282,12 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; if( pSorter ){ + /* Increment db->nSpill by the total number of bytes of data written + ** to temp files by this sort operation. */ + int ii; + for(ii=0; iinTask; ii++){ + db->nSpill += pSorter->aTask[ii].nSpill; + } sqlite3VdbeSorterReset(db, pSorter); sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); @@ -1505,6 +1513,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); p->iBufStart = p->iBufEnd = 0; p->iWriteOff += p->nBuffer; } @@ -1521,17 +1530,20 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the -** last byte written to the file. +** last byte written to the file. Also, increment (*pnSpill) by the total +** number of bytes written to the file. */ -static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof, u64 *pnSpill){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); + p->nPmaSpill += (p->iBufEnd - p->iBufStart); } *piEof = (p->iWriteOff + p->iBufEnd); + *pnSpill += p->nPmaSpill; sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(PmaWriter)); @@ -1611,7 +1623,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ if( pList->aMemory==0 ) sqlite3_free(p); } pList->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof, &pTask->nSpill); } vdbeSorterWorkDebug(pTask, "exit"); @@ -1925,7 +1937,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } - rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof, &pTask->nSpill); if( rc==SQLITE_OK ) rc = rc2; vdbeSorterPopulateDebug(pTask, "exit"); return rc; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 5e9ea888a6..cca606755e 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -41,6 +41,10 @@ proc db_spill {db {reset 0}} { sqlite3_db_status $db CACHE_SPILL $reset } +proc db_temp_spill {db {reset 0}} { + sqlite3_db_status $db SPILL $reset +} + do_test 1.1 { db close sqlite3 db test.db @@ -111,5 +115,50 @@ do_execsql_test 3.2 { UPDATE t1 SET b=randomblob(1000); } {delete} do_test 3.3 { db_spill db 0 } {0 8 0} + + +if {$::TEMP_STORE<3} { + do_execsql_test 4.0 { + PRAGMA temp_store = file; + PRAGMA cache_size = -1024; + } {} + + do_test 4.1 { db_temp_spill db 0 } {0 0 0} + + do_execsql_test 4.2 { + CREATE TABLE data(a INTEGER, b BLOB); + + -- Insert 5-6 MB of data. + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<50000 ) + INSERT INTO data SELECT i, hex(randomblob(50)) FROM s; + } + + do_test 4.3 { db_temp_spill db 0 } {0 0 0} + + do_test 4.4 { + execsql { SELECT a, b FROM data ORDER BY a } + set nTmpSpill [lindex [db_temp_spill db 1] 1] + expr ($nTmpSpill>5*1000*1000) && ($nTmpSpill<6*1000*1000) + } 1 + + # The previous test case reset the status value. + do_test 4.5 { db_temp_spill db 0 } {0 0 0} + + do_test 4.6 { + execsql { CREATE INDEX i1 ON data(a) } + set nTmpSpill [lindex [db_temp_spill db 1] 1] + expr ($nTmpSpill>256*1000) && ($nTmpSpill<512*1000) + } 1 + + # The previous test case reset the status value. + do_test 4.7 { db_temp_spill db 0 } {0 0 0} + + # Same query as in (4.4). Now does not require temp space. + do_test 4.8 { + execsql { SELECT a, b FROM data ORDER BY a } + db_temp_spill db 0 + } {0 0 0} +} + finish_test