From: dan Date: Wed, 9 Dec 2020 16:32:11 +0000 (+0000) Subject: Avoid loading large intkey rows when VACUUMing, even if the page-size is changing. X-Git-Tag: version-3.35.0~190^2~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ebbf36878ce2f1b7210be890eb676caf93c95ab2;p=thirdparty%2Fsqlite.git Avoid loading large intkey rows when VACUUMing, even if the page-size is changing. FossilOrigin-Name: 0d2c3776065dc94119899ae4164995193b82fca7ac31868f3141b729d0b65ab9 --- diff --git a/manifest b/manifest index 6cf3a15c36..9e63b89316 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\sto\svacuum\sto\savoid\sloading\slarge\srecords\sentirely\sinto\smemory.\sCurrently\sonly\sworks\sin\slimited\scases\sonly\s-\sfor\srowid\stables\swhen\sthe\spage-size\sdoes\snot\schange. -D 2020-12-08T20:19:07.385 +C Avoid\sloading\slarge\sintkey\srows\swhen\sVACUUMing,\seven\sif\sthe\spage-size\sis\schanging. +D 2020-12-09T16:32:11.593 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -481,7 +481,7 @@ F src/auth.c a3d5bfdba83d25abed1013a8c7a5f204e2e29b0c25242a56bc02bb0c07bf1e06 F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33 F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 -F src/btree.c 6d5cfd0e411e1db8042c94593a4847d85fc5398fa4860689d6f0d74cefa4e534 +F src/btree.c 2dd52a7d6f473ef345ac028f49f3cc14e1207a3596b2d9fd92fe532e57da6328 F src/btree.h cff4cd182eb0d97802d82f398e4c8447f01bd2f5e501f70386b1097c910091cb F src/btreeInt.h ffd66480520d9d70222171b3a026d78b80833b5cea49c89867949f3e023d5f43 F src/build.c f6449d4e85e998e14d3f537e8ea898dca2fcb83c277db3e60945af9b9177db81 @@ -1650,10 +1650,11 @@ F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum-into.test 48f4cec354fb6f27c98ef58d2fe49a11b71ff131af0cd9140efacc9858b9f670 F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d -F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b +F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520 F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010 F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c +F test/vacuum6.test 7d3aea27ef3c671dafccbad5adc7848205c38ab14ee7a1dabe592e0794e132d7 F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2 F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 @@ -1888,10 +1889,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4b286129138d44e6f8e9b3450289941e01d20fdfb9d0b5d846031425e8ca6b49 -R ebe894c92d077c4bc1f846403c247ebe -T *branch * vacuum-lomem -T *sym-vacuum-lomem * -T -sym-trunk * +P c90e063ca9ddcdd1e9f1a2e25a3f7d6e7ee798373ad8acf65b90536b0a124c0d +R 306b82fcb1780d3a4b54ba51609ce03f U dan -Z e00eb83dabf2298b62b079f62bce894b +Z fc13f09d46076f2b65a2bba6a14de43b diff --git a/manifest.uuid b/manifest.uuid index ccd89eef6a..e9576184e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c90e063ca9ddcdd1e9f1a2e25a3f7d6e7ee798373ad8acf65b90536b0a124c0d \ No newline at end of file +0d2c3776065dc94119899ae4164995193b82fca7ac31868f3141b729d0b65ab9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index abc28c179a..96ea9c4a04 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1143,6 +1143,24 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; } +/* +** Given a record with nPayload bytes of payload stored within btree +** page pPage, return the number of bytes of payload stored locally. +*/ +static int btreePayloadToLocal(MemPage *pPage, int nPayload){ + int maxLocal; /* Maximum amount of payload held locally */ + maxLocal = pPage->maxLocal; + if( nPayload<=maxLocal ){ + return nPayload; + }else{ + int minLocal; /* Minimum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + minLocal = pPage->minLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); + return ( surplus <= maxLocal ) ? surplus : minLocal; + } +} + /* ** The following routines are implementations of the MemPage.xParseCell() ** method. @@ -6500,7 +6518,7 @@ static int fillInCell( /* Fill in the header. */ nHeader = pPage->childPtrSize; if( pPage->intKey ){ - if( pX->pData==(const void*)pPage->pBt->pTmpSpace ){ + if( pX->nZero<0 ){ *pnSize = pX->nData; return SQLITE_OK; } @@ -8924,72 +8942,89 @@ int sqlite3BtreeTransfer( ){ int rc = SQLITE_OK; BtreePayload x; + Pager *pSrcPager = pSrc->pBt->pPager; + u8 *pCell = pDest->pBt->pTmpSpace; + u8 *aOut; /* Pointer to next output buffer */ + int nOut; /* Size of output buffer aOut[] */ + const u8 *aIn; /* Pointer to next input buffer */ + int nIn; /* Size of input buffer aIn[] */ + int nRem; /* Bytes of data still to copy */ + u8 *pPgnoOut = 0; + Pgno ovflIn = 0; + DbPage *pPageIn = 0; + MemPage *pPageOut = 0; memset(&x, 0, sizeof(x)); x.nKey = iKey; getCellInfo(pSrc); - if( pDest->pBt->usableSize!=pSrc->pBt->usableSize ){ - void *pFree = 0; - x.nData = pSrc->info.nPayload; - if( pSrc->info.nLocal==pSrc->info.nPayload ){ - x.pData = pSrc->info.pPayload; - }else{ - x.pData = pFree = sqlite3_malloc(pSrc->info.nPayload); - if( pFree==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3BtreePayload(pSrc, 0, x.nData, pFree); + + x.pData = pCell; + x.nData = putVarint32(pCell, pSrc->info.nPayload); + x.nData += putVarint(&pCell[x.nData], iKey); + x.nZero = -1; + + nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); + aOut = &pCell[x.nData]; + nIn = pSrc->info.nLocal; + aIn = pSrc->info.pPayload; + nRem = pSrc->info.nPayload; + + x.nData += nOut; + if( nOutinfo.nPayload ){ + pPgnoOut = &pCell[x.nData]; + x.nData += 4; + } + + if( nRem>nIn ){ + ovflIn = get4byte(&pSrc->info.pPayload[nIn]); + } + + do { + nRem -= nOut; + do{ + assert( nOut>0 ); + if( nIn>0 ){ + int nCopy = MIN(nOut, nIn); + memcpy(aOut, aIn, nCopy); + nOut -= nCopy; + nIn -= nCopy; + aOut += nCopy; + aIn += nCopy; } - } - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult); - } - sqlite3_free(pFree); - }else{ - /* Page sizes match. This means each overflow page (if any) and the - ** cell itself can be copied verbatim. */ - u8 *pCell = pDest->pBt->pTmpSpace; - x.pData = pCell; - x.nData = putVarint32(pCell, pSrc->info.nPayload); - x.nData += putVarint(&pCell[x.nData], iKey); - memcpy(&pCell[x.nData], pSrc->info.pPayload, pSrc->info.nLocal); - x.nData += pSrc->info.nLocal; - assert( pSrc->info.nLocal<=pSrc->info.nPayload ); - if( pSrc->info.nLocalinfo.nPayload ){ - Pager *pSrcPager = pSrc->pBt->pPager; - u8 *pPgno = &pCell[x.nData]; - Pgno ovfl; - x.nData += 4; - ovfl = get4byte(pSrc->info.pPayload + pSrc->info.nLocal); - do{ - MemPage *pNew = 0; - Pgno pgnoNew = 0; - if( rc==SQLITE_OK ){ - rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0); - put4byte(pPgno, pgnoNew); - } - if( rc==SQLITE_OK ){ - pPgno = pNew->aData; - rc = sqlite3PagerWrite(pNew->pDbPage); - } + if( nOut>0 ){ + sqlite3PagerUnref(pPageIn); + pPageIn = 0; + rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY); if( rc==SQLITE_OK ){ - DbPage *pOrig = 0; - void *pOrigData; - rc = sqlite3PagerGet(pSrcPager, ovfl, &pOrig, PAGER_GET_READONLY); - if( rc==SQLITE_OK ){ - pOrigData = sqlite3PagerGetData(pOrig); - memcpy(pNew->aData, pOrigData, pDest->pBt->usableSize); - put4byte(pNew->aData, 0); - ovfl = get4byte(pOrigData); - } - sqlite3PagerUnref(pOrig); + aIn = (const u8*)sqlite3PagerGetData(pPageIn); + ovflIn = get4byte(aIn); + aIn += 4; + nIn = pSrc->pBt->usableSize - 4; } - releasePage(pNew); - }while( rc==SQLITE_OK && ovfl>0 ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult); + } + }while( rc==SQLITE_OK && nOut>0 ); + + if( rc==SQLITE_OK && nRem>0 ){ + Pgno pgnoNew; + MemPage *pNew = 0; + rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0); + put4byte(pPgnoOut, pgnoNew); + releasePage(pPageOut); + pPageOut = pNew; + if( pPageOut ){ + pPgnoOut = pPageOut->aData; + put4byte(pPgnoOut, 0); + aOut = &pPgnoOut[4]; + nOut = MIN(pDest->pBt->usableSize - 4, nRem); + } } + }while( nRem>0 && rc==SQLITE_OK ); + + releasePage(pPageOut); + sqlite3PagerUnref(pPageIn); + + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult); } return rc; diff --git a/test/vacuum2.test b/test/vacuum2.test index 0350c8ec42..89fdb506c5 100644 --- a/test/vacuum2.test +++ b/test/vacuum2.test @@ -56,7 +56,7 @@ do_test vacuum2-2.1 { } hexio_get_int [hexio_read test.db 24 4] } [expr {[hexio_get_int [hexio_read test.db 24 4]]+3}] -do_test vacuum2-2.1 { +do_test vacuum2-2.2 { execsql { VACUUM } diff --git a/test/vacuum6.test b/test/vacuum6.test new file mode 100644 index 0000000000..203b3329e2 --- /dev/null +++ b/test/vacuum6.test @@ -0,0 +1,94 @@ +# 2016-08-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements a test for VACUUM on attached databases. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix vacuum6 + +# If the VACUUM statement is disabled in the current build, skip all +# the tests in this file. +# +ifcapable !vacuum { + finish_test + return +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(1, 1); +} {} +do_execsql_test 1.1 { + VACUUM +} + +#------------------------------------------------------------------------- +reset_db +foreach {tn sz} {1 400 2 4000 3 9999} { + reset_db + do_execsql_test 2.$tn.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO t1 SELECT i, randomblob($sz) FROM s; + } + + do_execsql_test 2.$tn.2 { + vacuum; + } + + do_execsql_test 2.$tn.3 { + PRAGMA integrity_check; + } {ok} +} + +reset_db +do_execsql_test 3.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(2, randomblob(1200)); +} {} +do_execsql_test 3.1 { + PRAGMA page_size = 512; + VACUUM; +} +do_execsql_test 3.2 { + PRAGMA integrity_check +} {ok} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE tx(a, b); + CREATE INDEX i1 ON tx(b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO tx SELECT i, randomblob(i) FROM s; +} +foreach {tn pgsz} { + 1 2048 + 2 1024 + 3 65536 + 4 8192 +} { + do_execsql_test 4.1.$tn.1 " + PRAGMA page_size = $pgsz + " + do_execsql_test 4.1.$tn.2 VACUUM + integrity_check 4.1.$tn.3 +} + +finish_test