-C Experimental\schanges\sto\svacuum\sto\savoid\sloading\slarge\srecords\sentirely\sinto\smemory.\sCurrently\sonly\sworks\sin\slimited\scases\sonly\s-\sfor\srowid\stables\swhen\sthe\spage-size\sdoes\snot\schange.
-D 2020-12-08T20:19:07.385
+C Avoid\sloading\slarge\sintkey\srows\swhen\sVACUUMing,\seven\sif\sthe\spage-size\sis\schanging.
+D 2020-12-09T16:32:11.593
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F src/backup.c 3014889fa06e20e6adfa0d07b60097eec1f6e5b06671625f476a714d2356513d
F src/bitvec.c 17ea48eff8ba979f1f5b04cc484c7bb2be632f33
F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
-F src/btree.c 6d5cfd0e411e1db8042c94593a4847d85fc5398fa4860689d6f0d74cefa4e534
+F src/btree.c 2dd52a7d6f473ef345ac028f49f3cc14e1207a3596b2d9fd92fe532e57da6328
F src/btree.h cff4cd182eb0d97802d82f398e4c8447f01bd2f5e501f70386b1097c910091cb
F src/btreeInt.h ffd66480520d9d70222171b3a026d78b80833b5cea49c89867949f3e023d5f43
F src/build.c f6449d4e85e998e14d3f537e8ea898dca2fcb83c277db3e60945af9b9177db81
F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae
F test/vacuum-into.test 48f4cec354fb6f27c98ef58d2fe49a11b71ff131af0cd9140efacc9858b9f670
F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d
-F test/vacuum2.test aa048abee196c16c9ba308465494009057b79f9b
+F test/vacuum2.test 9fd45ce6ce29f5614c249e03938d3567c06a9e772d4f155949f8eafe2d8af520
F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d
F test/vacuum4.test 7ea76b769fffeb41f925303b04cbcf5a5bbeabe55e4c60ae754ff24eeeb7c010
F test/vacuum5.test 263b144d537e92ad8e9ca8a73cc6e1583f41cfd0dda9432b87f7806174a2f48c
+F test/vacuum6.test 7d3aea27ef3c671dafccbad5adc7848205c38ab14ee7a1dabe592e0794e132d7
F test/vacuummem.test 7b42abb3208bd82dd23a7536588396f295a314f2
F test/varint.test bbce22cda8fc4d135bcc2b589574be8410614e62
F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4b286129138d44e6f8e9b3450289941e01d20fdfb9d0b5d846031425e8ca6b49
-R ebe894c92d077c4bc1f846403c247ebe
-T *branch * vacuum-lomem
-T *sym-vacuum-lomem *
-T -sym-trunk *
+P c90e063ca9ddcdd1e9f1a2e25a3f7d6e7ee798373ad8acf65b90536b0a124c0d
+R 306b82fcb1780d3a4b54ba51609ce03f
U dan
-Z e00eb83dabf2298b62b079f62bce894b
+Z fc13f09d46076f2b65a2bba6a14de43b
pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4;
}
+/*
+** Given a record with nPayload bytes of payload stored within btree
+** page pPage, return the number of bytes of payload stored locally.
+*/
+static int btreePayloadToLocal(MemPage *pPage, int nPayload){
+ int maxLocal; /* Maximum amount of payload held locally */
+ maxLocal = pPage->maxLocal;
+ if( nPayload<=maxLocal ){
+ return nPayload;
+ }else{
+ int minLocal; /* Minimum amount of payload held locally */
+ int surplus; /* Overflow payload available for local storage */
+ minLocal = pPage->minLocal;
+ surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4);
+ return ( surplus <= maxLocal ) ? surplus : minLocal;
+ }
+}
+
/*
** The following routines are implementations of the MemPage.xParseCell()
** method.
/* Fill in the header. */
nHeader = pPage->childPtrSize;
if( pPage->intKey ){
- if( pX->pData==(const void*)pPage->pBt->pTmpSpace ){
+ if( pX->nZero<0 ){
*pnSize = pX->nData;
return SQLITE_OK;
}
){
int rc = SQLITE_OK;
BtreePayload x;
+ Pager *pSrcPager = pSrc->pBt->pPager;
+ u8 *pCell = pDest->pBt->pTmpSpace;
+ u8 *aOut; /* Pointer to next output buffer */
+ int nOut; /* Size of output buffer aOut[] */
+ const u8 *aIn; /* Pointer to next input buffer */
+ int nIn; /* Size of input buffer aIn[] */
+ int nRem; /* Bytes of data still to copy */
+ u8 *pPgnoOut = 0;
+ Pgno ovflIn = 0;
+ DbPage *pPageIn = 0;
+ MemPage *pPageOut = 0;
memset(&x, 0, sizeof(x));
x.nKey = iKey;
getCellInfo(pSrc);
- if( pDest->pBt->usableSize!=pSrc->pBt->usableSize ){
- void *pFree = 0;
- x.nData = pSrc->info.nPayload;
- if( pSrc->info.nLocal==pSrc->info.nPayload ){
- x.pData = pSrc->info.pPayload;
- }else{
- x.pData = pFree = sqlite3_malloc(pSrc->info.nPayload);
- if( pFree==0 ){
- rc = SQLITE_NOMEM_BKPT;
- }else{
- rc = sqlite3BtreePayload(pSrc, 0, x.nData, pFree);
+
+ x.pData = pCell;
+ x.nData = putVarint32(pCell, pSrc->info.nPayload);
+ x.nData += putVarint(&pCell[x.nData], iKey);
+ x.nZero = -1;
+
+ nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload);
+ aOut = &pCell[x.nData];
+ nIn = pSrc->info.nLocal;
+ aIn = pSrc->info.pPayload;
+ nRem = pSrc->info.nPayload;
+
+ x.nData += nOut;
+ if( nOut<pSrc->info.nPayload ){
+ pPgnoOut = &pCell[x.nData];
+ x.nData += 4;
+ }
+
+ if( nRem>nIn ){
+ ovflIn = get4byte(&pSrc->info.pPayload[nIn]);
+ }
+
+ do {
+ nRem -= nOut;
+ do{
+ assert( nOut>0 );
+ if( nIn>0 ){
+ int nCopy = MIN(nOut, nIn);
+ memcpy(aOut, aIn, nCopy);
+ nOut -= nCopy;
+ nIn -= nCopy;
+ aOut += nCopy;
+ aIn += nCopy;
}
- }
- if( rc==SQLITE_OK ){
- rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
- }
- sqlite3_free(pFree);
- }else{
- /* Page sizes match. This means each overflow page (if any) and the
- ** cell itself can be copied verbatim. */
- u8 *pCell = pDest->pBt->pTmpSpace;
- x.pData = pCell;
- x.nData = putVarint32(pCell, pSrc->info.nPayload);
- x.nData += putVarint(&pCell[x.nData], iKey);
- memcpy(&pCell[x.nData], pSrc->info.pPayload, pSrc->info.nLocal);
- x.nData += pSrc->info.nLocal;
- assert( pSrc->info.nLocal<=pSrc->info.nPayload );
- if( pSrc->info.nLocal<pSrc->info.nPayload ){
- Pager *pSrcPager = pSrc->pBt->pPager;
- u8 *pPgno = &pCell[x.nData];
- Pgno ovfl;
- x.nData += 4;
- ovfl = get4byte(pSrc->info.pPayload + pSrc->info.nLocal);
- do{
- MemPage *pNew = 0;
- Pgno pgnoNew = 0;
- if( rc==SQLITE_OK ){
- rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0);
- put4byte(pPgno, pgnoNew);
- }
- if( rc==SQLITE_OK ){
- pPgno = pNew->aData;
- rc = sqlite3PagerWrite(pNew->pDbPage);
- }
+ if( nOut>0 ){
+ sqlite3PagerUnref(pPageIn);
+ pPageIn = 0;
+ rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY);
if( rc==SQLITE_OK ){
- DbPage *pOrig = 0;
- void *pOrigData;
- rc = sqlite3PagerGet(pSrcPager, ovfl, &pOrig, PAGER_GET_READONLY);
- if( rc==SQLITE_OK ){
- pOrigData = sqlite3PagerGetData(pOrig);
- memcpy(pNew->aData, pOrigData, pDest->pBt->usableSize);
- put4byte(pNew->aData, 0);
- ovfl = get4byte(pOrigData);
- }
- sqlite3PagerUnref(pOrig);
+ aIn = (const u8*)sqlite3PagerGetData(pPageIn);
+ ovflIn = get4byte(aIn);
+ aIn += 4;
+ nIn = pSrc->pBt->usableSize - 4;
}
- releasePage(pNew);
- }while( rc==SQLITE_OK && ovfl>0 );
- }
- if( rc==SQLITE_OK ){
- rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
+ }
+ }while( rc==SQLITE_OK && nOut>0 );
+
+ if( rc==SQLITE_OK && nRem>0 ){
+ Pgno pgnoNew;
+ MemPage *pNew = 0;
+ rc = allocateBtreePage(pDest->pBt, &pNew, &pgnoNew, 0, 0);
+ put4byte(pPgnoOut, pgnoNew);
+ releasePage(pPageOut);
+ pPageOut = pNew;
+ if( pPageOut ){
+ pPgnoOut = pPageOut->aData;
+ put4byte(pPgnoOut, 0);
+ aOut = &pPgnoOut[4];
+ nOut = MIN(pDest->pBt->usableSize - 4, nRem);
+ }
}
+ }while( nRem>0 && rc==SQLITE_OK );
+
+ releasePage(pPageOut);
+ sqlite3PagerUnref(pPageIn);
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3BtreeInsert(pDest, &x, OPFLAG_APPEND, seekResult);
}
return rc;
--- /dev/null
+# 2016-08-19
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements a test for VACUUM on attached databases.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix vacuum6
+
+# If the VACUUM statement is disabled in the current build, skip all
+# the tests in this file.
+#
+ifcapable !vacuum {
+ finish_test
+ return
+}
+
+
+do_execsql_test 1.0 {
+ CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+ INSERT INTO t1 VALUES(1, 1);
+} {}
+do_execsql_test 1.1 {
+ VACUUM
+}
+
+#-------------------------------------------------------------------------
+reset_db
+foreach {tn sz} {1 400 2 4000 3 9999} {
+ reset_db
+ do_execsql_test 2.$tn.1 {
+ CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+ )
+ INSERT INTO t1 SELECT i, randomblob($sz) FROM s;
+ }
+
+ do_execsql_test 2.$tn.2 {
+ vacuum;
+ }
+
+ do_execsql_test 2.$tn.3 {
+ PRAGMA integrity_check;
+ } {ok}
+}
+
+reset_db
+do_execsql_test 3.0 {
+ PRAGMA page_size = 1024;
+ CREATE TABLE t1(x INTEGER PRIMARY KEY, y);
+ INSERT INTO t1 VALUES(2, randomblob(1200));
+} {}
+do_execsql_test 3.1 {
+ PRAGMA page_size = 512;
+ VACUUM;
+}
+do_execsql_test 3.2 {
+ PRAGMA integrity_check
+} {ok}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+ CREATE TABLE tx(a, b);
+ CREATE INDEX i1 ON tx(b);
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000
+ )
+ INSERT INTO tx SELECT i, randomblob(i) FROM s;
+}
+foreach {tn pgsz} {
+ 1 2048
+ 2 1024
+ 3 65536
+ 4 8192
+} {
+ do_execsql_test 4.1.$tn.1 "
+ PRAGMA page_size = $pgsz
+ "
+ do_execsql_test 4.1.$tn.2 VACUUM
+ integrity_check 4.1.$tn.3
+}
+
+finish_test
projects / thirdparty / sqlite.git / commitdiff
