From fa5d2efbfc28a9e402e4ff720e81eefd83802c67 Mon Sep 17 00:00:00 2001 From: drh <> Date: Sat, 30 May 2026 10:15:58 +0000 Subject: [PATCH] Remove the untested and undocumented "scrub" utility program. It was written in 2016, but made obsolete by the VACUUM INTO command in 2019. FossilOrigin-Name: 20c458172e0b09f7ada7f8cb8be95450c3988e620cbf6f174470bf8e78dcfbf0 --- Makefile.msc | 3 - ext/misc/scrub.c | 610 ----------------------------------------------- main.mk | 5 - manifest | 17 +- manifest.uuid | 2 +- 5 files changed, 9 insertions(+), 628 deletions(-) delete mode 100644 ext/misc/scrub.c diff --git a/Makefile.msc b/Makefile.msc index 8e06c11cb6..0737e6aee6 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1896,9 +1896,6 @@ RSYNC_OPT = \ sqlite3_rsync.exe: $(RSYNC_SRC) $(LIBRESOBJS) $(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS) -scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) - $(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) - srcck1.exe: $(TOP)\tool\srcck1.c $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c diff --git a/ext/misc/scrub.c b/ext/misc/scrub.c deleted file mode 100644 index 2406d39f25..0000000000 --- a/ext/misc/scrub.c +++ /dev/null @@ -1,610 +0,0 @@ -/* -** 2016-05-05 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file implements a utility function (and a utility program) that -** makes a copy of an SQLite database while simultaneously zeroing out all -** deleted content. -** -** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite -** deletes content, it does not overwrite the deleted content but rather marks -** the region of the file that held that content as being reusable. This can -** cause deleted content to recoverable from the database file. This stale -** content is removed by the VACUUM command, but VACUUM can be expensive for -** large databases. When in PRAGMA secure_delete=ON mode, the deleted content -** is zeroed, but secure_delete=ON has overhead as well. -** -** This utility attempts to make a copy of a complete SQLite database where -** all of the deleted content is zeroed out in the copy, and it attempts to -** do so while being faster than running VACUUM. -** -** Usage: -** -** int sqlite3_scrub_backup( -** const char *zSourceFile, // Source database filename -** const char *zDestFile, // Destination database filename -** char **pzErrMsg // Write error message here -** ); -** -** Simply call the API above specifying the filename of the source database -** and the name of the backup copy. The source database must already exist -** and can be in active use. (A read lock is held during the backup.) The -** destination file should not previously exist. If the pzErrMsg parameter -** is non-NULL and if an error occurs, then an error message might be written -** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to -** that error message. But if the error is an OOM, the error might not be -** reported. The routine always returns non-zero if there is an error. -** -** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and -** this file becomes a standalone program that can be run as follows: -** -** ./sqlite3scrub SOURCE DEST -*/ -#include "sqlite3.h" -#include -#include -#include -#include -#include - -typedef struct ScrubState ScrubState; -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; - - -/* State information for a scrub-and-backup operation */ -struct ScrubState { - const char *zSrcFile; /* Name of the source file */ - const char *zDestFile; /* Name of the destination file */ - int rcErr; /* Error code */ - char *zErr; /* Error message text */ - sqlite3 *dbSrc; /* Source database connection */ - sqlite3_file *pSrc; /* Source file handle */ - sqlite3 *dbDest; /* Destination database connection */ - sqlite3_file *pDest; /* Destination file handle */ - u32 szPage; /* Page size */ - u32 szUsable; /* Usable bytes on each page */ - u32 nPage; /* Number of pages */ - u32 iLastPage; /* Page number of last page written so far*/ - u8 *page1; /* Content of page 1 */ -}; - -/* Store an error message */ -static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ - va_list ap; - sqlite3_free(p->zErr); - va_start(ap, zFormat); - p->zErr = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR; -} - -/* Allocate memory to hold a single page of content */ -static u8 *scrubBackupAllocPage(ScrubState *p){ - u8 *pPage; - if( p->rcErr ) return 0; - pPage = sqlite3_malloc64( p->szPage ); - if( pPage==0 ) p->rcErr = SQLITE_NOMEM; - return pPage; -} - -/* Read a page from the source database into memory. Use the memory -** provided by pBuf if not NULL or allocate a new page if pBuf==NULL. -*/ -static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){ - int rc; - sqlite3_int64 iOff; - u8 *pOut = pBuf; - if( p->rcErr ) return 0; - if( pOut==0 ){ - pOut = scrubBackupAllocPage(p); - if( pOut==0 ) return 0; - } - iOff = (pgno-1)*(sqlite3_int64)p->szPage; - rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff); - if( rc!=SQLITE_OK ){ - if( pBuf==0 ) sqlite3_free(pOut); - pOut = 0; - scrubBackupErr(p, "read failed for page %d", pgno); - p->rcErr = SQLITE_IOERR; - } - return pOut; -} - -/* Write a page to the destination database */ -static void scrubBackupWrite(ScrubState *p, int pgno, const u8 *pData){ - int rc; - sqlite3_int64 iOff; - if( p->rcErr ) return; - iOff = (pgno-1)*(sqlite3_int64)p->szPage; - rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff); - if( rc!=SQLITE_OK ){ - scrubBackupErr(p, "write failed for page %d", pgno); - p->rcErr = SQLITE_IOERR; - } - if( (u32)pgno>p->iLastPage ) p->iLastPage = pgno; -} - -/* Prepare a statement against the "db" database. */ -static sqlite3_stmt *scrubBackupPrepare( - ScrubState *p, /* Backup context */ - sqlite3 *db, /* Database to prepare against */ - const char *zSql /* SQL statement */ -){ - sqlite3_stmt *pStmt; - if( p->rcErr ) return 0; - p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( p->rcErr ){ - scrubBackupErr(p, "SQL error \"%s\" on \"%s\"", - sqlite3_errmsg(db), zSql); - sqlite3_finalize(pStmt); - return 0; - } - return pStmt; -} - - -/* Open the source database file */ -static void scrubBackupOpenSrc(ScrubState *p){ - sqlite3_stmt *pStmt; - int rc; - /* Open the source database file */ - p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc, - SQLITE_OPEN_READWRITE | - SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); - if( p->rcErr ){ - scrubBackupErr(p, "cannot open source database: %s", - sqlite3_errmsg(p->dbSrc)); - return; - } - p->rcErr = sqlite3_exec(p->dbSrc, "SELECT 1 FROM sqlite_schema; BEGIN;", - 0, 0, 0); - if( p->rcErr ){ - scrubBackupErr(p, - "cannot start a read transaction on the source database: %s", - sqlite3_errmsg(p->dbSrc)); - return; - } - rc = sqlite3_wal_checkpoint_v2(p->dbSrc, "main", SQLITE_CHECKPOINT_FULL, - 0, 0); - if( rc ){ - scrubBackupErr(p, "cannot checkpoint the source database"); - return; - } - pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size"); - if( pStmt==0 ) return; - rc = sqlite3_step(pStmt); - if( rc==SQLITE_ROW ){ - p->szPage = sqlite3_column_int(pStmt, 0); - }else{ - scrubBackupErr(p, "unable to determine the page size"); - } - sqlite3_finalize(pStmt); - if( p->rcErr ) return; - pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count"); - if( pStmt==0 ) return; - rc = sqlite3_step(pStmt); - if( rc==SQLITE_ROW ){ - p->nPage = sqlite3_column_int(pStmt, 0); - }else{ - scrubBackupErr(p, "unable to determine the size of the source database"); - } - sqlite3_finalize(pStmt); - sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc); - if( p->pSrc==0 || p->pSrc->pMethods==0 ){ - scrubBackupErr(p, "cannot get the source file handle"); - p->rcErr = SQLITE_ERROR; - } -} - -/* Create and open the destination file */ -static void scrubBackupOpenDest(ScrubState *p){ - sqlite3_stmt *pStmt; - int rc; - char *zSql; - if( p->rcErr ) return; - p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); - if( p->rcErr ){ - scrubBackupErr(p, "cannot open destination database: %s", - sqlite3_errmsg(p->dbDest)); - return; - } - zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage); - if( zSql==0 ){ - p->rcErr = SQLITE_NOMEM; - return; - } - p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0); - sqlite3_free(zSql); - if( p->rcErr ){ - scrubBackupErr(p, - "cannot set the page size on the destination database: %s", - sqlite3_errmsg(p->dbDest)); - return; - } - sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0); - p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0); - if( p->rcErr ){ - scrubBackupErr(p, - "cannot start a write transaction on the destination database: %s", - sqlite3_errmsg(p->dbDest)); - return; - } - pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;"); - if( pStmt==0 ) return; - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ - scrubBackupErr(p, "cannot measure the size of the destination"); - }else if( sqlite3_column_int(pStmt, 0)>1 ){ - scrubBackupErr(p, "destination database is not empty - holds %d pages", - sqlite3_column_int(pStmt, 0)); - } - sqlite3_finalize(pStmt); - sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest); - if( p->pDest==0 || p->pDest->pMethods==0 ){ - scrubBackupErr(p, "cannot get the destination file handle"); - p->rcErr = SQLITE_ERROR; - } -} - -/* Read a 32-bit big-endian integer */ -static u32 scrubBackupInt32(const u8 *a){ - u32 v = a[3]; - v += ((u32)a[2])<<8; - v += ((u32)a[1])<<16; - v += ((u32)a[0])<<24; - return v; -} - -/* Read a 16-bit big-endian integer */ -static u32 scrubBackupInt16(const u8 *a){ - return (a[0]<<8) + a[1]; -} - -/* -** Read a varint. Put the value in *pVal and return the number of bytes. -*/ -static int scrubBackupVarint(const u8 *z, sqlite3_int64 *pVal){ - sqlite3_int64 v = 0; - int i; - for(i=0; i<8; i++){ - v = (v<<7) + (z[i]&0x7f); - if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } - } - v = (v<<8) + (z[i]&0xff); - *pVal = v; - return 9; -} - -/* -** Return the number of bytes in a varint. -*/ -static int scrubBackupVarintSize(const u8 *z){ - int i; - for(i=0; i<8; i++){ - if( (z[i]&0x80)==0 ){ return i+1; } - } - return 9; -} - -/* -** Copy the freelist trunk page given, and all its descendents, -** zeroing out as much as possible in the process. -*/ -static void scrubBackupFreelist(ScrubState *p, int pgno, u32 nFree){ - u8 *a, *aBuf; - u32 n, mx; - - if( p->rcErr ) return; - aBuf = scrubBackupAllocPage(p); - if( aBuf==0 ) return; - - while( pgno && nFree){ - a = scrubBackupRead(p, pgno, aBuf); - if( a==0 ) break; - n = scrubBackupInt32(&a[4]); - mx = p->szUsable/4 - 2; - if( nszPage); - } - scrubBackupWrite(p, iLeaf, aZero); - nFree--; - } -#endif - } - sqlite3_free(aBuf); -} - -/* -** Copy an overflow chain from source to destination. Zero out any -** unused tail at the end of the overflow chain. -*/ -static void scrubBackupOverflow(ScrubState *p, int pgno, u32 nByte){ - u8 *a, *aBuf; - - aBuf = scrubBackupAllocPage(p); - if( aBuf==0 ) return; - while( nByte>0 && pgno!=0 ){ - a = scrubBackupRead(p, pgno, aBuf); - if( a==0 ) break; - if( nByte >= (p->szUsable)-4 ){ - nByte -= (p->szUsable) - 4; - }else{ - u32 x = (p->szUsable - 4) - nByte; - u32 i = p->szUsable - x; - memset(&a[i], 0, x); - nByte = 0; - } - scrubBackupWrite(p, pgno, a); - pgno = scrubBackupInt32(a); - } - sqlite3_free(aBuf); -} - - -/* -** Copy B-Tree page pgno, and all of its children, from source to destination. -** Zero out deleted content during the copy. -*/ -static void scrubBackupBtree(ScrubState *p, int pgno, int iDepth){ - u8 *a; - u32 i, n, pc; - u32 nCell; - u32 nPrefix; - u32 szHdr; - u32 iChild; - u8 *aTop; - u8 *aCell; - u32 x, y; - int ln = 0; - - - if( p->rcErr ) return; - if( iDepth>50 ){ - scrubBackupErr(p, "corrupt: b-tree too deep at page %d", pgno); - return; - } - if( pgno==1 ){ - a = p->page1; - }else{ - a = scrubBackupRead(p, pgno, 0); - if( a==0 ) return; - } - nPrefix = pgno==1 ? 100 : 0; - aTop = &a[nPrefix]; - szHdr = 8 + 4*(aTop[0]==0x02 || aTop[0]==0x05); - aCell = aTop + szHdr; - nCell = scrubBackupInt16(&aTop[3]); - - /* Zero out the gap between the cell index and the start of the - ** cell content area */ - x = scrubBackupInt16(&aTop[5]); /* First byte of cell content area */ - if( x>p->szUsable ){ ln=__LINE__; goto btree_corrupt; } - y = szHdr + nPrefix + nCell*2; - if( y>x ){ ln=__LINE__; goto btree_corrupt; } - if( y0 && pc(p->szUsable)-4 ){ ln=__LINE__; goto btree_corrupt; } - n = scrubBackupInt16(&a[pc+2]); - if( pc+n>(p->szUsable) ){ ln=__LINE__; goto btree_corrupt; } - if( n>4 ) memset(&a[pc+4], 0, n-4); - x = scrubBackupInt16(&a[pc]); - if( x0 ){ ln=__LINE__; goto btree_corrupt; } - pc = x; - } - - /* Write this one page */ - scrubBackupWrite(p, pgno, a); - - /* Walk the tree and process child pages */ - for(i=0; i p->szUsable-3 ){ ln=__LINE__; goto btree_corrupt; } - if( aTop[0]==0x05 || aTop[0]==0x02 ){ - if( pc+4 > p->szUsable ){ ln=__LINE__; goto btree_corrupt; } - iChild = scrubBackupInt32(&a[pc]); - pc += 4; - scrubBackupBtree(p, iChild, iDepth+1); - if( aTop[0]==0x05 ) continue; - } - pc += scrubBackupVarint(&a[pc], &P); - if( pc >= p->szUsable ){ ln=__LINE__; goto btree_corrupt; } - if( aTop[0]==0x0d ){ - X = p->szUsable - 35; - }else{ - X = ((p->szUsable - 12)*64/255) - 23; - } - if( P<=X ){ - /* All content is local. No overflow */ - continue; - } - M = ((p->szUsable - 12)*32/255)-23; - K = M + ((P-M)%(p->szUsable-4)); - if( aTop[0]==0x0d ){ - pc += scrubBackupVarintSize(&a[pc]); - if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; } - } - nLocal = K<=X ? K : M; - if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; } - iChild = scrubBackupInt32(&a[pc+nLocal]); - scrubBackupOverflow(p, iChild, (u32)(P-nLocal)); - } - - /* Walk the right-most tree */ - if( aTop[0]==0x05 || aTop[0]==0x02 ){ - iChild = scrubBackupInt32(&aTop[8]); - scrubBackupBtree(p, iChild, iDepth+1); - } - - /* All done */ - if( pgno>1 ) sqlite3_free(a); - return; - -btree_corrupt: - scrubBackupErr(p, "corruption on page %d of source database (errid=%d)", - pgno, ln); - if( pgno>1 ) sqlite3_free(a); -} - -/* -** Copy all ptrmap pages from source to destination. -** This routine is only called if the source database is in autovacuum -** or incremental vacuum mode. -*/ -static void scrubBackupPtrmap(ScrubState *p){ - u32 pgno = 2; - u32 J = p->szUsable/5; - u32 iLock = (1073742335/p->szPage)+1; - u8 *a, *pBuf; - if( p->rcErr ) return; - pBuf = scrubBackupAllocPage(p); - if( pBuf==0 ) return; - while( pgno<=p->nPage ){ - a = scrubBackupRead(p, pgno, pBuf); - if( a==0 ) break; - scrubBackupWrite(p, pgno, a); - pgno += J+1; - if( pgno==iLock ) pgno++; - } - sqlite3_free(pBuf); -} - -int sqlite3_scrub_backup( - const char *zSrcFile, /* Source file */ - const char *zDestFile, /* Destination file */ - char **pzErr /* Write error here if non-NULL */ -){ - ScrubState s; - u32 n, i; - sqlite3_stmt *pStmt; - - memset(&s, 0, sizeof(s)); - s.zSrcFile = zSrcFile; - s.zDestFile = zDestFile; - - /* Open both source and destination databases */ - scrubBackupOpenSrc(&s); - scrubBackupOpenDest(&s); - - /* Read in page 1 */ - s.page1 = scrubBackupRead(&s, 1, 0); - if( s.page1==0 ) goto scrub_abort; - s.szUsable = s.szPage - s.page1[20]; - - /* Copy the freelist */ - n = scrubBackupInt32(&s.page1[36]); - i = scrubBackupInt32(&s.page1[32]); - if( n ) scrubBackupFreelist(&s, i, n); - - /* Copy ptrmap pages */ - n = scrubBackupInt32(&s.page1[52]); - if( n ) scrubBackupPtrmap(&s); - - /* Copy all of the btrees */ - scrubBackupBtree(&s, 1, 0); - pStmt = scrubBackupPrepare(&s, s.dbSrc, - "SELECT rootpage FROM sqlite_schema WHERE coalesce(rootpage,0)>0"); - if( pStmt==0 ) goto scrub_abort; - while( sqlite3_step(pStmt)==SQLITE_ROW ){ - i = (u32)sqlite3_column_int(pStmt, 0); - scrubBackupBtree(&s, i, 0); - } - sqlite3_finalize(pStmt); - - /* If the last page of the input db file is a free-list leaf, then the - ** backup file on disk is still smaller than the size indicated within - ** the database header. In this case, write a page of zeroes to the - ** last page of the backup database so that SQLite does not mistakenly - ** think the db is corrupt. */ - if( s.iLastPage