From: drh Date: Thu, 5 May 2016 17:15:23 +0000 (+0000) Subject: Initial check-in of the "scrub.exe" utility program prototype. Not yet X-Git-Tag: version-3.14.0~165^2~5 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=da110bfecec0b590ee6321c4f57bf98c950905da;p=thirdparty%2Fsqlite.git Initial check-in of the "scrub.exe" utility program prototype. Not yet fully functional. In particular, no scrubbing is done. FossilOrigin-Name: bdf2ec77d1542d4e9b68218f558710a3efc15823 --- diff --git a/Makefile.in b/Makefile.in index 78480de9e0..4e10d4e231 100644 --- a/Makefile.in +++ b/Makefile.in @@ -589,6 +589,10 @@ sqlite3$(TEXE): $(TOP)/src/shell.c sqlite3.c sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) +scrub$(TEXE): $(TOP)/ext/misc/scrub.c sqlite3.o + $(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \ + $(TOP)/ext/misc/scrub.c sqlite3.o $(TLIBS) + srcck1$(BEXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c diff --git a/Makefile.msc b/Makefile.msc index bb9790d312..408a516364 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1455,6 +1455,9 @@ $(SQLITE3EXE): $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_S sqldiff.exe: $(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) +scrub.exe: $(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H) + $(LTLINK) $(NO_WARN) $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) + srcck1.exe: $(TOP)\tool\srcck1.c $(BCC) $(NO_WARN) -Fe$@ $(TOP)\tool\srcck1.c diff --git a/ext/misc/scrub.c b/ext/misc/scrub.c new file mode 100644 index 0000000000..fbf2f8c2b7 --- /dev/null +++ b/ext/misc/scrub.c @@ -0,0 +1,306 @@ +/* +** 2016-05-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a utility function (and a utility program) that +** makes a copy of an SQLite database while simultaneously zeroing out all +** deleted content. +** +** Normally (when PRAGMA secure_delete=OFF, which is the default) when SQLite +** deletes content, it does not overwrite the deleted content but rather marks +** the region of the file that held that content as being reusable. This can +** cause deleted content to recoverable from the database file. This stale +** content is removed by the VACUUM command, but VACUUM can be expensive for +** large databases. When in PRAGMA secure_delete=ON mode, the deleted content +** is zeroed, but secure_delete=ON has overhead as well. +** +** This utility attempts to make a copy of a complete SQLite database where +** all of the deleted content is zeroed out in the copy, and it attempts to +** do so while being faster than running VACUUM. +** +** Usage: +** +** int sqlite3_scrub_backup( +** const char *zSourceFile, // Source database filename +** const char *zDestFile, // Destination database filename +** char **pzErrMsg // Write error message here +** ); +** +** Simply call the API above specifying the filename of the source database +** and the name of the backup copy. The source database must already exist +** and can be in active use. (A read lock is held during the backup.) The +** destination file should not previously exist. If the pzErrMsg parameter +** is non-NULL and if an error occurs, then an error message might be written +** into memory obtained from sqlite3_malloc() and *pzErrMsg made to point to +** that error message. But if the error is an OOM, the error might not be +** reported. The routine always returns non-zero if there is an error. +** +** If compiled with -DSCRUB_STANDALONE then a main() procedure is added and +** this file becomes a standalone program that can be run as follows: +** +** ./sqlite3scrub SOURCE DEST +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include + +typedef struct ScrubState ScrubState; +typedef unsigned char u8; + +/* State information for a scrub-and-backup operation */ +struct ScrubState { + const char *zSrcFile; /* Name of the source file */ + const char *zDestFile; /* Name of the destination file */ + int rcErr; /* Error code */ + char *zErr; /* Error message text */ + sqlite3 *dbSrc; /* Source database connection */ + sqlite3_file *pSrc; /* Source file handle */ + sqlite3 *dbDest; /* Destination database connection */ + sqlite3_file *pDest; /* Destination file handle */ + unsigned int szPage; /* Page size */ + unsigned int nPage; /* Number of pages */ + u8 *page1; /* Content of page 1 */ +}; + +/* Store an error message */ +static void scrubBackupErr(ScrubState *p, const char *zFormat, ...){ + va_list ap; + sqlite3_free(p->zErr); + va_start(ap, zFormat); + p->zErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( p->rcErr==0 ) p->rcErr = SQLITE_ERROR; +} + +/* Allocate memory to hold a single page of content */ +static u8 *scrubBackupAllocPage(ScrubState *p){ + u8 *pPage; + if( p->rcErr ) return 0; + pPage = sqlite3_malloc( p->szPage ); + if( pPage==0 ) p->rcErr = SQLITE_NOMEM; + return pPage; +} + +/* Read a page from the source database into memory. Use the memory +** provided by pBuf if not NULL or allocate a new page if pBuf==NULL. +*/ +static u8 *scrubBackupRead(ScrubState *p, int pgno, u8 *pBuf){ + int rc; + sqlite3_int64 iOff; + u8 *pOut = pBuf; + if( p->rcErr ) return 0; + if( pOut==0 ){ + pOut = scrubBackupAllocPage(p); + if( pOut==0 ) return 0; + } + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pSrc->pMethods->xRead(p->pSrc, pOut, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + if( pBuf==0 ) sqlite3_free(pOut); + pOut = 0; + scrubBackupErr(p, "read failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } + return pOut; +} + +/* Write a page to the destination database */ +static void scrubBackupWrite(ScrubState *p, int pgno, u8 *pData){ + int rc; + sqlite3_int64 iOff; + if( p->rcErr ) return; + iOff = (pgno-1)*(sqlite3_int64)p->szPage; + rc = p->pDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff); + if( rc!=SQLITE_OK ){ + scrubBackupErr(p, "write failed for page %d", pgno); + p->rcErr = SQLITE_IOERR; + } +} + +/* Prepare a statement against the "db" database. */ +static sqlite3_stmt *scrubBackupPrepare( + ScrubState *p, /* Backup context */ + sqlite3 *db, /* Database to prepare against */ + const char *zSql /* SQL statement */ +){ + sqlite3_stmt *pStmt; + if( p->rcErr ) return 0; + p->rcErr = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( p->rcErr ){ + scrubBackupErr(p, "SQL error \"%s\" on \"%s\"", + sqlite3_errmsg(db), zSql); + sqlite3_finalize(pStmt); + return 0; + } + return pStmt; +} + + +/* Open the source database file */ +static void scrubBackupOpenSrc(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + /* Open the source database file */ + p->rcErr = sqlite3_open_v2(p->zSrcFile, &p->dbSrc, + SQLITE_OPEN_READONLY | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + p->rcErr = sqlite3_exec(p->dbSrc, "BEGIN", 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a read transaction on the source database: %s", + sqlite3_errmsg(p->dbSrc)); + return; + } + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_size"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->szPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the page size"); + } + sqlite3_finalize(pStmt); + if( p->rcErr ) return; + pStmt = scrubBackupPrepare(p, p->dbSrc, "PRAGMA page_count"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nPage = sqlite3_column_int(pStmt, 0); + }else{ + scrubBackupErr(p, "unable to determine the size of the source database"); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbSrc, "main", SQLITE_FCNTL_FILE_POINTER, &p->pSrc); + if( p->pSrc==0 || p->pSrc->pMethods==0 ){ + scrubBackupErr(p, "cannot get the source file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +/* Create and open the destination file */ +static void scrubBackupOpenDest(ScrubState *p){ + sqlite3_stmt *pStmt; + int rc; + char *zSql; + if( p->rcErr ) return; + p->rcErr = sqlite3_open_v2(p->zDestFile, &p->dbDest, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_URI | SQLITE_OPEN_PRIVATECACHE, 0); + if( p->rcErr ){ + scrubBackupErr(p, "cannot open destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + zSql = sqlite3_mprintf("PRAGMA page_size(%u);", p->szPage); + if( zSql==0 ){ + p->rcErr = SQLITE_NOMEM; + return; + } + p->rcErr = sqlite3_exec(p->dbDest, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot set the page size on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + sqlite3_exec(p->dbDest, "PRAGMA journal_mode=OFF;", 0, 0, 0); + p->rcErr = sqlite3_exec(p->dbDest, "BEGIN EXCLUSIVE;", 0, 0, 0); + if( p->rcErr ){ + scrubBackupErr(p, + "cannot start a write transaction on the destination database: %s", + sqlite3_errmsg(p->dbDest)); + return; + } + pStmt = scrubBackupPrepare(p, p->dbDest, "PRAGMA page_count;"); + if( pStmt==0 ) return; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + scrubBackupErr(p, "cannot measure the size of the destination"); + }else if( sqlite3_column_int(pStmt, 0)>1 ){ + scrubBackupErr(p, "destination database is not empty - holds %d pages", + sqlite3_column_int(pStmt, 0)); + } + sqlite3_finalize(pStmt); + sqlite3_file_control(p->dbDest, "main", SQLITE_FCNTL_FILE_POINTER, &p->pDest); + if( p->pDest==0 || p->pDest->pMethods==0 ){ + scrubBackupErr(p, "cannot get the destination file handle"); + p->rcErr = SQLITE_ERROR; + } +} + +int sqlite3_scrub_backup( + const char *zSrcFile, /* Source file */ + const char *zDestFile, /* Destination file */ + char **pzErr /* Write error here if non-NULL */ +){ + ScrubState s; + unsigned int i; + u8 *pBuf = 0; + u8 *pData; + + memset(&s, 0, sizeof(s)); + s.zSrcFile = zSrcFile; + s.zDestFile = zDestFile; + + scrubBackupOpenSrc(&s); + scrubBackupOpenDest(&s); + pBuf = scrubBackupAllocPage(&s); + + for(i=1; s.rcErr==0 && i<=s.nPage; i++){ + pData = scrubBackupRead(&s, i, pBuf); + scrubBackupWrite(&s, i, pData); + } + + /* Close the destination database without closing the transaction. If we + ** commit, page zero will be overwritten. */ + sqlite3_close(s.dbDest); + + sqlite3_close(s.dbSrc); + sqlite3_free(s.page1); + if( pzErr ){ + *pzErr = s.zErr; + }else{ + sqlite3_free(s.zErr); + } + return s.rcErr; +} + +#ifdef SCRUB_STANDALONE +/* The main() routine when this utility is run as a stand-alone program */ +int main(int argc, char **argv){ + char *zErr = 0; + int rc; + if( argc!=3 ){ + fprintf(stderr,"Usage: %s SOURCE DESTINATION\n", argv[0]); + exit(1); + } + rc = sqlite3_scrub_backup(argv[1], argv[2], &zErr); + if( rc==SQLITE_NOMEM ){ + fprintf(stderr, "%s: out of memory\n", argv[0]); + exit(1); + } + if( zErr ){ + fprintf(stderr, "%s: %s\n", argv[0], zErr); + sqlite3_free(zErr); + exit(1); + } + return 0; +} +#endif diff --git a/main.mk b/main.mk index 1e791416d8..f200dc2092 100644 --- a/main.mk +++ b/main.mk @@ -488,6 +488,9 @@ sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) +scrub$(EXE): $(TOP)/ext/misc/scrub.c sqlite3.o + $(TCC) -I. -DSCRUB_STANDALONE -o scrub$(EXE) $(TOP)/ext/misc/scrub.c sqlite3.o $(THREADLIB) + srcck1$(EXE): $(TOP)/tool/srcck1.c $(BCC) -o srcck1$(EXE) $(TOP)/tool/srcck1.c diff --git a/manifest b/manifest index 8e5f6ffbdc..919f2e5e0d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Renumber\sinternal\sconstants\sin\sthe\sprintf()\simplemention\sfor\sa\ssmall\nperformance\simprovement. -D 2016-05-05T11:53:12.439 -F Makefile.in 9eda6e1c90d05c199c3ec8a7069b0682ad307657 +C Initial\scheck-in\sof\sthe\s"scrub.exe"\sutility\sprogram\sprototype.\s\sNot\syet\nfully\sfunctional.\s\sIn\sparticular,\sno\sscrubbing\sis\sdone. +D 2016-05-05T17:15:23.301 +F Makefile.in f59e0763ff448719fc1bd25513882b0567286317 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc db82b35aef27f412fef14d8534afc022138bcdfd +F Makefile.msc 306d73e854b1a92ea06e5d1e637faa5c44de53c7 F README.md 8ecc12493ff9f820cdea6520a9016001cb2e59b7 F VERSION 5d234da9b5dae329fab75ff75884cfe0a9cb3fda F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -215,6 +215,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c a68d25c659bd2d893cd1215667bbf75ecb9dc7d4 F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a +F ext/misc/scrub.c 76a9c795078a65bd021d5b5cd9e4655e7d0306c0 F ext/misc/series.c e11e534ada797d5b816d7e7a93c022306563ca35 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c bf1b922c2750698e9a3d4c50cce6974adb7e93be @@ -302,7 +303,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a283660f75c3c4b75d8c9d12a40fa38a066eee9d +F main.mk 0fb2b051c9dbcf7f26a8455cf115ae9457448b93 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1487,7 +1488,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 71af9ac165ac02272f4886f69bd9ab4770fd7bb6 -R 89655379113728f544ce9a18ad294b0b +P 69d11447f4b1a8c536c3b6573d2a3419da870412 +R 98e582093972832acc05b11fc975c102 +T *branch * scrub-backup +T *sym-scrub-backup * +T -sym-trunk * U drh -Z f65addcdc03169dedb02067b28293d6c +Z e2196386eaeffd3910bcf89f26dcb9bb diff --git a/manifest.uuid b/manifest.uuid index 4a4e62d8ed..6832a682a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69d11447f4b1a8c536c3b6573d2a3419da870412 \ No newline at end of file +bdf2ec77d1542d4e9b68218f558710a3efc15823 \ No newline at end of file