From: dan Date: Thu, 1 Sep 2022 21:00:39 +0000 (+0000) Subject: Further work on making recovery extension compatible with the shell tool ".recover... X-Git-Tag: version-3.40.0~91^2~45 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=6c86783f9a212781f6e69dbf102c1048e1c066f4;p=thirdparty%2Fsqlite.git Further work on making recovery extension compatible with the shell tool ".recover" code. FossilOrigin-Name: 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 --- diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index c4348baea0..73832c8a12 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -40,9 +40,11 @@ proc compare_dbs {db1 db2} { proc do_recover_test {tn} { forcedelete test.db2 + forcedelete rstate.db uplevel [list do_test $tn.1 { set R [sqlite3_recover_init db main test.db2] + $R config testdb rstate.db $R step $R finish } {}] diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 6c7828c007..d8b151616b 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -48,6 +48,12 @@ struct RecoverTable { RecoverTable *pNext; }; +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[0]; /* Array of 32-bit bitmasks */ +}; + /* ** */ @@ -59,7 +65,7 @@ struct RecoverTable { " pgno, child, PRIMARY KEY(child, pgno)" \ " ) WITHOUT ROWID;" \ " CREATE TABLE recovery.map(" \ -" pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" \ +" pgno INTEGER PRIMARY KEY, parent INT" \ " );" \ " CREATE TABLE recovery.schema(" \ " type, name, tbl_name, rootpage, sql" \ @@ -75,11 +81,14 @@ struct sqlite3_recover { char *zDb; char *zUri; RecoverTable *pTblList; + RecoverBitmap *pUsed; /* Used by recoverWriteLostAndFound() */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ char *zStateDb; + int bLostAndFound; + }; /* @@ -124,6 +133,41 @@ static int recoverError( return errCode; } + +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); + + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; +} + +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} + +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + + static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); } @@ -194,6 +238,24 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +/* +** Execute "PRAGMA page_count" against the input database. If successful, +** return the integer result. Or, if an error occurs, leave an error code +** and error message in the sqlite3_recover handle. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + /* ** The implementation of a user-defined SQL function invoked by the ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages @@ -219,24 +281,23 @@ static void recoverGetPage( assert( nArg==1 ); if( pgno==0 ){ - pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); - }else if( p->pGetPage==0 ){ - pStmt = recoverPreparePrintf( - p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb - ); + sqlite3_result_int64(pCtx, recoverPageCount(p)); + return; }else{ - pStmt = p->pGetPage; - } - - if( pStmt ){ - if( pgno ) sqlite3_bind_int64(pStmt, 1, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); - } - if( pgno ){ - p->pGetPage = recoverReset(p, pStmt); + if( p->pGetPage==0 ){ + pStmt = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); }else{ - recoverFinalize(p, pStmt); + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + } + p->pGetPage = recoverReset(p, pStmt); } } @@ -311,7 +372,7 @@ static int recoverCacheSchema(sqlite3_recover *p){ "WITH RECURSIVE pages(p) AS (" " SELECT 1" " UNION" - " SELECT child FROM recovery.dbptr, pages WHERE pgno=p" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" ")" "INSERT INTO recovery.schema SELECT" " max(CASE WHEN field=0 THEN value ELSE NULL END)," @@ -504,6 +565,77 @@ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ return pRet; } +static int recoverWriteLostAndFound(sqlite3_recover *p){ + i64 nPg = 0; + RecoverBitmap *pMap = 0; + + nPg = recoverPageCount(p); + pMap = p->pUsed = recoverBitmapAlloc(p, nPg); + if( pMap ){ + sqlite3_stmt *pStmt = 0; + char *zField = 0; + const char *zSep = 0; + int ii; + + sqlite3_stmt *pStmt = recoverPrepare( + p, p->dbOut, + "WITH RECURSIVE used(page) AS (" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + ); + while( pStmt && sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(pStmt); + + pStmt = recoverPreparePrintf( + p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "INSERT INTO recover.map(pgno) " + " SELECT ii FROM seq WHERE !page_is_used(ii)" + ); + sqlite3_step(pStmt); + recoverFinalize(pStmt); + + pStmt = recoverPrepare( + p, p->dbOut, + "UPDATE recover.map SET parent = ptr.pgno " + " FROM sqlite_dbptr('getpage()') WHERE recover.map.pgno=ptr.child" + ); + sqlite3_step(pStmt); + recoverFinalize(pStmt); + + pStmt = recoverPrepare( + p, p->dbOut, + "SELECT max(field) FROM sqlite_dbdata('getpage') WHERE pgno IN (" + " SELECT pgno FROM recover.map" + ")" + ); + if( pStmt && sqlite3_step(pStmt) ){ + nMaxField = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(pStmt); + + if( nMaxField==0 || p->errCode!=SQLITE_OK ) return p->errCode; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; iierrCode = SQLITE_NOMEM; + } + } + } +} + static int recoverWriteData(sqlite3_recover *p){ RecoverTable *pTbl; int nMax = 0; @@ -522,7 +654,8 @@ static int recoverWriteData(sqlite3_recover *p){ "WITH RECURSIVE pages(root, page) AS (" " SELECT rootpage, rootpage FROM recovery.schema" " UNION" - " SELECT root, child FROM recovery.dbptr, pages WHERE pgno=page" + " SELECT root, child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" ") " "SELECT root, page, cell, field, value " "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " @@ -677,6 +810,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->zStateDb = sqlite3_mprintf("%s", (char*)pArg); break; + case SQLITE_RECOVER_LOST_AND_FOUND: + p->bLostAndFound = (pArg ? 1 : 0); + break; + default: rc = SQLITE_NOTFOUND; break; @@ -695,6 +832,7 @@ static void recoverStep(sqlite3_recover *p){ if( recoverCacheSchema(p) ) return; if( recoverWriteSchema1(p) ) return; if( recoverWriteData(p) ) return; + if( p->bLostAndFound && recoverWriteLostAndFound(p) ) return; if( recoverWriteSchema2(p) ) return; } } @@ -722,6 +860,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ p->pGetPage = 0; rc = p->errCode; + sqlite3_free(p->zStateDb); sqlite3_free(p); return rc; } diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 401f83ea28..0c83f8dea5 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -38,7 +38,8 @@ sqlite3_recover *sqlite3_recover_init( /* Details TBD. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); -#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_LOST_AND_FOUND 790 /* Step the recovery object. Return SQLITE_DONE if recovery is complete, ** SQLITE_OK if recovery is not complete but no error has occurred, or diff --git a/manifest b/manifest index 0fab53a46f..7f8f333f25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sfiles\sfor\san\sextension\sto\srecover\sdata\sfrom\scorrupted\sdatabases. -D 2022-08-31T20:45:43.730 +C Further\swork\son\smaking\srecovery\sextension\scompatible\swith\sthe\sshell\stool\s".recover"\scode. +D 2022-09-01T21:00:39.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,10 +387,10 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 861ad5140566102a8c5a3d1f936a7d6da569f34c86597c274de695f597031bac +F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/sqlite3recover.c 594fb45777a14f0b88b944b9fb2ccb3e85a29ef5b17522b8dac3e3944c4c27ea -F ext/recover/sqlite3recover.h 3255f6491007e57be310aedb72a848c88f79fc14e7222bda4b8d4dab1a2450c3 +F ext/recover/sqlite3recover.c d81b430f968d838035ebf5ca168b43ae8a0bec1e7c2c950b74ec4fd5e16ca47b +F ext/recover/sqlite3recover.h 94e277a9b314a03df46b5e94cc44b70ed6c6893d2776d09c7ea0b55c969ad854 F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 @@ -2004,11 +2004,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 -R 563b8320bf923831e4768bc403655fc2 -T *branch * recover-extension -T *sym-recover-extension * -T -sym-trunk * +P f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 +R 185cdcc7bc591773e93e55f9de4bfb4c U dan -Z 1c7612740eb933f84d589533d182c6df +Z 5d96e275b1b3f6a2d23b93e2a033e9c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16703e1b2f..5c5b60080f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 \ No newline at end of file +8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 \ No newline at end of file