From: dan Date: Sat, 3 Sep 2022 21:08:38 +0000 (+0000) Subject: Add an option to assume the freelist is corrupt when recovering data. X-Git-Tag: version-3.40.0~91^2~42 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7302079dbe01c7b136ccc0853816bd7cc4319dde;p=thirdparty%2Fsqlite.git Add an option to assume the freelist is corrupt when recovering data. FossilOrigin-Name: 253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c --- diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 001426b88d..8f70a7473d 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -47,7 +47,6 @@ proc do_recover_test {tn {tsql {}} {res {}}} { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found - $R config testdb rstate.db $R step $R finish @@ -148,6 +147,18 @@ do_recover_test 2.5.1 { 2 2 3 {} 8 9 7 } +do_test 2.6 { + forcedelete test.db2 + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R config freelistcorrupt 1 + $R step + $R finish + sqlite3 db2 test.db2 + execsql { SELECT count(*) FROM lost_and_found_1; } db2 +} {103} +db2 close + #------------------------------------------------------------------------- breakpoint reset_db diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 34cc0cdd50..14cc29e476 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -93,7 +93,7 @@ struct sqlite3_recover { char *zStateDb; char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ - + int bFreelistCorrupt; }; /* @@ -888,27 +888,29 @@ static int recoverLostAndFound(sqlite3_recover *p){ recoverFinalize(p, pStmt); /* Add all pages that appear to be part of the freelist to the bitmap. */ - pStmt = recoverPrepare(p, p->dbOut, - "WITH trunk(pgno) AS (" - " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" - " UNION" - " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" - ")," - "trunkdata(pgno, data) AS (" - " SELECT pgno, getpage(pgno) FROM trunk" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" - " UNION ALL" - " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" - ")" - "SELECT freepgno FROM freelist" - ); - while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iPg = sqlite3_column_int64(pStmt, 0); - recoverBitmapSet(pMap, iPg); + if( p->bFreelistCorrupt==0 ){ + pStmt = recoverPrepare(p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")" + "SELECT freepgno FROM freelist" + ); + while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt, 0); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(p, pStmt); } - recoverFinalize(p, pStmt); /* Add an entry for each page not already added to the bitmap to ** the recovery.map table. This loop leaves the "parent" column @@ -1133,6 +1135,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ } break; + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = (pArg ? 1 : 0); + break; + default: rc = SQLITE_NOTFOUND; break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 8306c67a12..638a8b1c92 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -41,7 +41,6 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); /* ** SQLITE_RECOVER_TESTDB: ** -** ** SQLITE_RECOVER_LOST_AND_FOUND: ** The pArg argument points to a string buffer containing the name ** of a "lost-and-found" table in the output database, or NULL. If @@ -49,9 +48,20 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** valid pages that cannot be associated with any table in the ** recovered part of the schema, data is extracted from these ** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be integer (type "int") value 0 or 1 +** cast as a (void*). If this option is set (argument is 1) and +** a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. */ -#define SQLITE_RECOVER_TESTDB 789 -#define SQLITE_RECOVER_LOST_AND_FOUND 790 +#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_LOST_AND_FOUND 790 +#define SQLITE_RECOVER_FREELIST_CORRUPT 791 /* Step the recovery object. Return SQLITE_DONE if recovery is complete, ** SQLITE_OK if recovery is not complete but no error has occurred, or diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index cdd5d090fa..835a203f66 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -81,6 +81,7 @@ static int testRecoverCmd( const char *aOp[] = { "testdb", /* 0 */ "lostandfound", /* 1 */ + "freelistcorrupt", /* 2 */ 0 }; int iOp = 0; @@ -99,6 +100,14 @@ static int testRecoverCmd( SQLITE_RECOVER_LOST_AND_FOUND, (void*)Tcl_GetString(objv[3]) ); break; + case 2: { + int iVal = 0; + if( Tcl_GetIntFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_FREELIST_CORRUPT, (void*)iVal + ); + break; + } } Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; diff --git a/manifest b/manifest index 00fb244d2e..04d150cbd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sthe\sfreelist\sinto\saccount\swhen\srecovering\sdata\sthat\sis\snot\slinked\sin\sto\sany\stree\sassociated\swith\sa\sschema\sentry. -D 2022-09-03T20:31:36.832 +C Add\san\soption\sto\sassume\sthe\sfreelist\sis\scorrupt\swhen\srecovering\sdata. +D 2022-09-03T21:08:38.958 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,10 +389,10 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverold.test 7578e9b938db15dc469a4af247e15866226f366bde0cbe09a40b0aef4a0506c8 -F ext/recover/sqlite3recover.c 6c9cbc993a970060f9fb881d78f6c7e182ec988a5e48acbf15bb4a5f05ce2902 -F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b -F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2 +F ext/recover/recoverold.test e7e00c78ec35b60488369ddf99e36a3b30e686566571969b05781e5063bdffe8 +F ext/recover/sqlite3recover.c 395c9f623cf84bd8c2e651ec112898d81e1908bbda66fe5f0efcfaa85ad2b262 +F ext/recover/sqlite3recover.h 35aacde3b3834d8ceefb20a2cf0ba221cbb5d802efc11a0529aafc018c462e13 +F ext/recover/test_recover.c 112c580e7cd765a20bbc94998f8b43b629db47fa6bfd696484ca722e418f4172 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 -R 471b646b541e0fcab850e84cb036ac46 +P dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a +R f1692b1a741d9ca9573411dcac80d98d U dan -Z 9f579b130a06a2078244b88aebbd5365 +Z 9a22d599f360a7b8179802a74accdf31 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f01f3f35e4..9852bbbed1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a \ No newline at end of file +253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c \ No newline at end of file