From: dan Date: Sat, 24 Sep 2022 19:54:49 +0000 (+0000) Subject: Add the SQLITE_RECOVER_SLOWINDEXES option, for specifying that indexes should be... X-Git-Tag: version-3.40.0~91^2~15 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=8d8fc58413a78f932567e7d7021aa96b54270959;p=thirdparty%2Fsqlite.git Add the SQLITE_RECOVER_SLOWINDEXES option, for specifying that indexes should be created and populated along with tables, instead of separately at the end of the recovery operation. FossilOrigin-Name: ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 --- diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test new file mode 100644 index 0000000000..838b771eaf --- /dev/null +++ b/ext/recover/recoverslowidx.test @@ -0,0 +1,84 @@ +# 2022 September 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_SLOWINDEXES option. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoverrowid + +ifcapable !vtab { + finish_test; return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); +} + +proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 +} + +do_test 1.1 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.2 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {CREATE TABLE t1(a, b)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {CREATE INDEX i1 ON t1(a)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + +do_test 1.3 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config slowindexes 1 + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.4 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {CREATE TABLE t1(a, b)} + {CREATE INDEX i1 ON t1(a)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + + +finish_test + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index fb046fbfc8..aac1eea306 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -173,6 +173,7 @@ struct sqlite3_recover { char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ /* Error code and error message */ int errCode; /* For sqlite3_recover_errcode() */ @@ -991,16 +992,16 @@ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pTblname = 0; pSelect = recoverPrepare(p, p->dbOut, - "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isUnique) AS (" + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" " SELECT rootpage, name, sql, " " type='table', " " sql LIKE 'create virtual%'," - " (type='index' AND sql LIKE '%unique%')" + " (type='index' AND (sql LIKE '%unique%' OR ?1))" " FROM recovery.schema" ")" "SELECT rootpage, tbl, isVirtual, name, sql" " FROM dbschema " - " WHERE tbl OR isUnique" + " WHERE tbl OR isIndex" " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" ); @@ -1010,6 +1011,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ ); if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); while( sqlite3_step(pSelect)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(pSelect, 0); int bTable = sqlite3_column_int(pSelect, 1); @@ -1064,6 +1066,10 @@ static int recoverWriteSchema2(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" ); @@ -1897,6 +1903,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->bRecoverRowid = *(int*)pArg; break; + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + default: rc = SQLITE_NOTFOUND; break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 128d823795..d0a37504e7 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -145,10 +145,28 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** that are not also INTEGER PRIMARY KEY values. If this option is ** clear, then new rowids are assigned to all recovered rows. The ** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. */ #define SQLITE_RECOVER_LOST_AND_FOUND 1 #define SQLITE_RECOVER_FREELIST_CORRUPT 2 #define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 /* ** Perform a unit of work towards the recovery operation. This function diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index d660a5345e..99c7aeca34 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -89,6 +89,7 @@ static int testRecoverCmd( { "errmsg", 0, "" }, /* 2 */ { "errcode", 0, "" }, /* 3 */ { "finish", 0, "" }, /* 4 */ + { "step", 0, "" }, /* 5 */ { 0 } }; int rc = TCL_OK; @@ -115,7 +116,8 @@ static int testRecoverCmd( "lostandfound", /* 1 */ "freelistcorrupt", /* 2 */ "rowids", /* 3 */ - "invalid", /* 4 */ + "slowindexes", /* 4 */ + "invalid", /* 5 */ 0 }; int iOp = 0; @@ -153,6 +155,14 @@ static int testRecoverCmd( break; } case 4: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_SLOWINDEXES, (void*)&iVal + ); + break; + } + case 5: { res = sqlite3_recover_config(pTest->p, 12345, 0); break; } @@ -187,6 +197,11 @@ static int testRecoverCmd( if( res ) return TCL_ERROR; break; } + case 5: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_step(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } } return TCL_OK; diff --git a/manifest b/manifest index b345ea4aaf..f6cebeee1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqlite3_recover_step()\sto\sheader\sfile\ssqlite3recover.h.\sUpdate\sthe\sAPI\sdocs\sin\sthis\sfile. -D 2022-09-24T19:17:20.785 +C Add\sthe\sSQLITE_RECOVER_SLOWINDEXES\soption,\sfor\sspecifying\sthat\sindexes\sshould\sbe\screated\sand\spopulated\salong\swith\stables,\sinstead\sof\sseparately\sat\sthe\send\sof\sthe\srecovery\soperation. +D 2022-09-24T19:54:49.642 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -396,10 +396,11 @@ F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 +F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 7752d374356a2346b5959161d5512d6f8a0bd7fb259ea81fd2473ad00dd39132 -F ext/recover/sqlite3recover.h 09a0f70b821954fc219d7583d74871e8578e4f1e9267304ec029c2bfc02bd27a -F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 +F ext/recover/sqlite3recover.c 7cd26aeb413b5b1cec1ae6213488dcec3e4e822911d3f255272b32cf0f32c406 +F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 +F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2013,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f -R b60fe138c3457719a54bec9e37633f69 +P 47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 +R 9e568d3e4f6455e4cabb845dec54247e U dan -Z c514b2b66b0aba624f215b19ab2fba51 +Z a8bd0fc6063574ee27e4134a0a1e02e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0de769690b..8f9e8a36bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 \ No newline at end of file +ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 \ No newline at end of file