From: dan Date: Fri, 23 Feb 2024 15:13:53 +0000 (+0000) Subject: Fix problems with resuming integrity-check operations on indexes with mixed ASC and... X-Git-Tag: version-3.46.0~193^2~6 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1545243368a908367f357dbf4e3c705574d645db;p=thirdparty%2Fsqlite.git Fix problems with resuming integrity-check operations on indexes with mixed ASC and DESC columns, and on indexes that contain NULL values. FossilOrigin-Name: 0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada --- diff --git a/ext/intck/intck1.test b/ext/intck/intck1.test index 4110ece049..4e86e4f2fc 100644 --- a/ext/intck/intck1.test +++ b/ext/intck/intck1.test @@ -181,16 +181,10 @@ do_test 3.2 { sqlite3 db test.db } {} -#puts "[intck_sql db x1a]" -#execsql_pp "EXPLAIN QUERY PLAN [intck_sql db x1a]" do_intck_test 3.3 { {entry (4,'six',5) missing from index x1a} } -#explain_i [intck_sql db x1] -#puts [intck_sql db x1] -#puts [intck_sql db x1a] - #------------------------------------------------------------------------- reset_db do_execsql_test 4.0 { @@ -199,13 +193,17 @@ do_execsql_test 4.0 { INSERT INTO www VALUES(1, 1, 1), (2, 2, 2); } -#puts [intck_sql db w1] -#execsql_pp [intck_sql db www] -#execsql_pp [intck_sql db w1] -#puts [intck_sql db w1] - do_intck_test 4.1 { } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a COLLATE NOCASE); +} + +#puts [intck_sql db i1] + finish_test diff --git a/ext/intck/intck2.test b/ext/intck/intck2.test index 90f263f880..bb57886968 100644 --- a/ext/intck/intck2.test +++ b/ext/intck/intck2.test @@ -61,6 +61,58 @@ do_execsql_test 2.0 { } do_intck_test 2.1 {} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE c=7; +} +puts [intck_sql db x1b] +do_intck_test 2.2 { + {surplus entry ('ONE',6,3) in index x1a} + {surplus entry ('ONE6 "''" ',3) in index x1b} + {surplus entry ('ONE','7',3) in index x1c} +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c); + CREATE INDEX x1all ON x1(a DESC, b ASC, c DESC); + INSERT INTO x1 VALUES(2, 1, 2); + INSERT INTO x1 VALUES(2, 1, 1); + INSERT INTO x1 VALUES(2, 2, 2); + INSERT INTO x1 VALUES(2, 2, 1); + INSERT INTO x1 VALUES(1, 1, 2); + INSERT INTO x1 VALUES(1, 1, 1); + INSERT INTO x1 VALUES(1, 2, 2); + INSERT INTO x1 VALUES(1, 2, 1); +} + +do_intck_test 3.1 { +} + +imposter_edit x1 { + CREATE TABLE imp(a, b, c); +} { + DELETE FROM imp WHERE 1; +} + +db close +sqlite3 db test.db + +do_intck_test 3.2 { + {surplus entry (2,1,2,1) in index x1all} + {surplus entry (2,1,1,2) in index x1all} + {surplus entry (2,2,2,3) in index x1all} + {surplus entry (2,2,1,4) in index x1all} + {surplus entry (1,1,2,5) in index x1all} + {surplus entry (1,1,1,6) in index x1all} + {surplus entry (1,2,2,7) in index x1all} + {surplus entry (1,2,1,8) in index x1all} +} + +#puts [intck_sql db x1all] #puts [intck_sql db x1] finish_test diff --git a/ext/intck/intck_common.tcl b/ext/intck/intck_common.tcl index cea1a247a3..7d6579ae03 100644 --- a/ext/intck/intck_common.tcl +++ b/ext/intck/intck_common.tcl @@ -24,7 +24,12 @@ proc do_intck {db {bSuspend 0}} { if {$msg!=""} { lappend ret $msg } - if {$bSuspend} { $ic unlock } + if {$bSuspend} { + $ic unlock + #puts "SQL: [$ic test_sql {}]" + #execsql_pp "EXPLAIN query plan [$ic test_sql {}]" + #explain_i [$ic test_sql {}] + } } set err [$ic error] diff --git a/ext/intck/sqlite3intck.c b/ext/intck/sqlite3intck.c index e683577bb7..287ec157c3 100644 --- a/ext/intck/sqlite3intck.c +++ b/ext/intck/sqlite3intck.c @@ -15,6 +15,9 @@ #include #include +#include +#include + /* ** nKeyVal: ** The number of values that make up the 'key' for the current pCheck @@ -187,22 +190,87 @@ static char *intckMprintf(sqlite3_intck *p, const char *zFmt, ...){ /* ** This is used by sqlite3_intck_unlock() to save the vector key value ** required to restart the current pCheck query as a nul-terminated string -** in p->zKey. +** in p->zKey. */ static void intckSaveKey(sqlite3_intck *p){ int ii; - const char *zSep = "SELECT '(' || "; char *zSql = 0; sqlite3_stmt *pStmt = 0; + sqlite3_stmt *pXinfo = 0; + const char *zDir = 0; assert( p->pCheck ); assert( p->zKey==0 ); - for(ii=0; iinKeyVal; ii++){ - zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); - zSep = " || ', ' || "; + pXinfo = intckPrepareFmt(p, + "SELECT group_concat(desc, '') FROM %Q.sqlite_schema s, " + "pragma_index_xinfo(%Q, %Q) " + "WHERE s.type='index' AND s.name=%Q", + p->zDb, p->zObj, p->zDb, p->zObj + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXinfo) ){ + zDir = (const char*)sqlite3_column_text(pXinfo, 0); + } + + if( zDir==0 ){ + /* Object is a table, not an index. This is the easy case,as there are + ** no DESC columns or NULL values in a primary key. */ + const char *zSep = "SELECT '(' || "; + for(ii=0; iinKeyVal; ii++){ + zSql = intckMprintf(p, "%z%squote(?)", zSql, zSep); + zSep = " || ', ' || "; + } + zSql = intckMprintf(p, "%z || ')'", zSql); + }else{ + + /* Object is an index. */ + assert( p->nKeyVal>1 ); + for(ii=p->nKeyVal; ii>0; ii--){ + int bLastIsDesc = zDir[ii-1]=='1'; + int bLastIsNull = sqlite3_column_type(p->pCheck, ii)==SQLITE_NULL; + const char *zLast = sqlite3_column_name(p->pCheck, ii); + char *zLhs = 0; + char *zRhs = 0; + char *zWhere = 0; + + if( bLastIsNull ){ + if( bLastIsDesc ) continue; + zWhere = intckMprintf(p, "'%s IS NOT NULL'", zLast); + }else{ + const char *zOp = bLastIsDesc ? "<" : ">"; + zWhere = intckMprintf(p, "'%s %s ' || quote(?%d)", zLast, zOp, ii); + } + + if( ii>1 ){ + const char *zLhsSep = ""; + const char *zRhsSep = ""; + int jj; + for(jj=0; jjpCheck,jj+1); + zLhs = intckMprintf(p, "%z%s%s", zLhs, zLhsSep, zAlias); + zRhs = intckMprintf(p, "%z%squote(?%d)", zRhs, zRhsSep, jj+1); + zLhsSep = ","; + zRhsSep = " || ',' || "; + } + + zWhere = intckMprintf(p, + "'(%z) IS (' || %z || ') AND ' || %z", + zLhs, zRhs, zWhere); + } + zWhere = intckMprintf(p, "'WHERE ' || %z", zWhere); + + zSql = intckMprintf(p, "%z%s(quote( %z ) )", + zSql, + (zSql==0 ? "VALUES" : ",\n "), + zWhere + ); + } + zSql = intckMprintf(p, + "WITH wc(q) AS (\n%z\n)" + "SELECT 'VALUES' || group_concat('(' || q || ')', ',\n ') FROM wc" + , zSql + ); } - zSql = intckMprintf(p, "%z || ')'", zSql); pStmt = intckPrepare(p, zSql); if( p->rc==SQLITE_OK ){ @@ -214,7 +282,9 @@ static void intckSaveKey(sqlite3_intck *p){ } intckFinalize(p, pStmt); } + sqlite3_free(zSql); + intckFinalize(p, pXinfo); } /* @@ -521,7 +591,7 @@ static char *intckCheckObjectSql( " SELECT i.col_name, i.col_alias FROM idx_cols i WHERE i.idx_ispk" " )" " SELECT t.db, t.tab, t.idx, " - " group_concat('o.'||a, ', '), " + " group_concat(a, ', '), " " group_concat('i.'||quote(f), ', '), " " group_concat('quote(o.'||a||')', ' || '','' || '), " " format('(%s)==(%s)'," @@ -586,24 +656,26 @@ static char *intckCheckObjectSql( /* Table idxname contains a single row. The first column, "db", contains ** the name of the db containing the table (e.g. "main") and the second, ** "tab", the name of the table itself. */ - "WITH tabname(db, tab, idx, prev) AS (" - " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), " - " %Q, %Q " + "WITH tabname(db, tab, idx) AS (" + " SELECT %Q, (SELECT tbl_name FROM %Q.sqlite_schema WHERE name=%Q), %Q " ")" + "" + ", whereclause(w_c) AS (%s)" + "" "%s" /* zCommon */ "" ", case_statement(c) AS (" " SELECT " - " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " + " 'CASE WHEN (' || group_concat(col_alias, ', ') || ') IS (\n ' " " || 'SELECT ' || group_concat(col_expr, ', ') || ' FROM '" " || format('%%Q.%%Q NOT INDEXED WHERE %%s\n', t.db, t.tab, p.eq_pk)" - " || ' )\n THEN NULL\n '" + " || ' )\n THEN NULL\n '" " || 'ELSE format(''surplus entry ('" " || group_concat('%%s', ',') || ',' || p.ps_pk" " || ') in index ' || t.idx || ''', ' " " || group_concat('quote('||i.col_alias||')', ', ') || ', ' || p.pk_pk" " || ')'" - " || '\nEND AS error_message'" + " || '\n END AS error_message'" " FROM tabname t, tabpk p, idx_cols i WHERE i.idx_name=t.idx" ")" "" @@ -613,26 +685,25 @@ static char *intckCheckObjectSql( " FROM tabpk p, idx_cols i WHERE i.idx_name=p.idx" ")" "" - ", whereclause(w_c) AS (" - " SELECT CASE WHEN prev!='' THEN " - " '\nWHERE (' || group_concat(i.col_alias, ',') || ',' " - " || o_pk || ') > ' || prev" - " ELSE ''" - " END" - " FROM tabpk, tabname, idx_cols i WHERE i.idx_name=tabpk.idx" - ")" - "" ", main_select(m, n) AS (" " SELECT format(" - " 'WITH %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o%%s'," - " ww.s, c, t.k, whereclause.w_c" + " 'WITH %%s\n' ||" + " ', idx_checker AS (\n' ||" + " ' SELECT %%s,\n' ||" + " ' %%s\n' || " + " ' FROM intck_wrapper AS o\n' ||" + " ')\n'," + " ww.s, c, t.k" " ), t.n" - " FROM case_statement, wrapper_with ww, thiskey t, whereclause" + " FROM case_statement, wrapper_with ww, thiskey t" ")" - "SELECT m, n FROM main_select" + "SELECT m || " + " group_concat('SELECT * FROM idx_checker ' || w_c, ' UNION ALL '), n" + " FROM " + "main_select, whereclause " , p->zDb, p->zDb, zObj, zObj - , zPrev, zCommon + , zPrev ? zPrev : "VALUES('')", zCommon ); }else{ pStmt = intckPrepareFmt(p, @@ -689,7 +760,7 @@ static char *intckCheckObjectSql( ** format('(%d,%d)', _rowid_, n.ii) */ ", thiskey(k, n) AS (" - " SELECT o_pk || ', n.ii', n_pk+1 FROM tabpk" + " SELECT o_pk || ', ii', n_pk+1 FROM tabpk" ")" "" ", whereclause(w_c) AS (" @@ -702,7 +773,7 @@ static char *intckCheckObjectSql( "" ", main_select(m, n) AS (" " SELECT format(" - " '%%s, %%s\nSELECT %%s,\n%%s AS thiskey\nFROM intck_wrapper AS o" + " '%%s, %%s\nSELECT %%s,\n%%s\nFROM intck_wrapper AS o" ", intck_counter AS n%%s\nORDER BY %%s', " " w, ww.s, c, thiskey.k, whereclause.w_c, t.o_pk" " ), thiskey.n" diff --git a/manifest b/manifest index 316837de51..7508c5a2f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sissues\sin\ssqlite3intck.c. -D 2024-02-21T20:58:48.924 +C Fix\sproblems\swith\sresuming\sintegrity-check\soperations\son\sindexes\swith\smixed\sASC\sand\sDESC\scolumns,\sand\son\sindexes\sthat\scontain\sNULL\svalues. +D 2024-02-23T15:13:53.489 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -248,11 +248,11 @@ F ext/fts5/tool/showfts5.tcl d54da0e067306663e2d5d523965ca487698e722c F ext/icu/README.txt 7ab7ced8ae78e3a645b57e78570ff589d4c672b71370f5aa9e1cd7024f400fc9 F ext/icu/icu.c c074519b46baa484bb5396c7e01e051034da8884bad1a1cb7f09bbe6be3f0282 F ext/icu/sqliteicu.h fa373836ed5a1ee7478bdf8a1650689294e41d0c89c1daab26e9ae78a32075a8 -F ext/intck/intck1.test 15e94ee868e939c6d3aef6884c5652d969b28d3eac940a15585511cf8aec71ad -F ext/intck/intck2.test b65d7f627342f767e1d2c447d25619666cec36f516786dd56568bd741e5d7e67 -F ext/intck/intck_common.tcl b8a63ae820dbcdf50ddaba474f130b43c26ba81f4b7720ff1d8b9a6592bdd420 +F ext/intck/intck1.test e73c4f87b54176291b9a01b52dbc9dd88f42f3ec8aea48c8e0bd7f87a1440a40 +F ext/intck/intck2.test 9d083ccb06c9239400569846b077fb9867a19b12233ce9dcbc124540e12d38df +F ext/intck/intck_common.tcl 9e51458126576783f11051ac0fd25bea3f6b17f570a55884223737f3200b214b F ext/intck/intckcorrupt.test 3211ef68ac53e83951b6c8f6a8d2396506d123fe5898f97f848a25837744ec56 -F ext/intck/sqlite3intck.c 3fa96647dde4c719a477f7dbdd1edf76b2f245549b1a4de7f7d60447b24a14df +F ext/intck/sqlite3intck.c b7dd8354b4c3255cecc8cc190f7f980a667d3aec7409900703248296472b358f F ext/intck/sqlite3intck.h 2b40c38e7063ab822c974c0bd4aed97dabb579ccfe2e180a4639bb3bbef0f1c9 F ext/intck/test_intck.c dec07fc82e2626a1450e58be474e351643627b04ee08ce8fae6495a533b87e85 F ext/jni/GNUmakefile 59eb05f2a363bdfac8d15d66bed624bfe1ff289229184f3861b95f98a19cf4b2 @@ -2169,8 +2169,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0e39962baae8a82a3021077676b792ac30c79426bcd8c075b5e92bee55e8c3a5 -R 2d481f6c889817bf845056ed7696633c +P 8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 +R afa6cae1c52ecb818ac1cff0c8856821 U dan -Z 13c318cbe203f9a86dd5c12df0ee017f +Z d57af7b6bfe17ed12101d02b696a832f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 092b5c01d4..391414ddf6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a7bfa74525a495f45b1ea212b1718633b637295090d514dd777f9263477d514 \ No newline at end of file +0f68b35a000ef9f4691c59797c66ed6c3435fc5c503e9d24f891afec6aceeada \ No newline at end of file