From: dan Date: Mon, 30 Oct 2017 19:38:41 +0000 (+0000) Subject: Add support for indexes on expressions to incremental_index_check. X-Git-Tag: version-3.22.0~215^2~12 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e7a579d27d12ce936d5a536dffd7453bd0503235;p=thirdparty%2Fsqlite.git Add support for indexes on expressions to incremental_index_check. FossilOrigin-Name: 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 --- diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 0393358567..fc91b74244 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -274,6 +274,110 @@ static void cidxFreeIndex(CidxIndex *pIdx){ } } +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +static int cidxFindNext(const char *zIn, const char **pzOut){ + const char *z = zIn; + + while( 1 ){ + *pzOut = z; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + default: + z++; + } + } + + assert( 0 ); + return -1; +} + +static int cidx_isspace(char c){ + return c==' ' || c=='\t' || c=='\r' || c=='\n'; +} + +static int cidx_isident(char c){ + return c<0 + || (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ + const char *z = zSql; + const char *z1; + int e; + int rc = SQLITE_OK; + int nParen = 1; + CidxColumn *pCol = pIdx->aCol; + + e = cidxFindNext(z, &z); + if( e!=CIDX_PARSE_OPEN ) goto parse_error; + z1 = z+1; + z++; + while( nParen>0 ){ + e = cidxFindNext(z, &z); + if( e==CIDX_PARSE_EOF ) goto parse_error; + if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ + const char *z2 = z; + if( pCol->zExpr ) goto parse_error; + + while( cidx_isspace(z[-1]) ) z--; + if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + + while( cidx_isspace(z1[0]) ) z1++; + pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); + pCol++; + z = z1 = z2+1; + } + if( e==CIDX_PARSE_OPEN ) nParen++; + if( e==CIDX_PARSE_CLOSE ) nParen--; + z++; + } + + return rc; + + parse_error: + cidxCursorError(pCsr, "Parse error in: %s", zSql); + return SQLITE_ERROR; +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ @@ -289,39 +393,50 @@ static int cidxLookupIndex( /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", + "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'", zIdx ); if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; + if( zName==0 ) zName = "rowid"; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); + nAlloc += 8; + } + p = &pIdx->aCol[iCol++]; + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + if( zSql==0 || p->bKey==0 ){ + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + }else{ + p->zExpr = 0; + } + pIdx->nCol = iCol; + } + cidxFinalize(&rc, pInfo); + } + + if( rc==SQLITE_OK && zSql ){ + rc = cidxParseSQL(pCsr, pIdx, zSql); + } } + cidxFinalize(&rc, pFindTab); if( rc==SQLITE_OK && zTab==0 ){ rc = SQLITE_ERROR; } - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - } - p = &pIdx->aCol[iCol++]; - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - pIdx->nCol = iCol; - } - cidxFinalize(&rc, pInfo); - } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); @@ -411,22 +526,22 @@ static char *cidxWhere( int i; for(i=0; i"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); + zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); } return zRet; @@ -450,7 +565,7 @@ static char *cidxColumnList( ){ char *zRet = 0; if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {" ASC", " DESC"}; + const char *aDir[2] = {"", " DESC"}; int i; const char *zSep = ""; @@ -460,19 +575,19 @@ static char *cidxColumnList( switch( eType ){ case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); zSep = ","; break; case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); zSep = "||','||"; break; case CIDX_CLIST_SUBWHERE: if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -480,8 +595,8 @@ static char *cidxColumnList( case CIDX_CLIST_SUBEXPR: if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -489,8 +604,8 @@ static char *cidxColumnList( default: assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); - zSep = ","; + zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); + zSep = ", "; break; } } @@ -536,7 +651,7 @@ static int cidxFilter( zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ + zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); if( rc==SQLITE_OK && zAfterKey ){ rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); @@ -544,17 +659,19 @@ static int cidxFilter( if( rc || zAfterKey==0 ){ pCsr->pStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", - zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; - zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); for(i=pIdx->nCol-1; i>=0; i--){ @@ -562,16 +679,15 @@ static int cidxFilter( if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, - "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", - zSql, zSep, zList, zTab, zWhere, zOrderBy - ); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zSrcList, zTab, zWhere, zOrderBy + ); zSep = " UNION ALL "; if( pIdx->aCol[i].bDesc==0 ) break; } } - zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); - sqlite3_free(zList); + zSql = cidxMprintf(&rc, "%z) AS i", zSql); /* printf("SQL: %s\n", zSql); */ pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); @@ -582,6 +698,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(zSrcList); cidxFreeIndex(pIdx); sqlite3_free(azAfter); } diff --git a/manifest b/manifest index 48fe0a91a2..59ee792f38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. -D 2017-10-30T17:05:18.290 +C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check. +D 2017-10-30T19:38:41.046 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 +F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 +F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 -R 45b731228881e6db68d5d1d151d2789f +P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 +R 32cef05dfb77ed0c73cb51913be9d3e3 U dan -Z 8f0b667700c59bebe92a45558d3d25e5 +Z 4d91b1f15edf1ab6cd873cde5a983874 diff --git a/manifest.uuid b/manifest.uuid index 5d8e25d480..f363d67abb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file +8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 6648187559..a91455558f 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -259,6 +259,64 @@ do_index_check_test 4.3 t4cc { {} 'AAB','EEE',4 } +#-------------------------------------------------------------------------- +# Test an index on an expression. +# +do_execsql_test 5.0 { + CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); + INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); + INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); + INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); + INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); + INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); + + CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); + CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); +} + +do_index_check_test 5.1.1 t5x { + {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 +} + +do_index_check_test 5.1.2 t5y { + {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 +} + +do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { + {} {'{"w":4, "z":4}',4} + {} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} + +do_test 5.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + execsql { + UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; + DELETE FROM xt5 WHERE rowid = 4; + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_index_check_test 5.3.1 t5x { + {row missing} NULL,4 + {row data mismatch} 1,1 + {} 2,2 + {} 3,3 + {} 5,5 +} + +do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { + {row missing} {'{"w":4, "z":4}',4} + {row data mismatch} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} finish_test