From: dan Date: Mon, 30 Oct 2017 17:05:18 +0000 (+0000) Subject: In checkindex.c, use C code instead of SQL/group_concat() to compose various X-Git-Tag: version-3.22.0~215^2~13 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=75650d4f9453db917f8198a8d276ed96590ffb8a;p=thirdparty%2Fsqlite.git In checkindex.c, use C code instead of SQL/group_concat() to compose various SQL clauses. This is to make it easier to support indexes on expressions. FossilOrigin-Name: 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 --- diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index b70b57122c..0393358567 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -47,9 +47,15 @@ struct CidxCursor { typedef struct CidxColumn CidxColumn; struct CidxColumn { - char *zName; - char *zColl; - int bDesc; + char *zExpr; /* Text for indexed expression */ + int bDesc; /* True for DESC columns, otherwise false */ + int bKey; /* Part of index, not PK */ +}; + +typedef struct CidxIndex CidxIndex; +struct CidxIndex { + int nCol; /* Elements in aCol[] array */ + CidxColumn aCol[1]; /* Array of indexed columns */ }; static void *cidxMalloc(int *pRc, int n){ @@ -258,29 +264,30 @@ char *cidxStrdup(int *pRc, const char *zStr){ return zRet; } +static void cidxFreeIndex(CidxIndex *pIdx){ + if( pIdx ){ + int i; + for(i=0; inCol; i++){ + sqlite3_free(pIdx->aCol[i].zExpr); + } + sqlite3_free(pIdx); + } +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ - int *pnCol, /* OUT: Number of columns in index */ - CidxColumn **paCol, /* OUT: Columns */ - char **pzTab, /* OUT: Table name */ - char **pzCurrentKey, /* OUT: Expression for current_key */ - char **pzOrderBy, /* OUT: ORDER BY expression list */ - char **pzSubWhere, /* OUT: sub-query WHERE clause */ - char **pzSubExpr /* OUT: sub-query WHERE clause */ + CidxIndex **ppIdx, /* OUT: Description of columns */ + char **pzTab /* OUT: Table name */ ){ int rc = SQLITE_OK; char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - CidxColumn *aCol = 0; + CidxIndex *pIdx = 0; sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pGroup = 0; + sqlite3_stmt *pInfo = 0; - /* Find the table */ + /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", zIdx @@ -293,94 +300,35 @@ static int cidxLookupIndex( rc = SQLITE_ERROR; } - pGroup = cidxPrepare(&rc, pCsr, - "SELECT group_concat(" - " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" - ") AS zCurrentKey," - " group_concat(" - " coalesce(name, 'rowid') || ' COLLATE ' || coll " - " || CASE WHEN desc THEN ' DESC' ELSE '' END," - " ', '" - ") AS zOrderBy," - " group_concat(" - " CASE WHEN key==1 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubWhere," - " group_concat(" - " CASE WHEN key==0 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubExpr," - " count(*) AS nCol" - " FROM pragma_index_xinfo(%Q);" - , zIdx, zIdx, zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){ - zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0)); - zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); - zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); - zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); - *pnCol = sqlite3_column_int(pGroup, 4); - } - cidxFinalize(&rc, pGroup); - - pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); if( rc==SQLITE_OK ){ - int nByte = 0; - int nCol = 0; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; if( zName==0 ) zName = "rowid"; - nCol++; - nByte += strlen(zName)+1 + strlen(zColl)+1; - } - rc = sqlite3_reset(pGroup); - aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte); - - if( rc==SQLITE_OK ){ - int iCol = 0; - char *z = (char*)&aCol[nCol]; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - int nName, nColl; - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); - if( zName==0 ) zName = "rowid"; - - nName = strlen(zName); - nColl = strlen(zColl); - memcpy(z, zName, nName); - aCol[iCol].zName = z; - z += nName+1; - - memcpy(z, zColl, nColl); - aCol[iCol].zColl = z; - z += nColl+1; - - aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3); - iCol++; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); } + p = &pIdx->aCol[iCol++]; + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + pIdx->nCol = iCol; } - cidxFinalize(&rc, pGroup); + cidxFinalize(&rc, pInfo); } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); }else{ *pzTab = zTab; - *pzCurrentKey = zCurrentKey; - *pzOrderBy = zOrderBy; - *pzSubWhere = zSubWhere; - *pzSubExpr = zSubExpr; - *paCol = aCol; + *ppIdx = pIdx; } return rc; @@ -463,35 +411,91 @@ static char *cidxWhere( int i; for(i=0; i"), + zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, + zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName); + zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); } return zRet; } -static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){ - int i; +#define CIDX_CLIST_ALL 0 +#define CIDX_CLIST_ORDERBY 1 +#define CIDX_CLIST_CURRENT_KEY 2 +#define CIDX_CLIST_SUBWHERE 3 +#define CIDX_CLIST_SUBEXPR 4 + +/* +** This function returns various strings based on the contents of the +** CidxIndex structure and the eType parameter. +*/ +static char *cidxColumnList( + int *pRc, /* IN/OUT: Error code */ + const char *zIdx, + CidxIndex *pIdx, /* Indexed columns */ + int eType /* True to include ASC/DESC */ +){ char *zRet = 0; - const char *zSep = ""; - for(i=0; inCol; i++){ + CidxColumn *p = &pIdx->aCol[i]; + assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); + switch( eType ){ + + case CIDX_CLIST_ORDERBY: + zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zSep = ","; + break; + + case CIDX_CLIST_CURRENT_KEY: + zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zSep = "||','||"; + break; + + case CIDX_CLIST_SUBWHERE: + if( p->bKey==0 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + case CIDX_CLIST_SUBEXPR: + if( p->bKey==1 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + default: + assert( eType==CIDX_CLIST_ALL ); + zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); + zSep = ","; + break; + } + } } + return zRet; } @@ -516,21 +520,26 @@ static int cidxFilter( } if( zIdxName ){ - int nCol = 0; char *zTab = 0; char *zCurrentKey = 0; char *zOrderBy = 0; char *zSubWhere = 0; char *zSubExpr = 0; + char *zSrcList = 0; + char **azAfter = 0; - CidxColumn *aCol = 0; + CidxIndex *pIdx = 0; - rc = cidxLookupIndex(pCsr, zIdxName, - &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr - ); + rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); + + zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); + zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); + zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); + zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); + /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter); + rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); } if( rc || zAfterKey==0 ){ @@ -538,9 +547,9 @@ static int cidxFilter( "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, aCol, nCol); + char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; @@ -548,17 +557,17 @@ static int cidxFilter( zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); - for(i=nCol-1; i>=0; i--){ + for(i=pIdx->nCol-1; i>=0; i--){ int j; - if( aCol[i].bDesc && azAfter[i]==0 ) continue; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j); + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); zSql = cidxMprintf(&rc, "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", zSql, zSep, zList, zTab, zWhere, zOrderBy ); zSep = " UNION ALL "; - if( aCol[i].bDesc==0 ) break; + if( pIdx->aCol[i].bDesc==0 ) break; } } zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); @@ -573,7 +582,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); sqlite3_free(azAfter); } @@ -584,7 +593,9 @@ static int cidxFilter( return rc; } -/* Return a column for the sqlite_btreeinfo table */ +/* +** Return a column value. +*/ static int cidxColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, diff --git a/manifest b/manifest index 1aefe38098..48fe0a91a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences. -D 2017-10-30T08:04:38.448 +C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. +D 2017-10-30T17:05:18.290 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0 +F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2 +F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 -R 35b9a81e4b8b6b857354a831ff9c29c1 +P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 +R 45b731228881e6db68d5d1d151d2789f U dan -Z cd0262c8d2e550da2f2d47bdfb3d2467 +Z 8f0b667700c59bebe92a45558d3d25e5 diff --git a/manifest.uuid b/manifest.uuid index 7ea946e219..5d8e25d480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 \ No newline at end of file +940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 706171ff19..6648187559 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -67,10 +67,11 @@ proc do_index_check_test {tn idx res} { " $res] uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] + uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] } + do_execsql_test 1.2 { SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); } {