In checkindex.c, use C code instead of SQL/group_concat() to compose various

SQL clauses. This is to make it easier to support indexes on expressions.

FossilOrigin-Name: 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index b70b57122ce48b0935ddc272ee27f9f7eda62a1f..0393358567bb3a22c8cc9a40bd58cf81bba5511c 100644 (file)
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -47,9 +47,15 @@ struct CidxCursor {
 
 typedef struct CidxColumn CidxColumn;
 struct CidxColumn {
-  char *zName;
-  char *zColl;
-  int bDesc;
+  char *zExpr;                    /* Text for indexed expression */
+  int bDesc;                      /* True for DESC columns, otherwise false */
+  int bKey;                       /* Part of index, not PK */
+};
+
+typedef struct CidxIndex CidxIndex;
+struct CidxIndex {
+  int nCol;                       /* Elements in aCol[] array */
+  CidxColumn aCol[1];             /* Array of indexed columns */
 };
 
 static void *cidxMalloc(int *pRc, int n){
@@ -258,29 +264,30 @@ char *cidxStrdup(int *pRc, const char *zStr){
   return zRet;
 }
 
+static void cidxFreeIndex(CidxIndex *pIdx){
+  if( pIdx ){
+    int i;
+    for(i=0; i<pIdx->nCol; i++){
+      sqlite3_free(pIdx->aCol[i].zExpr);
+    }
+    sqlite3_free(pIdx);
+  }
+}
+
 static int cidxLookupIndex(
   CidxCursor *pCsr,               /* Cursor object */
   const char *zIdx,               /* Name of index to look up */
-  int *pnCol,                     /* OUT: Number of columns in index */
-  CidxColumn **paCol,             /* OUT: Columns */
-  char **pzTab,                   /* OUT: Table name */
-  char **pzCurrentKey,            /* OUT: Expression for current_key */
-  char **pzOrderBy,               /* OUT: ORDER BY expression list */
-  char **pzSubWhere,              /* OUT: sub-query WHERE clause */
-  char **pzSubExpr                /* OUT: sub-query WHERE clause */
+  CidxIndex **ppIdx,              /* OUT: Description of columns */
+  char **pzTab                    /* OUT: Table name */
 ){
   int rc = SQLITE_OK;
   char *zTab = 0;
-  char *zCurrentKey = 0;
-  char *zOrderBy = 0;
-  char *zSubWhere = 0;
-  char *zSubExpr = 0;
-  CidxColumn *aCol = 0;
+  CidxIndex *pIdx = 0;
 
   sqlite3_stmt *pFindTab = 0;
-  sqlite3_stmt *pGroup = 0;
+  sqlite3_stmt *pInfo = 0;
     
-  /* Find the table */
+  /* Find the table for this index. */
   pFindTab = cidxPrepare(&rc, pCsr, 
       "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'",
       zIdx
@@ -293,94 +300,35 @@ static int cidxLookupIndex(
     rc = SQLITE_ERROR;
   }
 
-  pGroup = cidxPrepare(&rc, pCsr,
-      "SELECT group_concat("
-      "  coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'"
-      ") AS zCurrentKey,"
-      "       group_concat("
-      "  coalesce(name, 'rowid') || ' COLLATE ' || coll "
-      "  || CASE WHEN desc THEN ' DESC' ELSE '' END,"
-      "  ', '"
-      ") AS zOrderBy,"
-      "       group_concat("
-      "         CASE WHEN key==1 THEN NULL ELSE "
-      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
-      "         END,"
-      "  ' AND '"
-      ") AS zSubWhere,"
-      "       group_concat("
-      "         CASE WHEN key==0 THEN NULL ELSE "
-      "  coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') "
-      "         END,"
-      "  ' AND '"
-      ") AS zSubExpr,"
-      "      count(*) AS nCol"
-      " FROM pragma_index_xinfo(%Q);"
-      , zIdx, zIdx, zIdx
-  );
-  if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){
-    zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0));
-    zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1));
-    zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2));
-    zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3));
-    *pnCol = sqlite3_column_int(pGroup, 4);
-  }
-  cidxFinalize(&rc, pGroup);
-
-  pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+  pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
   if( rc==SQLITE_OK ){
-    int nByte = 0;
-    int nCol = 0;
-    while( sqlite3_step(pGroup)==SQLITE_ROW ){
-      const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
-      const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
+    int nAlloc = 0;
+    int iCol = 0;
+
+    while( sqlite3_step(pInfo)==SQLITE_ROW ){
+      const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
+      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+      CidxColumn *p;
       if( zName==0 ) zName = "rowid";
-      nCol++;
-      nByte += strlen(zName)+1 + strlen(zColl)+1;
-    }
-    rc = sqlite3_reset(pGroup);
-    aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte);
-
-    if( rc==SQLITE_OK ){
-      int iCol = 0;
-      char *z = (char*)&aCol[nCol];
-      while( sqlite3_step(pGroup)==SQLITE_ROW ){
-        int nName, nColl;
-        const char *zName = (const char*)sqlite3_column_text(pGroup, 2);
-        const char *zColl = (const char*)sqlite3_column_text(pGroup, 4);
-        if( zName==0 ) zName = "rowid";
-
-        nName = strlen(zName);
-        nColl = strlen(zColl);
-        memcpy(z, zName, nName);
-        aCol[iCol].zName = z;
-        z += nName+1;
-
-        memcpy(z, zColl, nColl);
-        aCol[iCol].zColl = z;
-        z += nColl+1;
-
-        aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3);
-        iCol++;
+      if( iCol==nAlloc ){
+        int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
+        pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
       }
+      p = &pIdx->aCol[iCol++];
+      p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
+      p->bDesc = sqlite3_column_int(pInfo, 3);
+      p->bKey = sqlite3_column_int(pInfo, 5);
+      pIdx->nCol = iCol;
     }
-    cidxFinalize(&rc, pGroup);
+    cidxFinalize(&rc, pInfo);
   }
   
   if( rc!=SQLITE_OK ){
     sqlite3_free(zTab);
-    sqlite3_free(zCurrentKey);
-    sqlite3_free(zOrderBy);
-    sqlite3_free(zSubWhere);
-    sqlite3_free(zSubExpr);
-    sqlite3_free(aCol);
+    cidxFreeIndex(pIdx);
   }else{
     *pzTab = zTab;
-    *pzCurrentKey = zCurrentKey;
-    *pzOrderBy = zOrderBy;
-    *pzSubWhere = zSubWhere;
-    *pzSubExpr = zSubExpr;
-    *paCol = aCol;
+    *ppIdx = pIdx;
   }
 
   return rc;
@@ -463,35 +411,91 @@ static char *cidxWhere(
   int i;
 
   for(i=0; i<iGt; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s IS %s", zRet, 
-        zSep, aCol[i].zName, aCol[i].zColl, (azAfter[i] ? azAfter[i] : "NULL")
+    zRet = cidxMprintf(pRc, "%z%s%s IS %s", zRet, 
+        zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL")
     );
     zSep = " AND ";
   }
 
   if( bLastIsNull ){
-    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zName);
+    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zExpr);
   }
   else if( azAfter[iGt] ){
-    zRet = cidxMprintf(pRc, "%z%s%s COLLATE %s %s %s", zRet, 
-        zSep, aCol[iGt].zName, aCol[iGt].zColl, (aCol[iGt].bDesc ? "<" : ">"), 
+    zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, 
+        zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), 
         azAfter[iGt]
     );
   }else{
-    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName);
+    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr);
   }
 
   return zRet;
 }
 
-static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){
-  int i;
+#define CIDX_CLIST_ALL         0
+#define CIDX_CLIST_ORDERBY     1
+#define CIDX_CLIST_CURRENT_KEY 2
+#define CIDX_CLIST_SUBWHERE    3
+#define CIDX_CLIST_SUBEXPR     4
+
+/*
+** This function returns various strings based on the contents of the
+** CidxIndex structure and the eType parameter.
+*/
+static char *cidxColumnList(
+  int *pRc,                       /* IN/OUT: Error code */
+  const char *zIdx,
+  CidxIndex *pIdx,                /* Indexed columns */
+  int eType                       /* True to include ASC/DESC */
+){
   char *zRet = 0;
-  const char *zSep = "";
-  for(i=0; i<nCol; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, aCol[i].zName);
-    zSep = ",";
+  if( *pRc==SQLITE_OK ){
+    const char *aDir[2] = {" ASC", " DESC"};
+    int i;
+    const char *zSep = "";
+
+    for(i=0; i<pIdx->nCol; i++){
+      CidxColumn *p = &pIdx->aCol[i];
+      assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 );
+      switch( eType ){
+
+        case CIDX_CLIST_ORDERBY:
+          zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]);
+          zSep = ",";
+          break;
+
+        case CIDX_CLIST_CURRENT_KEY:
+          zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr);
+          zSep = "||','||";
+          break;
+
+        case CIDX_CLIST_SUBWHERE:
+          if( p->bKey==0 ){
+            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
+                zSep, p->zExpr, zIdx, p->zExpr
+            );
+            zSep = " AND ";
+          }
+          break;
+
+        case CIDX_CLIST_SUBEXPR:
+          if( p->bKey==1 ){
+            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
+                zSep, p->zExpr, zIdx, p->zExpr
+            );
+            zSep = " AND ";
+          }
+          break;
+
+        default:
+          assert( eType==CIDX_CLIST_ALL );
+          zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr);
+          zSep = ",";
+          break;
+      }
+    }
   }
+
   return zRet;
 }
 
@@ -516,21 +520,26 @@ static int cidxFilter(
   }
 
   if( zIdxName ){
-    int nCol = 0;
     char *zTab = 0;
     char *zCurrentKey = 0;
     char *zOrderBy = 0;
     char *zSubWhere = 0;
     char *zSubExpr = 0;
+    char *zSrcList = 0;
+
     char **azAfter = 0;
-    CidxColumn *aCol = 0;
+    CidxIndex *pIdx = 0;
 
-    rc = cidxLookupIndex(pCsr, zIdxName, 
-        &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr
-    );
+    rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab);
+
+    zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY);
+    zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
+    zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
+    zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
+    /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */
 
     if( rc==SQLITE_OK && zAfterKey ){
-      rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter);
+      rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
     }
 
     if( rc || zAfterKey==0 ){
@@ -538,9 +547,9 @@ static int cidxFilter(
           "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
           zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
       );
-      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
+      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt));  */
     }else{
-      char *zList = cidxColumnList(&rc, aCol, nCol);
+      char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0);
       const char *zSep = "";
       char *zSql;
       int i;
@@ -548,17 +557,17 @@ static int cidxFilter(
       zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
           zSubExpr, zTab, zSubWhere, zCurrentKey
       );
-      for(i=nCol-1; i>=0; i--){
+      for(i=pIdx->nCol-1; i>=0; i--){
         int j;
-        if( aCol[i].bDesc && azAfter[i]==0 ) continue;
+        if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
         for(j=0; j<2; j++){
-          char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j);
+          char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
           zSql = cidxMprintf(&rc, 
               "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
               zSql, zSep, zList, zTab, zWhere, zOrderBy
               );
           zSep = " UNION ALL ";
-          if( aCol[i].bDesc==0 ) break;
+          if( pIdx->aCol[i].bDesc==0 ) break;
         }
       }
       zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName);
@@ -573,7 +582,7 @@ static int cidxFilter(
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
-    sqlite3_free(aCol);
+    cidxFreeIndex(pIdx);
     sqlite3_free(azAfter);
   }
 
@@ -584,7 +593,9 @@ static int cidxFilter(
   return rc;
 }
 
-/* Return a column for the sqlite_btreeinfo table */
+/* 
+** Return a column value.
+*/
 static int cidxColumn(
   sqlite3_vtab_cursor *pCursor, 
   sqlite3_context *ctx, 

diff --git a/manifest b/manifest
index 1aefe380980a0281293c11daf68c8455ce842c00..48fe0a91a21967137a7857577d30d26b0f989fbc 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences.
-D 2017-10-30T08:04:38.448
+C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions.
+D 2017-10-30T17:05:18.290
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0
+F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2
+F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0
-R 35b9a81e4b8b6b857354a831ff9c29c1
+P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
+R 45b731228881e6db68d5d1d151d2789f
 U dan
-Z cd0262c8d2e550da2f2d47bdfb3d2467
+Z 8f0b667700c59bebe92a45558d3d25e5

diff --git a/manifest.uuid b/manifest.uuid
index 7ea946e2194999886dbe799fd70221f40ebb8fb4..5d8e25d480162e36a33a920881c1771deba100d3 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
\ No newline at end of file
+940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
\ No newline at end of file

diff --git a/test/checkindex.test b/test/checkindex.test
index 706171ff195c3b6a3173bf2fa1bcad38a7e4ad24..66481875593279f62b698a9558e24fd3510d4017 100644 (file)
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -67,10 +67,11 @@ proc do_index_check_test {tn idx res} {
   " $res]
 
   uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]]
-  #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
-  #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
+  uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
+  uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
 }
 
+
 do_execsql_test 1.2 {
   SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1');
 } {