Add support for indexes on expressions to incremental_index_check.

FossilOrigin-Name: 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52

diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c
index 0393358567bb3a22c8cc9a40bd58cf81bba5511c..fc91b74244f9430448cfea3a9311f1be69e2aaf9 100644 (file)
--- a/ext/repair/checkindex.c
+++ b/ext/repair/checkindex.c
@@ -274,6 +274,110 @@ static void cidxFreeIndex(CidxIndex *pIdx){
   }
 }
 
+#define CIDX_PARSE_EOF   0
+#define CIDX_PARSE_COMMA 1      /*  "," */
+#define CIDX_PARSE_OPEN  2      /*  "(" */
+#define CIDX_PARSE_CLOSE 3      /*  ")" */
+
+static int cidxFindNext(const char *zIn, const char **pzOut){
+  const char *z = zIn;
+
+  while( 1 ){
+    *pzOut = z;
+    switch( *z ){
+      case '\0':
+        return CIDX_PARSE_EOF;
+      case '(':
+        return CIDX_PARSE_OPEN;
+      case ')':
+        return CIDX_PARSE_CLOSE;
+      case ',':
+        return CIDX_PARSE_COMMA;
+
+      case '"': 
+      case '\'': 
+      case '`': {
+        char q = *z;
+        z++;
+        while( *z ){
+          if( *z==q ){
+            z++;
+            if( *z!=q ) break;
+          }
+          z++;
+        }
+        break;
+      }
+
+      case '[':
+        while( *z++!=']' );
+        break;
+
+      default:
+        z++;
+    }
+  }
+
+  assert( 0 );
+  return -1;
+}
+
+static int cidx_isspace(char c){
+  return c==' ' || c=='\t' || c=='\r' || c=='\n';
+}
+
+static int cidx_isident(char c){
+  return c<0 
+    || (c>='0' && c<='9') || (c>='a' && c<='z') 
+    || (c>='A' && c<='Z') || c=='_';
+}
+
+static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){
+  const char *z = zSql;
+  const char *z1;
+  int e;
+  int rc = SQLITE_OK;
+  int nParen = 1;
+  CidxColumn *pCol = pIdx->aCol;
+
+  e = cidxFindNext(z, &z);
+  if( e!=CIDX_PARSE_OPEN ) goto parse_error;
+  z1 = z+1;
+  z++;
+  while( nParen>0 ){
+    e = cidxFindNext(z, &z);
+    if( e==CIDX_PARSE_EOF ) goto parse_error;
+    if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){
+      const char *z2 = z;
+      if( pCol->zExpr ) goto parse_error;
+
+      while( cidx_isspace(z[-1]) ) z--;
+      if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){
+        z -= 3;
+        while( cidx_isspace(z[-1]) ) z--;
+      }else
+      if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){
+        z -= 4;
+        while( cidx_isspace(z[-1]) ) z--;
+      }
+
+      while( cidx_isspace(z1[0]) ) z1++;
+      pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1);
+      pCol++;
+      z = z1 = z2+1;
+    }
+    if( e==CIDX_PARSE_OPEN ) nParen++;
+    if( e==CIDX_PARSE_CLOSE ) nParen--;
+    z++;
+  }
+
+  return rc;
+
+ parse_error:
+  cidxCursorError(pCsr, "Parse error in: %s", zSql);
+  return SQLITE_ERROR;
+}
+
 static int cidxLookupIndex(
   CidxCursor *pCsr,               /* Cursor object */
   const char *zIdx,               /* Name of index to look up */
@@ -289,39 +393,50 @@ static int cidxLookupIndex(
     
   /* Find the table for this index. */
   pFindTab = cidxPrepare(&rc, pCsr, 
-      "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'",
+      "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'",
       zIdx
   );
   if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){
+    const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1);
     zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0));
+
+    pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+    if( rc==SQLITE_OK ){
+      int nAlloc = 0;
+      int iCol = 0;
+
+      while( sqlite3_step(pInfo)==SQLITE_ROW ){
+        const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
+        const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+        CidxColumn *p;
+        if( zName==0 ) zName = "rowid";
+        if( iCol==nAlloc ){
+          int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
+          pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
+          nAlloc += 8;
+        }
+        p = &pIdx->aCol[iCol++];
+        p->bDesc = sqlite3_column_int(pInfo, 3);
+        p->bKey = sqlite3_column_int(pInfo, 5);
+        if( zSql==0 || p->bKey==0 ){
+          p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
+        }else{
+          p->zExpr = 0;
+        }
+        pIdx->nCol = iCol;
+      }
+      cidxFinalize(&rc, pInfo);
+    }
+
+    if( rc==SQLITE_OK && zSql ){
+      rc = cidxParseSQL(pCsr, pIdx, zSql);
+    }
   }
+
   cidxFinalize(&rc, pFindTab);
   if( rc==SQLITE_OK && zTab==0 ){
     rc = SQLITE_ERROR;
   }
-
-  pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
-  if( rc==SQLITE_OK ){
-    int nAlloc = 0;
-    int iCol = 0;
-
-    while( sqlite3_step(pInfo)==SQLITE_ROW ){
-      const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
-      const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
-      CidxColumn *p;
-      if( zName==0 ) zName = "rowid";
-      if( iCol==nAlloc ){
-        int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
-        pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
-      }
-      p = &pIdx->aCol[iCol++];
-      p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
-      p->bDesc = sqlite3_column_int(pInfo, 3);
-      p->bKey = sqlite3_column_int(pInfo, 5);
-      pIdx->nCol = iCol;
-    }
-    cidxFinalize(&rc, pInfo);
-  }
   
   if( rc!=SQLITE_OK ){
     sqlite3_free(zTab);
@@ -411,22 +526,22 @@ static char *cidxWhere(
   int i;
 
   for(i=0; i<iGt; i++){
-    zRet = cidxMprintf(pRc, "%z%s%s IS %s", zRet, 
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS %s", zRet, 
         zSep, aCol[i].zExpr, (azAfter[i] ? azAfter[i] : "NULL")
     );
     zSep = " AND ";
   }
 
   if( bLastIsNull ){
-    zRet = cidxMprintf(pRc, "%z%s%s IS NULL", zRet, zSep, aCol[iGt].zExpr);
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS NULL", zRet, zSep, aCol[iGt].zExpr);
   }
   else if( azAfter[iGt] ){
-    zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, 
+    zRet = cidxMprintf(pRc, "%z%s(%s) %s %s", zRet, 
         zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), 
         azAfter[iGt]
     );
   }else{
-    zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr);
+    zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr);
   }
 
   return zRet;
@@ -450,7 +565,7 @@ static char *cidxColumnList(
 ){
   char *zRet = 0;
   if( *pRc==SQLITE_OK ){
-    const char *aDir[2] = {" ASC", " DESC"};
+    const char *aDir[2] = {"", " DESC"};
     int i;
     const char *zSep = "";
 
@@ -460,19 +575,19 @@ static char *cidxColumnList(
       switch( eType ){
 
         case CIDX_CLIST_ORDERBY:
-          zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]);
+          zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]);
           zSep = ",";
           break;
 
         case CIDX_CLIST_CURRENT_KEY:
-          zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr);
+          zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i);
           zSep = "||','||";
           break;
 
         case CIDX_CLIST_SUBWHERE:
           if( p->bKey==0 ){
-            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
-                zSep, p->zExpr, zIdx, p->zExpr
+            zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, 
+                zSep, p->zExpr, i
             );
             zSep = " AND ";
           }
@@ -480,8 +595,8 @@ static char *cidxColumnList(
 
         case CIDX_CLIST_SUBEXPR:
           if( p->bKey==1 ){
-            zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, 
-                zSep, p->zExpr, zIdx, p->zExpr
+            zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, 
+                zSep, p->zExpr, i
             );
             zSep = " AND ";
           }
@@ -489,8 +604,8 @@ static char *cidxColumnList(
 
         default:
           assert( eType==CIDX_CLIST_ALL );
-          zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr);
-          zSep = ",";
+          zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i);
+          zSep = ", ";
           break;
       }
     }
@@ -536,7 +651,7 @@ static int cidxFilter(
     zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
     zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
     zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
-    /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */
+    zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL);
 
     if( rc==SQLITE_OK && zAfterKey ){
       rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
@@ -544,17 +659,19 @@ static int cidxFilter(
 
     if( rc || zAfterKey==0 ){
       pCsr->pStmt = cidxPrepare(&rc, pCsr, 
-          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s",
-          zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy
+          "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
+          "FROM (SELECT %s FROM %Q ORDER BY %s) AS i",
+          zSubExpr, zTab, zSubWhere, zCurrentKey, 
+          zSrcList, zTab, zOrderBy
       );
-      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt));  */
+      /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */
     }else{
-      char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0);
       const char *zSep = "";
       char *zSql;
       int i;
 
-      zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+      zSql = cidxMprintf(&rc, 
+          "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
           zSubExpr, zTab, zSubWhere, zCurrentKey
       );
       for(i=pIdx->nCol-1; i>=0; i--){
@@ -562,16 +679,15 @@ static int cidxFilter(
         if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
         for(j=0; j<2; j++){
           char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
-          zSql = cidxMprintf(&rc, 
-              "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
-              zSql, zSep, zList, zTab, zWhere, zOrderBy
-              );
+          zSql = cidxMprintf(&rc, "%z"
+              "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)",
+              zSql, zSep, zSrcList, zTab, zWhere, zOrderBy
+          );
           zSep = " UNION ALL ";
           if( pIdx->aCol[i].bDesc==0 ) break;
         }
       }
-      zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName);
-      sqlite3_free(zList);
+      zSql = cidxMprintf(&rc, "%z) AS i", zSql);
 
       /* printf("SQL: %s\n", zSql); */
       pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql);
@@ -582,6 +698,7 @@ static int cidxFilter(
     sqlite3_free(zOrderBy);
     sqlite3_free(zSubWhere);
     sqlite3_free(zSubExpr);
+    sqlite3_free(zSrcList);
     cidxFreeIndex(pIdx);
     sqlite3_free(azAfter);
   }

diff --git a/manifest b/manifest
index 48fe0a91a21967137a7857577d30d26b0f989fbc..59ee792f38a1aaff401352cdd9cbc772252cdd19 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions.
-D 2017-10-30T17:05:18.290
+C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check.
+D 2017-10-30T19:38:41.046
 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1
@@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0
 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054
-F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7
+F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3
 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef
 F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a
 F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b
-F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4
+F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3
 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8
 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
@@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67
-R 45b731228881e6db68d5d1d151d2789f
+P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
+R 32cef05dfb77ed0c73cb51913be9d3e3
 U dan
-Z 8f0b667700c59bebe92a45558d3d25e5
+Z 4d91b1f15edf1ab6cd873cde5a983874

diff --git a/manifest.uuid b/manifest.uuid
index 5d8e25d480162e36a33a920881c1771deba100d3..f363d67abb9b3fdc82d263058f07993247a58297 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269
\ No newline at end of file
+8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52
\ No newline at end of file

diff --git a/test/checkindex.test b/test/checkindex.test
index 66481875593279f62b698a9558e24fd3510d4017..a91455558fbd39f1c08edf86e746ad5db527d7a8 100644 (file)
--- a/test/checkindex.test
+++ b/test/checkindex.test
@@ -259,6 +259,64 @@ do_index_check_test 4.3 t4cc {
   {} 'AAB','EEE',4
 }
 
+#--------------------------------------------------------------------------
+# Test an index on an expression.
+#
+do_execsql_test 5.0 {
+  CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y));
+  INSERT INTO t5 VALUES(1, '{"x":1, "y":1}');
+  INSERT INTO t5 VALUES(2, '{"x":2, "y":2}');
+  INSERT INTO t5 VALUES(3, '{"x":3, "y":3}');
+  INSERT INTO t5 VALUES(4, '{"w":4, "z":4}');
+  INSERT INTO t5 VALUES(5, '{"x":5, "y":5}');
+
+  CREATE INDEX t5x ON t5( json_extract(y, '$.x') );
+  CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC );
+}
+
+do_index_check_test 5.1.1 t5x {
+  {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5
+}
+
+do_index_check_test 5.1.2 t5y {
+  {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4
+}
+
+do_index_check_test 5.1.3 sqlite_autoindex_t5_1 {
+  {} {'{"w":4, "z":4}',4} 
+  {} {'{"x":1, "y":1}',1} 
+  {} {'{"x":2, "y":2}',2} 
+  {} {'{"x":3, "y":3}',3} 
+  {} {'{"x":5, "y":5}',5}
+}
+
+do_test 5.2 {
+  set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }]
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot
+  db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);}
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0
+  execsql {
+    UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1;
+    DELETE FROM xt5 WHERE rowid = 4;
+  }
+  sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1
+} {}
+
+do_index_check_test 5.3.1 t5x {
+  {row missing} NULL,4 
+  {row data mismatch} 1,1 
+  {} 2,2 
+  {} 3,3 
+  {} 5,5
+}
+
+do_index_check_test 5.3.2 sqlite_autoindex_t5_1 {
+  {row missing} {'{"w":4, "z":4}',4} 
+  {row data mismatch} {'{"x":1, "y":1}',1} 
+  {} {'{"x":2, "y":2}',2} 
+  {} {'{"x":3, "y":3}',3} 
+  {} {'{"x":5, "y":5}',5}
+}
 
 
 finish_test