Rationalize some code in fts3 used by optimize operations, queries of the pending...

FossilOrigin-Name: 29476da353df4c67fe744c1c5f466ba5b9c1a54b

diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index c80d35d179ed33191e46c4241dee74c8aa2c654e..78b55aa85f91b6697e3471b416b9592188c527e9 100644 (file)
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -745,9 +745,10 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){
         PendingList *pList = (PendingList *)fts3HashData(pElem);
         pReader->zTerm = (char *)fts3HashKey(pElem);
         pReader->nTerm = fts3HashKeysize(pElem);
-        pReader->nNode = pReader->nDoclist = pList->nData;
+        pReader->nNode = pReader->nDoclist = pList->nData + 1;
         pReader->aNode = pReader->aDoclist = pList->aData;
         pReader->ppNextElem++;
+        assert( pReader->aNode );
       }
       return SQLITE_OK;
     }
@@ -995,20 +996,26 @@ int sqlite3Fts3SegReaderPending(
   int rc = SQLITE_OK;             /* Return Code */
 
   if( isPrefix ){
+    int nAlloc = 0;               /* Size of allocated array at aElem */
     Fts3HashElem *pE = 0;         /* Iterator variable */
 
     for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
       char *zKey = (char *)fts3HashKey(pE);
       int nKey = fts3HashKeysize(pE);
-      if( nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm) ){
-        int nByte = (1+nElem * sizeof(Fts3HashElem *));
-        Fts3HashElem **aElem2 = (Fts3HashElem **)sqlite3_realloc(aElem, nByte);
-        if( !aElem2 ){
-          rc = SQLITE_NOMEM;
-          nElem = 0;
-          break;
+      if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){
+        if( nElem==nAlloc ){
+          Fts3HashElem **aElem2;
+          nAlloc += 16;
+          aElem2 = (Fts3HashElem **)sqlite3_realloc(
+              aElem, nAlloc*sizeof(Fts3HashElem *)
+          );
+          if( !aElem2 ){
+            rc = SQLITE_NOMEM;
+            nElem = 0;
+            break;
+          }
+          aElem = aElem2;
         }
-        aElem = aElem2;
         aElem[nElem++] = pE;
       }
     }
@@ -2046,6 +2053,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
   SegmentWriter *pWriter = 0;
   int nSegment = 0;               /* Number of segments being merged */
   Fts3SegReader **apSegment = 0;  /* Array of Segment iterators */
+  Fts3SegReader *pPending = 0;    /* Iterator for pending-terms */
   Fts3SegFilter filter;           /* Segment term filter condition */
 
   if( iLevel<0 ){
@@ -2054,9 +2062,14 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
     ** greatest segment level currently present in the database. The index
     ** of the new segment is always 0.
     */
+    rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending);
+    if( rc!=SQLITE_OK ){
+      return rc;
+    }
     iIdx = 0;
     rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
-    if( nSegment==1 ){
+    nSegment += (pPending!=0);
+    if( nSegment<=1 ){
       return SQLITE_DONE;
     }
   }else{
@@ -2096,6 +2109,9 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
     }
   }
   rc = sqlite3_reset(pStmt);
+  if( pPending ){
+    apSegment[i] = pPending;
+  }
   pStmt = 0;
   if( rc!=SQLITE_OK ) goto finished;
 
@@ -2129,62 +2145,82 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
 ** Flush the contents of pendingTerms to a level 0 segment.
 */
 int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
-  Fts3HashElem *pElem;
-  int idx, rc, i;
-  Fts3HashElem **apElem;          /* Array of pointers to hash elements */
-  int nElem;                      /* Number of terms in new segment */
+  int rc;                         /* Return Code */
+  int idx;                        /* Index of new segment created */
   SegmentWriter *pWriter = 0;     /* Used to write the segment */
+  Fts3SegReader *pReader = 0;     /* Used to iterate through the hash table */
 
-  /* Find the number of terms that will make up the new segment. If there
-  ** are no terms, return early (do not bother to write an empty segment).
+  /* Allocate a SegReader object to iterate through the contents of the
+  ** pending-terms table. If an error occurs, or if there are no terms
+  ** in the pending-terms table, return immediately.
   */
-  nElem = fts3HashCount(&p->pendingTerms);
-  if( nElem==0 ){
-    assert( p->nPendingData==0 );
-    return SQLITE_OK;
+  rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pReader);
+  if( rc!=SQLITE_OK || pReader==0 ){
+    return rc;
   }
 
-  /* Determine the next index at level 0, merging as necessary. */
+  /* Determine the next index at level 0. If level 0 is already full, this
+  ** call may merge all existing level 0 segments into a single level 1
+  ** segment.
+  */
   rc = fts3AllocateSegdirIdx(p, 0, &idx);
-  if( rc!=SQLITE_OK ){
-    return rc;
-  } 
 
-  apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *));
-  if( !apElem ){
-    return SQLITE_NOMEM;
-  }
+  /* If no errors have occured, iterate through the contents of the 
+  ** pending-terms hash table using the Fts3SegReader iterator. The callback
+  ** writes each term (along with its doclist) to the database via the
+  ** SegmentWriter handle pWriter.
+  */
+  if( rc==SQLITE_OK ){
+    void *c = (void *)&pWriter;   /* SegReaderIterate() callback context */
+    Fts3SegFilter f;              /* SegReaderIterate() parameters */
 
-  i = 0;
-  for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
-    apElem[i++] = pElem;
+    memset(&f, 0, sizeof(Fts3SegFilter));
+    f.flags = FTS3_SEGMENT_REQUIRE_POS;
+    rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3MergeCallback, c);
   }
-  assert( i==nElem );
+  assert( pWriter || rc!=SQLITE_OK );
 
-  /* TODO(shess) Should we allow user-defined collation sequences,
-  ** here?  I think we only need that once we support prefix searches.
-  ** Also, should we be using qsort()?
+  /* If no errors have occured, flush the SegmentWriter object to the
+  ** database. Then delete the SegmentWriter and Fts3SegReader objects
+  ** allocated by this function.
   */
-  if( nElem>1 ){
-    qsort(apElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
+  if( rc==SQLITE_OK ){
+    rc = fts3SegWriterFlush(p, pWriter, 0, idx);
   }
+  fts3SegWriterFree(pWriter);
+  sqlite3Fts3SegReaderFree(p, pReader);
 
-
-  /* Write the segment tree into the database. */
-  for(i=0; rc==SQLITE_OK && i<nElem; i++){
-    const char *z = fts3HashKey(apElem[i]);
-    int n = fts3HashKeysize(apElem[i]);
-    PendingList *pList = fts3HashData(apElem[i]);
-    rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
-  }
   if( rc==SQLITE_OK ){
-    rc = fts3SegWriterFlush(p, pWriter, 0, idx);
+    sqlite3Fts3PendingTermsClear(p);
+  }
+  return rc;
+}
+
+/*
+** Handle a 'special' INSERT of the form:
+**
+**   "INSERT INTO tbl(tbl) VALUES(<expr>)"
+**
+** Argument pVal contains the result of <expr>. Currently the only 
+** meaningful value to insert is the text 'optimize'.
+*/
+static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
+  int rc;                         /* Return Code */
+  const char *zVal = sqlite3_value_text(pVal);
+  int nVal = sqlite3_value_bytes(pVal);
+
+  if( !zVal ){
+    return SQLITE_NOMEM;
+  }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){
+    rc = fts3SegmentMerge(p, -1);
+    if( rc==SQLITE_DONE || rc==SQLITE_OK ){
+      rc = SQLITE_OK;
+      sqlite3Fts3PendingTermsClear(p);
+    }
+  }else{
+    rc = SQLITE_ERROR;
   }
 
-  /* Free all allocated resources before returning */
-  fts3SegWriterFree(pWriter);
-  sqlite3_free(apElem);
-  sqlite3Fts3PendingTermsClear(p);
   return rc;
 }
 
@@ -2203,6 +2239,7 @@ int sqlite3Fts3UpdateMethod(
   int isRemove = 0;               /* True for an UPDATE or DELETE */
   sqlite3_int64 iRemove = 0;      /* Rowid removed by UPDATE or DELETE */
 
+
   /* If this is a DELETE or UPDATE operation, remove the old record. */
   if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
     int isEmpty;
@@ -2226,6 +2263,8 @@ int sqlite3Fts3UpdateMethod(
         }
       }
     }
+  }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){
+    return fts3SpecialInsert(p, apVal[p->nColumn+2]);
   }
   
   /* If this is an INSERT or UPDATE operation, insert the new record. */
@@ -2251,12 +2290,12 @@ int sqlite3Fts3Optimize(Fts3Table *p){
   int rc;
   rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0);
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts3PendingTermsFlush(p);
-    if( rc==SQLITE_OK ){
-      rc = fts3SegmentMerge(p, -1);
-    }
+    rc = fts3SegmentMerge(p, -1);
     if( rc==SQLITE_OK ){
       rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        sqlite3Fts3PendingTermsClear(p);
+      }
     }else{
       sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
       sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);

diff --git a/manifest b/manifest
index ad36ac0f06559acdca9c87b9dcd304485f1fca82..244deb89aa67e3781313fc1e4660d9ddb9f1dda7 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Add\scomment\sto\sfts3rnd.test\sto\sexplain\show\sthe\stest\sworks.
-D 2009-12-11T07:07:36
+C Rationalize\ssome\scode\sin\sfts3\sused\sby\soptimize\soperations,\squeries\sof\sthe\spending-terms\shash\stable\sand\ssegment\smerges.\sAdd\sthe\s"INSERT\sINTO\stbl(tbl)\sVALUES('optimize')"\ssyntax.
+D 2009-12-11T12:29:05
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -68,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
 F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff
 F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
 F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c ee50b8feb757bf0cddc522223ebd49f91985a1ad
+F ext/fts3/fts3_write.c 883db716247dd36a5d881e817c95b47e869c14cd
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@@ -326,7 +326,7 @@ F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f
 F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb
 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376
 F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17
-F test/e_fts3.test 3f3f70a0105d03d631b0db1408aa17bebd487d7f
+F test/e_fts3.test ad5d08ca8634b1636c6129d023a1139938b6be05
 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea
 F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398
 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041
@@ -778,7 +778,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P ea884e1ed8dba1aa0f3cf68fc71923954983f6c6
-R ae6e3fceaf15353a41abf5fd7eaec6f2
+P 6b740c7cd57d618623ed028be4213dfef860054a
+R 5a68bfd74081748d8df270fc22751e08
 U dan
-Z ad28381e408238f8547c0374a8ed6615
+Z e468921fcf4b1ed0f9a093998ca6e7b2

diff --git a/manifest.uuid b/manifest.uuid
index 82342c7952fc2f7cbf744d5ebb290dccc7df3266..a43e9ceb88ea9d9757d1e5e93eb323f4b47fe6fc 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6b740c7cd57d618623ed028be4213dfef860054a
\ No newline at end of file
+29476da353df4c67fe744c1c5f466ba5b9c1a54b
\ No newline at end of file

diff --git a/test/e_fts3.test b/test/e_fts3.test
index 146d7acc6703ad8b75b607570f46109bb1066e2d..321296cc83126977937a7680a3ed06a106651d62 100644 (file)
--- a/test/e_fts3.test
+++ b/test/e_fts3.test
@@ -184,7 +184,7 @@ write_test 1.2.2.4 docs_content {
 }
 read_test  1.2.2.5 { SELECT count(*) FROM docs_segdir } {3}
 write_test 1.2.2.6 docs_segdir {
-  SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0;
+  INSERT INTO docs(docs) VALUES('optimize');
 }
 read_test  1.2.2.7 { SELECT count(*) FROM docs_segdir } {1}
 ddl_test   1.2.2.8 { DROP TABLE docs }