From: dan Date: Fri, 11 Dec 2009 12:29:04 +0000 (+0000) Subject: Rationalize some code in fts3 used by optimize operations, queries of the pending... X-Git-Tag: version-3.7.2~713 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=4f887abc4c58e070807e2d43d5786f38835bd088;p=thirdparty%2Fsqlite.git Rationalize some code in fts3 used by optimize operations, queries of the pending-terms hash table and segment merges. Add the "INSERT INTO tbl(tbl) VALUES('optimize')" syntax. FossilOrigin-Name: 29476da353df4c67fe744c1c5f466ba5b9c1a54b --- diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index c80d35d179..78b55aa85f 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -745,9 +745,10 @@ static int fts3SegReaderNext(Fts3SegReader *pReader){ PendingList *pList = (PendingList *)fts3HashData(pElem); pReader->zTerm = (char *)fts3HashKey(pElem); pReader->nTerm = fts3HashKeysize(pElem); - pReader->nNode = pReader->nDoclist = pList->nData; + pReader->nNode = pReader->nDoclist = pList->nData + 1; pReader->aNode = pReader->aDoclist = pList->aData; pReader->ppNextElem++; + assert( pReader->aNode ); } return SQLITE_OK; } @@ -995,20 +996,26 @@ int sqlite3Fts3SegReaderPending( int rc = SQLITE_OK; /* Return Code */ if( isPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); - if( nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm) ){ - int nByte = (1+nElem * sizeof(Fts3HashElem *)); - Fts3HashElem **aElem2 = (Fts3HashElem **)sqlite3_realloc(aElem, nByte); - if( !aElem2 ){ - rc = SQLITE_NOMEM; - nElem = 0; - break; + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; } - aElem = aElem2; aElem[nElem++] = pE; } } @@ -2046,6 +2053,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ SegmentWriter *pWriter = 0; int nSegment = 0; /* Number of segments being merged */ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ + Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ Fts3SegFilter filter; /* Segment term filter condition */ if( iLevel<0 ){ @@ -2054,9 +2062,14 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ + rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending); + if( rc!=SQLITE_OK ){ + return rc; + } iIdx = 0; rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel); - if( nSegment==1 ){ + nSegment += (pPending!=0); + if( nSegment<=1 ){ return SQLITE_DONE; } }else{ @@ -2096,6 +2109,9 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ } } rc = sqlite3_reset(pStmt); + if( pPending ){ + apSegment[i] = pPending; + } pStmt = 0; if( rc!=SQLITE_OK ) goto finished; @@ -2129,62 +2145,82 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - Fts3HashElem *pElem; - int idx, rc, i; - Fts3HashElem **apElem; /* Array of pointers to hash elements */ - int nElem; /* Number of terms in new segment */ + int rc; /* Return Code */ + int idx; /* Index of new segment created */ SegmentWriter *pWriter = 0; /* Used to write the segment */ + Fts3SegReader *pReader = 0; /* Used to iterate through the hash table */ - /* Find the number of terms that will make up the new segment. If there - ** are no terms, return early (do not bother to write an empty segment). + /* Allocate a SegReader object to iterate through the contents of the + ** pending-terms table. If an error occurs, or if there are no terms + ** in the pending-terms table, return immediately. */ - nElem = fts3HashCount(&p->pendingTerms); - if( nElem==0 ){ - assert( p->nPendingData==0 ); - return SQLITE_OK; + rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pReader); + if( rc!=SQLITE_OK || pReader==0 ){ + return rc; } - /* Determine the next index at level 0, merging as necessary. */ + /* Determine the next index at level 0. If level 0 is already full, this + ** call may merge all existing level 0 segments into a single level 1 + ** segment. + */ rc = fts3AllocateSegdirIdx(p, 0, &idx); - if( rc!=SQLITE_OK ){ - return rc; - } - apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *)); - if( !apElem ){ - return SQLITE_NOMEM; - } + /* If no errors have occured, iterate through the contents of the + ** pending-terms hash table using the Fts3SegReader iterator. The callback + ** writes each term (along with its doclist) to the database via the + ** SegmentWriter handle pWriter. + */ + if( rc==SQLITE_OK ){ + void *c = (void *)&pWriter; /* SegReaderIterate() callback context */ + Fts3SegFilter f; /* SegReaderIterate() parameters */ - i = 0; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - apElem[i++] = pElem; + memset(&f, 0, sizeof(Fts3SegFilter)); + f.flags = FTS3_SEGMENT_REQUIRE_POS; + rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3MergeCallback, c); } - assert( i==nElem ); + assert( pWriter || rc!=SQLITE_OK ); - /* TODO(shess) Should we allow user-defined collation sequences, - ** here? I think we only need that once we support prefix searches. - ** Also, should we be using qsort()? + /* If no errors have occured, flush the SegmentWriter object to the + ** database. Then delete the SegmentWriter and Fts3SegReader objects + ** allocated by this function. */ - if( nElem>1 ){ - qsort(apElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + if( rc==SQLITE_OK ){ + rc = fts3SegWriterFlush(p, pWriter, 0, idx); } + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFree(p, pReader); - - /* Write the segment tree into the database. */ - for(i=0; rc==SQLITE_OK && iaData, pList->nData+1); - } if( rc==SQLITE_OK ){ - rc = fts3SegWriterFlush(p, pWriter, 0, idx); + sqlite3Fts3PendingTermsClear(p); + } + return rc; +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc; /* Return Code */ + const char *zVal = sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3SegmentMerge(p, -1); + if( rc==SQLITE_DONE || rc==SQLITE_OK ){ + rc = SQLITE_OK; + sqlite3Fts3PendingTermsClear(p); + } + }else{ + rc = SQLITE_ERROR; } - /* Free all allocated resources before returning */ - fts3SegWriterFree(pWriter); - sqlite3_free(apElem); - sqlite3Fts3PendingTermsClear(p); return rc; } @@ -2203,6 +2239,7 @@ int sqlite3Fts3UpdateMethod( int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ + /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ int isEmpty; @@ -2226,6 +2263,8 @@ int sqlite3Fts3UpdateMethod( } } } + }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ + return fts3SpecialInsert(p, apVal[p->nColumn+2]); } /* If this is an INSERT or UPDATE operation, insert the new record. */ @@ -2251,12 +2290,12 @@ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, -1); - } + rc = fts3SegmentMerge(p, -1); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); diff --git a/manifest b/manifest index ad36ac0f06..244deb89aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomment\sto\sfts3rnd.test\sto\sexplain\show\sthe\stest\sworks. -D 2009-12-11T07:07:36 +C Rationalize\ssome\scode\sin\sfts3\sused\sby\soptimize\soperations,\squeries\sof\sthe\spending-terms\shash\stable\sand\ssegment\smerges.\sAdd\sthe\s"INSERT\sINTO\stbl(tbl)\sVALUES('optimize')"\ssyntax. +D 2009-12-11T12:29:05 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -68,7 +68,7 @@ F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496 F ext/fts3/fts3_tokenizer.c 1a49ee3d79cbf0b9386250370d9cbfe4bb89c8ff F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b -F ext/fts3/fts3_write.c ee50b8feb757bf0cddc522223ebd49f91985a1ad +F ext/fts3/fts3_write.c 883db716247dd36a5d881e817c95b47e869c14cd F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33 F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2 @@ -326,7 +326,7 @@ F test/descidx3.test 3394ad4d089335cac743c36a14129d6d931c316f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_fkey.test fd1fcf89badd5f2773d7ac04775b5ff3488eda17 -F test/e_fts3.test 3f3f70a0105d03d631b0db1408aa17bebd487d7f +F test/e_fts3.test ad5d08ca8634b1636c6129d023a1139938b6be05 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 6d91a5286f59add0cfcbb2d0da913b76f2242398 F test/enc3.test 5c550d59ff31dccdba5d1a02ae11c7047d77c041 @@ -778,7 +778,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ea884e1ed8dba1aa0f3cf68fc71923954983f6c6 -R ae6e3fceaf15353a41abf5fd7eaec6f2 +P 6b740c7cd57d618623ed028be4213dfef860054a +R 5a68bfd74081748d8df270fc22751e08 U dan -Z ad28381e408238f8547c0374a8ed6615 +Z e468921fcf4b1ed0f9a093998ca6e7b2 diff --git a/manifest.uuid b/manifest.uuid index 82342c7952..a43e9ceb88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b740c7cd57d618623ed028be4213dfef860054a \ No newline at end of file +29476da353df4c67fe744c1c5f466ba5b9c1a54b \ No newline at end of file diff --git a/test/e_fts3.test b/test/e_fts3.test index 146d7acc67..321296cc83 100644 --- a/test/e_fts3.test +++ b/test/e_fts3.test @@ -184,7 +184,7 @@ write_test 1.2.2.4 docs_content { } read_test 1.2.2.5 { SELECT count(*) FROM docs_segdir } {3} write_test 1.2.2.6 docs_segdir { - SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0; + INSERT INTO docs(docs) VALUES('optimize'); } read_test 1.2.2.7 { SELECT count(*) FROM docs_segdir } {1} ddl_test 1.2.2.8 { DROP TABLE docs }