From 789cb8f5cf62812350e5ea09551c162a4f02a8c1 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Oct 2010 16:44:39 +0000 Subject: [PATCH] Add missing comments and fix compiler warnings in new FTS3/4 code. Other minor fixes too. FossilOrigin-Name: 1c9c70fec3c88319f7b2efe5316694a6ce0ab1a5 --- ext/fts3/fts3.c | 319 ++++++++++++++++++++++++++++-------------- ext/fts3/fts3Int.h | 20 +-- ext/fts3/fts3_write.c | 136 ++++++++---------- manifest | 18 +-- manifest.uuid | 2 +- test/fts3cov.test | 8 +- 6 files changed, 297 insertions(+), 206 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index c3bbb08297..0f8b887f7a 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -970,7 +970,21 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ } } - +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ static int fts3ScanInteriorNode( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ @@ -985,22 +999,23 @@ static int fts3ScanInteriorNode( const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ char *zBuffer = 0; /* Buffer to load terms into */ int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ - int isFirstTerm = 1; /* True when processing first term on page */ - int dummy; - sqlite3_int64 iChild; /* Block id of child node to descend to */ - int nBlock; /* Size of child node in bytes */ - - zCsr += sqlite3Fts3GetVarint32(zCsr, &dummy); + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); while( zCsr1 ){ - const char *zBlob; - int nBlob; + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } + sqlite3_free(zBlob); piLeaf = 0; + zBlob = 0; } rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); } + sqlite3_free(zBlob); } return rc; @@ -1946,38 +1971,24 @@ static int fts3TermSegReaderArray( */ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); }else{ - int rc2; /* Return value of sqlite3Fts3ReadBlock() */ sqlite3_int64 i1; /* First leaf that may contain zTerm */ - sqlite3_int64 i2; /* Last leaf that may contain zTerm */ + sqlite3_int64 i2; /* Final leaf that may contain zTerm */ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); if( isPrefix==0 ) i2 = i1; if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); } - - /* The following call to ReadBlock() serves to reset the SQL statement - ** used to retrieve blocks of data from the %_segments table. If it is - ** not reset here, then it may remain classified as an active statement - ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands - ** failing. - */ - rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = rc2; - } } - iAge++; + assert( (pNew==0)==(rc!=SQLITE_OK) ); /* If a new Fts3SegReader was allocated, add it to the array. */ - assert( pNew!=0 || rc!=SQLITE_OK ); if( rc==SQLITE_OK ){ rc = fts3SegReaderArrayAdd(&pArray, pNew); - }else{ - sqlite3Fts3SegReaderFree(p, pNew); } if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); } + iAge++; } if( rc==SQLITE_DONE ){ @@ -2109,12 +2120,21 @@ static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ return rc; } -static void fts3DoclistStripPositions(char *aList, int *pnList){ +/* +** This function removes the position information from a doclist. When +** called, buffer aList (size *pnList bytes) contains a doclist that includes +** position information. This function removes the position information so +** that aList contains only docids, and adjusts *pnList to reflect the new +** (possibly reduced) size of the doclist. +*/ +static void fts3DoclistStripPositions( + char *aList, /* IN/OUT: Buffer containing doclist */ + int *pnList /* IN/OUT: Size of doclist in bytes */ +){ if( aList ){ char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ char *p = aList; /* Input cursor */ char *pOut = aList; /* Output cursor */ - sqlite3_int64 iPrev = 0; while( peType==FTSQUERY_PHRASE ); + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->isLoaded && pRight->isLoaded ); + if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ sqlite3_free(pLeft->aDoclist); sqlite3_free(pRight->aDoclist); @@ -2317,8 +2360,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ pLeft->aDoclist = 0; rc = SQLITE_OK; }else{ - char *aOut; - int nOut; + char *aOut; /* Buffer in which to assemble new doclist */ + int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, @@ -2342,41 +2385,12 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ return rc; } -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; - -int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - nCost += pPhrase->aToken[ii].pArray->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr);; - (*ppExprCost)++; - } -} +/* +** Allocate an Fts3SegReaderArray for each token in the expression pExpr. +** The allocated objects are stored in the Fts3PhraseToken.pArray member +** variables of each token structure. +*/ static int fts3ExprAllocateSegReaders( Fts3Cursor *pCsr, /* FTS3 table */ Fts3Expr *pExpr, /* Expression to create seg-readers for */ @@ -2391,7 +2405,6 @@ static int fts3ExprAllocateSegReaders( } if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; @@ -2412,6 +2425,11 @@ static int fts3ExprAllocateSegReaders( return rc; } +/* +** Free the Fts3SegReaderArray objects associated with each token in the +** expression pExpr. In other words, this function frees the resources +** allocated by fts3ExprAllocateSegReaders(). +*/ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -2428,10 +2446,89 @@ static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ } /* -** Evaluate the full-text expression pExpr against fts3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for +** Return the sum of the costs of all tokens in the expression pExpr. This +** function must be called after Fts3SegReaderArrays have been allocated +** for all tokens using fts3ExprAllocateSegReaders(). +*/ +int fts3ExprCost(Fts3Expr *pExpr){ + int nCost; /* Return value */ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + nCost = 0; + for(ii=0; iinToken; ii++){ + nCost += pPhrase->aToken[ii].pArray->nCost; + } + }else{ + nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); + } + return nCost; +} + +/* +** The following is a helper function (and type) for fts3EvalExpr(). It +** must be called after Fts3SegReaders have been allocated for every token +** in the expression. See the context it is called from in fts3EvalExpr() +** for further explanation. +*/ +typedef struct ExprAndCost ExprAndCost; +struct ExprAndCost { + Fts3Expr *pExpr; + int nCost; +}; +static void fts3ExprAssignCosts( + Fts3Expr *pExpr, /* Expression to create seg-readers for */ + ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ +){ + if( pExpr->eType==FTSQUERY_AND ){ + fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); + fts3ExprAssignCosts(pExpr->pRight, ppExprCost); + }else{ + (*ppExprCost)->pExpr = pExpr; + (*ppExprCost)->nCost = fts3ExprCost(pExpr);; + (*ppExprCost)++; + } +} + +/* +** Evaluate the full-text expression pExpr against FTS3 table pTab. Store +** the resulting doclist in *paOut and *pnOut. This routine mallocs for +** the space needed to store the output. The caller is responsible for ** freeing the space when it has finished. +** +** This function is called in two distinct contexts: +** +** * From within the virtual table xFilter() method. In this case, the +** output doclist contains entries for all rows in the table, based on +** data read from the full-text index. +** +** In this case, if the query expression contains one or more tokens that +** are very common, then the returned doclist may contain a superset of +** the documents that actually match the expression. +** +** * From within the virtual table xNext() method. This call is only made +** if the call from within xFilter() found that there were very common +** tokens in the query expression and did return a superset of the +** matching documents. In this case the returned doclist contains only +** entries that correspond to the current row of the table. Instead of +** reading the data for each token from the full-text index, the data is +** already available in-memory in the Fts3PhraseToken.pDeferred structures. +** See fts3EvalDeferred() for how it gets there. +** +** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is +** required) Fts3Cursor.doDeferred==1. +** +** If the SQLite invokes the snippet(), offsets() or matchinfo() function +** as part of a SELECT on an FTS3 table, this function is called on each +** individual phrase expression in the query. If there were very common tokens +** found in the xFilter() call, then this function is called once for phrase +** for each row visited, and the returned doclist contains entries for the +** current row only. Otherwise, if there were no very common tokens, then this +** function is called once only for each phrase in the query and the returned +** doclist contains entries for all rows of the table. +** +** Fts3Cursor.doDeferred==1 when this function is called on phrases as a +** result of a snippet(), offsets() or matchinfo() invocation. */ static int fts3EvalExpr( Fts3Cursor *p, /* Virtual table cursor handle */ @@ -2596,9 +2693,26 @@ static int fts3EvalExpr( } /* -** -*/ -static int fts3EvalDeferred(Fts3Cursor *pCsr, int *pbRes){ +** This function is called from within xNext() for each row visited by +** an FTS3 query. If evaluating the FTS3 query expression within xFilter() +** was able to determine the exact set of matching rows, this function sets +** *pbRes to true and returns SQLITE_IO immediately. +** +** Otherwise, if evaluating the query expression within xFilter() returned a +** superset of the matching documents instead of an exact set (this happens +** when the query includes very common tokens and it is deemed too expensive to +** load their doclists from disk), this function tests if the current row +** really does match the FTS3 query. +** +** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK +** is returned and *pbRes is set to true if the current row matches the +** FTS3 query (and should be included in the results returned to SQLite), or +** false otherwise. +*/ +static int fts3EvalDeferred( + Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ + int *pbRes /* OUT: Set to true if row is a match */ +){ int rc = SQLITE_OK; if( pCsr->pDeferred==0 ){ *pbRes = 1; @@ -2677,11 +2791,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ -/* TODO(shess) Upgrade the cursor initialization and destruction to -** account for fts3FilterMethod() being called multiple times on the -** same cursor. The current solution is very fragile. Apply fix to -** fts3 as appropriate. -*/ static int fts3FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ @@ -2785,6 +2894,11 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ + /* This branch runs if the query is implemented using a full-table scan + ** (not using the full-text index). In this case grab the rowid from the + ** SELECT statement. + */ + assert( pCsr->isRequireSeek==0 ); *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; @@ -3210,19 +3324,20 @@ static void hashDestroy(void *p){ } /* -** The fts3 built-in tokenizers - "simple" and "porter" - are implemented -** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following -** two forward declarations are for functions declared in these files -** used to retrieve the respective implementations. +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point to the "simple" tokenizer implementation. -** Function ...PorterTokenizerModule() sets *pModule to point to the -** porter tokenizer/stemmer implementation. +** And so on. */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifdef SQLITE_ENABLE_ICU void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif /* ** Initialise the fts3 extension. If this extension is built as part diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c982717294..5d73b2c773 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -122,13 +122,13 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[25]; + sqlite3_stmt *aStmt[24]; - char *zSegmentsTbl; /* Name of %_segments table */ - int nPgsz; /* Page size for host database */ int nNodeSize; /* Soft limit for node size */ u8 bHasContent; /* True if %_content table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* The following hash table is used to buffer pending index updates during @@ -163,7 +163,6 @@ struct Fts3Cursor { int nDoclist; /* Size of buffer at aDoclist */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ - int doDeferred; int nRowAvg; /* Average size of database rows, in pages */ }; @@ -193,13 +192,12 @@ struct Fts3Cursor { ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. */ - struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ - Fts3SegReaderArray *pArray; - Fts3DeferredToken *pDeferred; + Fts3SegReaderArray *pArray; /* Segment-reader for this token */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ }; struct Fts3Phrase { @@ -258,11 +256,6 @@ struct Fts3Expr { #define FTSQUERY_PHRASE 5 -/* fts3_init.c */ -int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); -int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, - sqlite3_vtab **, char **); - /* fts3_write.c */ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); @@ -276,11 +269,12 @@ int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, int (*)(Fts3Table *, void *, char *, int, char *, int), void * ); -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); +int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); int sqlite3Fts3ReadLock(Fts3Table *); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index b54ece3365..76f11405bf 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -78,11 +78,9 @@ struct Fts3SegReader { sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ - sqlite3_blob *pBlob; /* Blob open on iStartBlock */ char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ - int nTermAlloc; /* Allocated size of zTerm buffer */ Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -92,6 +90,7 @@ struct Fts3SegReader { */ int nTerm; /* Number of bytes in current term */ char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ @@ -170,12 +169,11 @@ struct SegmentNode { #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 -#define SQL_GET_BLOCK 19 -#define SQL_DELETE_DOCSIZE 20 -#define SQL_REPLACE_DOCSIZE 21 -#define SQL_SELECT_DOCSIZE 22 -#define SQL_SELECT_DOCTOTAL 23 -#define SQL_REPLACE_DOCTOTAL 24 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_DOCTOTAL 22 +#define SQL_REPLACE_DOCTOTAL 23 /* ** This function is used to obtain an SQLite prepared statement handle @@ -220,12 +218,11 @@ static int fts3SqlStmt( /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", -/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", -/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", -/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", -/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -300,45 +297,6 @@ static void fts3SqlExec( } -/* -** Read a single block from the %_segments table. If the specified block -** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO -** etc.) occurs, return the appropriate SQLite error code. -** -** Otherwise, if successful, set *pzBlock to point to a buffer containing -** the block read from the database, and *pnBlock to the size of the read -** block in bytes. -** -** WARNING: The returned buffer is only valid until the next call to -** sqlite3Fts3ReadBlock(). -*/ -int sqlite3Fts3ReadBlock( - Fts3Table *p, - sqlite3_int64 iBlock, - char const **pzBlock, - int *pnBlock -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_reset(pStmt); - - if( pzBlock ){ - sqlite3_bind_int64(pStmt, 1, iBlock); - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ - return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); - } - - *pnBlock = sqlite3_column_bytes(pStmt, 0); - *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ - return SQLITE_CORRUPT; - } - } - return SQLITE_OK; -} - /* ** This function ensures that the caller has obtained a shared-cache ** table-lock on the %_content table. This is required before reading @@ -595,6 +553,9 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ return SQLITE_OK; } +/* +** Discard the contents of the pending-terms hash table. +*/ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ @@ -807,24 +768,49 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** The %_segments table is declared as follows: ** ** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. */ -static int fts3SegmentsBlob( - Fts3Table *p, - sqlite3_int64 iSegment, - char **paBlob, - int *pnBlob +int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob /* OUT: Size of blob data */ ){ - int rc; + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob); if( p->pSegments ){ - rc = sqlite3_blob_reopen(p->pSegments, iSegment); + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); }else{ if( 0==p->zSegmentsTbl ){ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; } rc = sqlite3_blob_open( - p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, &p->pSegments + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments ); } @@ -849,12 +835,15 @@ static int fts3SegmentsBlob( return rc; } +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } - /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, @@ -872,8 +861,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - sqlite3_blob *pBlob; - int rc; + int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -898,17 +886,15 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ return SQLITE_OK; } - rc = fts3SegmentsBlob( + rc = sqlite3Fts3ReadBlock( p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode ); - - if( rc!=SQLITE_OK ){ - return rc; - } + if( rc!=SQLITE_OK ) return rc; pNext = pReader->aNode; } @@ -1011,7 +997,6 @@ int sqlite3Fts3SegReaderCost( Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; int rc = SQLITE_OK; /* Return code */ int nCost = 0; /* Cost in bytes to return */ - sqlite3_int64 iLeaf; /* Used to iterate through required leaves */ int pgsz = p->nPgsz; /* Database page size */ /* If this seg-reader is reading the pending-terms table, or if all data @@ -1065,7 +1050,7 @@ int sqlite3Fts3SegReaderCost( ** confirms this). */ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = fts3SegmentsBlob(p, iBlock, 0, &nBlob); + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ int nOvfl = (nBlob + 34)/pgsz; @@ -1134,7 +1119,6 @@ int sqlite3Fts3SegReaderNew( } rc = fts3SegReaderNext(p, pReader); - finished: if( rc==SQLITE_OK ){ *ppReader = pReader; }else{ @@ -1471,7 +1455,7 @@ static int fts3PrefixCompress( ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( - Fts3Table *p, /* Virtual table handle */ + Fts3Table *p, /* Virtual table handle */ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ int isCopyTerm, /* True if zTerm/nTerm is transient */ const char *zTerm, /* Pointer to buffer containing term */ @@ -2244,7 +2228,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int i; /* Iterator variable */ int rc; /* Return code */ int iIdx; /* Index of new segment */ - int iNewLevel; /* Level to create new segment at */ + int iNewLevel = 0; /* Level to create new segment at */ sqlite3_stmt *pStmt = 0; SegmentWriter *pWriter = 0; int nSegment = 0; /* Number of segments being merged */ @@ -2722,7 +2706,7 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ iDocid = sqlite3_column_int64(pCsr->pStmt, 0); for(i=0; inColumn && rc==SQLITE_OK; i++){ - const char *zText = sqlite3_column_text(pCsr->pStmt, i+1); + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); sqlite3_tokenizer_cursor *pTC = 0; rc = pModule->xOpen(pT, zText, -1, &pTC); @@ -2812,7 +2796,7 @@ int sqlite3Fts3UpdateMethod( /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty; + int isEmpty = 0; rc = fts3IsEmpty(p, apVal, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ diff --git a/manifest b/manifest index 334c32ac0c..f272f609a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sbranch. -D 2010-10-21T15:49:47 +C Add\smissing\scomments\sand\sfix\scompiler\swarnings\sin\snew\sFTS3/4\scode.\sOther\sminor\sfixes\stoo. +D 2010-10-22T16:44:39 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2c8cefd962eca0147132c7cf9eaa4bb24c656f3f F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f423181b76fc35c38c4dcf4d8aac012813817f34 +F ext/fts3/fts3.c e2f031ea6b213371a31cc5bf181c2177fef86aad F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h f80be5abfb24e51cb816287230c0d0c8f1712f59 +F ext/fts3/fts3Int.h 068d80157cc7a4bf674d2df817f3b427001ad94a F ext/fts3/fts3_expr.c a5aee50edde20e5c9116199bd58be869a3a22c9f F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec @@ -73,7 +73,7 @@ F ext/fts3/fts3_snippet.c ca60a2a47de5e7abb22a804ccd1a743f81c2fe3e F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c be47d30cf80bc91e050ece18e2de7e207432be1a +F ext/fts3/fts3_write.c 54ddeed7323f62af6e55162f0d4102822b991684 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -431,7 +431,7 @@ F test/fts3ao.test 8fee868a0e131b98ce3e8907dc69936278e8b29a F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 -F test/fts3cov.test 6f1ff88ff6b5abcfff6979098cb9d0c68a69202e +F test/fts3cov.test 54cf1f98c72abee246447cd688590898c9ecbaf7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test cf66bf69afcc2fb8373d3aed31c55399409e83f2 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 @@ -875,7 +875,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d0a450ce78e99f55c862f26f9332786660007a0a f91471e7234db490f97298b1ccb8d6c7fc45b089 -R 3c9178950d1e4be3aecf1d9f3dffa25a +P fd1e5cade04961c2f5438a1dfcc2e15eafb4503f +R 206dc3599103f123f01ecb7943eabb05 U dan -Z 5bbfebd98739f68617c4d86986bdd88f +Z dd51b6660510db15d5633c2bf486673b diff --git a/manifest.uuid b/manifest.uuid index 278a756614..a2460faecb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd1e5cade04961c2f5438a1dfcc2e15eafb4503f \ No newline at end of file +1c9c70fec3c88319f7b2efe5316694a6ce0ab1a5 \ No newline at end of file diff --git a/test/fts3cov.test b/test/fts3cov.test index 92def056cd..89a42df896 100644 --- a/test/fts3cov.test +++ b/test/fts3cov.test @@ -6,9 +6,7 @@ # #*********************************************************************** # -# The tests in this file are structural coverage tests. They are designed -# to complement the tests in fts3rnd.test and fts3doc.test. Between them, -# the three files should provide full coverage of the fts3 extension code. +# The tests in this file are structural coverage tests for FTS3. # set testdir [file dirname $argv0] @@ -97,7 +95,7 @@ do_test fts3cov-2.2 { } {} do_error_test fts3cov-2.3 { SELECT * FROM t1 WHERE t1 MATCH 'c*' -} {database disk image is malformed} +} {SQL logic error or missing database} # Test the "replaced with NULL" case: do_test fts3cov-2.4 { @@ -105,7 +103,7 @@ do_test fts3cov-2.4 { } {} do_error_test fts3cov-2.5 { SELECT * FROM t1 WHERE t1 MATCH 'cloud' -} {database disk image is malformed} +} {SQL logic error or missing database} #-------------------------------------------------------------------------- # The following tests are to test the effects of OOM errors while storing -- 2.47.2