From: dan Date: Fri, 5 Jun 2015 19:05:57 +0000 (+0000) Subject: Make use of range constraints on the rowid field of an fts5 table in full-text queries. X-Git-Tag: version-3.8.11~114^2~16 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e4449454c575a4a8926bf29d7ebeb013149000ac;p=thirdparty%2Fsqlite.git Make use of range constraints on the rowid field of an fts5 table in full-text queries. FossilOrigin-Name: 32cbc0ed3699cc21302f0b6a159493117ad4bd4f --- diff --git a/ext/fts5/fts5.c b/ext/fts5/fts5.c index 07a92c3744..c3da8bccf6 100644 --- a/ext/fts5/fts5.c +++ b/ext/fts5/fts5.c @@ -156,10 +156,25 @@ struct Fts5Sorter { ** iSpecial: ** If this is a 'special' query (refer to function fts5SpecialMatch()), ** then this variable contains the result of the query. +** +** iFirstRowid, iLastRowid: +** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the +** cursor iterates in ascending order of rowids, iFirstRowid is the lower +** limit of rowids to return, and iLastRowid the upper. In other words, the +** WHERE clause in the user's query might have been: +** +** MATCH AND rowid BETWEEN $iFirstRowid AND $iLastRowid +** +** If the cursor iterates in descending order of rowid, iFirstRowid +** is the upper limit (i.e. the "first" rowid visited) and iLastRowid +** the lower. */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - int idxNum; /* idxNum passed to xFilter() */ + int ePlan; /* FTS5_PLAN_XXX value */ + int bDesc; /* True for "ORDER BY rowid DESC" queries */ + i64 iFirstRowid; /* Return no rowids earlier than this */ + i64 iLastRowid; /* Return no rowids later than this */ sqlite3_stmt *pStmt; /* Statement used to read %_content */ Fts5Expr *pExpr; /* Expression for MATCH queries */ Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ @@ -181,10 +196,25 @@ struct Fts5Cursor { Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ int *aColumnSize; /* Values for xColumnSize() */ + /* Cache used by auxiliary functions xInst() and xInstCount() */ int nInstCount; /* Number of phrase instances */ int *aInst; /* 3 integers per phrase instance */ }; +/* +** Bits that make up the "idxNum" parameter passed indirectly by +** xBestIndex() to xFilter(). +*/ +#define FTS5_BI_MATCH 0x0001 /* MATCH ? */ +#define FTS5_BI_RANK 0x0002 /* rank MATCH ? */ +#define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */ +#define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */ +#define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */ + +#define FTS5_BI_ORDER_RANK 0x0020 +#define FTS5_BI_ORDER_ROWID 0x0040 +#define FTS5_BI_ORDER_DESC 0x0080 + /* ** Values for Fts5Cursor.csrflags */ @@ -194,6 +224,18 @@ struct Fts5Cursor { #define FTS5CSR_FREE_ZRANK 0x08 #define FTS5CSR_REQUIRE_RESEEK 0x10 +#define BitFlagAllTest(x,y) (((x) & (y))==(y)) +#define BitFlagTest(x,y) (((x) & (y))!=0) + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +** These are copied from sqliteInt.h. +*/ +#ifndef SQLITE_AMALGAMATION +# define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +#endif + /* ** Macros to Set(), Clear() and Test() cursor flags. */ @@ -394,7 +436,7 @@ static int fts5CreateMethod( } /* -** The three query plans xBestIndex may choose between. +** The different query plans. */ #define FTS5_PLAN_SCAN 1 /* No usable constraint */ #define FTS5_PLAN_MATCH 2 /* ( MATCH ?) */ @@ -403,85 +445,131 @@ static int fts5CreateMethod( #define FTS5_PLAN_SOURCE 5 /* A source cursor for SORTED_MATCH */ #define FTS5_PLAN_SPECIAL 6 /* An internal query */ -#define FTS5_PLAN(idxNum) ((idxNum) & 0x7) - -#define FTS5_ORDER_DESC 8 /* ORDER BY rowid DESC */ -#define FTS5_ORDER_ASC 16 /* ORDER BY rowid ASC */ - /* -** Search the object passed as the first argument for a usable constraint -** on column iCol using operator eOp. If one is found, return its index in -** the pInfo->aConstraint[] array. If no such constraint is found, return -** a negative value. -*/ -static int fts5FindConstraint(sqlite3_index_info *pInfo, int eOp, int iCol){ - int i; - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - if( p->usable && p->iColumn==iCol && p->op==eOp ) return i; - } - return -1; -} - -/* -** Implementation of the xBestIndex method for FTS5 tables. There -** are three possible strategies, in order of preference: +** Implementation of the xBestIndex method for FTS5 tables. Within the +** WHERE constraint, it searches for the following: ** -** 1. Full-text search using a MATCH operator. -** 2. A by-rowid lookup. -** 3. A full-table scan. +** 1. A MATCH constraint against the special column. +** 2. A MATCH constraint against the "rank" column. +** 3. An == constraint against the rowid column. +** 4. A < or <= constraint against the rowid column. +** 5. A > or >= constraint against the rowid column. +** +** Within the ORDER BY, either: +** +** 5. ORDER BY rank [ASC|DESC] +** 6. ORDER BY rowid [ASC|DESC] +** +** Costs are assigned as follows: +** +** a) If an unusable MATCH operator is present in the WHERE clause, the +** cost is unconditionally set to 1e50 (a really big number). +** +** a) If a MATCH operator is present, the cost depends on the other +** constraints also present. As follows: +** +** * No other constraints: cost=1000.0 +** * One rowid range constraint: cost=750.0 +** * Both rowid range constraints: cost=500.0 +** * An == rowid constraint: cost=100.0 +** +** b) Otherwise, if there is no MATCH: +** +** * No other constraints: cost=1000000.0 +** * One rowid range constraint: cost=750000.0 +** * Both rowid range constraints: cost=250000.0 +** * An == rowid constraint: cost=10.0 +** +** Costs are not modified by the ORDER BY clause. */ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; - int iCons; - int ePlan = FTS5_PLAN_SCAN; - int iRankMatch; - - iCons = fts5FindConstraint(pInfo,SQLITE_INDEX_CONSTRAINT_MATCH,pConfig->nCol); - if( iCons>=0 ){ - ePlan = FTS5_PLAN_MATCH; - pInfo->estimatedCost = 1.0; - }else{ - iCons = fts5FindConstraint(pInfo, SQLITE_INDEX_CONSTRAINT_EQ, -1); - if( iCons>=0 ){ - ePlan = FTS5_PLAN_ROWID; - pInfo->estimatedCost = 2.0; - } - } + int idxFlags = 0; /* Parameter passed through to xFilter() */ + int bHasMatch; + int iNext; + int i; - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = 1; - pInfo->aConstraintUsage[iCons].omit = 1; - }else{ - pInfo->estimatedCost = 10000000.0; + struct Constraint { + int op; /* Mask against sqlite3_index_constraint.op */ + int fts5op; /* FTS5 mask for idxFlags */ + int iCol; /* 0==rowid, 1==tbl, 2==rank */ + int omit; /* True to omit this if found */ + int iConsIndex; /* Index in pInfo->aConstraint[] */ + } aConstraint[] = { + {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_MATCH, 1, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_RANK, 2, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, + {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, + FTS5_BI_ROWID_LE, 0, 0, -1}, + {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, + FTS5_BI_ROWID_GE, 0, 0, -1}, + }; + + int aColMap[3]; + aColMap[0] = -1; + aColMap[1] = pConfig->nCol; + aColMap[2] = pConfig->nCol+1; + + /* Set idxFlags flags for all WHERE clause terms that will be used. */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + int j; + for(j=0; jiColumn==aColMap[pC->iCol] && p->op & pC->op ){ + if( p->usable ){ + pC->iConsIndex = i; + idxFlags |= pC->fts5op; + }else if( j==0 ){ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Set a prohibitively high cost. */ + pInfo->estimatedCost = 1e50; + return SQLITE_OK; + } + } + } } + /* Set idxFlags flags for the ORDER BY clause */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort<0 ){ - /* ORDER BY rowid [ASC|DESC] */ - pInfo->orderByConsumed = 1; - }else if( iSort==(pConfig->nCol+1) && ePlan==FTS5_PLAN_MATCH ){ - /* ORDER BY rank [ASC|DESC] */ + if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ + idxFlags |= FTS5_BI_ORDER_RANK; + }else if( iSort==-1 ){ + idxFlags |= FTS5_BI_ORDER_ROWID; + } + if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ pInfo->orderByConsumed = 1; - ePlan = FTS5_PLAN_SORTED_MATCH; + if( pInfo->aOrderBy[0].desc ){ + idxFlags |= FTS5_BI_ORDER_DESC; + } } + } - if( pInfo->orderByConsumed ){ - ePlan |= pInfo->aOrderBy[0].desc ? FTS5_ORDER_DESC : FTS5_ORDER_ASC; - } + /* Calculate the estimated cost based on the flags set in idxFlags. */ + bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); + if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ + pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; + }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ + pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; + }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ + pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; + }else{ + pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; } - iRankMatch = fts5FindConstraint( - pInfo, SQLITE_INDEX_CONSTRAINT_MATCH, pConfig->nCol+1 - ); - if( iRankMatch>=0 ){ - pInfo->aConstraintUsage[iRankMatch].argvIndex = 1 + (iCons>=0); - pInfo->aConstraintUsage[iRankMatch].omit = 1; + /* Assign argvIndex values to each constraint in use. */ + iNext = 1; + for(i=0; iiConsIndex>=0 ){ + pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; + pInfo->aConstraintUsage[pC->iConsIndex].omit = pC->omit; + } } - - pInfo->idxNum = ePlan; + + pInfo->idxNum = idxFlags; return SQLITE_OK; } @@ -511,9 +599,9 @@ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return rc; } -static int fts5StmtType(int idxNum){ - if( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ){ - return (idxNum&FTS5_ORDER_DESC) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; +static int fts5StmtType(Fts5Cursor *pCsr){ + if( pCsr->ePlan==FTS5_PLAN_SCAN ){ + return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC; } return FTS5_STMT_LOOKUP; } @@ -544,7 +632,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ fts5CsrNewrow(pCsr); if( pCsr->pStmt ){ - int eStmt = fts5StmtType(pCsr->idxNum); + int eStmt = fts5StmtType(pCsr); sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt); } if( pCsr->pSorter ){ @@ -553,7 +641,7 @@ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3_free(pSorter); } - if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){ + if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){ sqlite3Fts5ExprFree(pCsr->pExpr); } @@ -622,7 +710,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ static void fts5TripCursors(Fts5Table *pTab){ Fts5Cursor *pCsr; for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ - if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH + if( pCsr->ePlan==FTS5_PLAN_MATCH && pCsr->base.pVtab==(sqlite3_vtab*)pTab ){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); @@ -647,18 +735,12 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ assert( *pbSkip==0 ); if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - int bDesc = ((pCsr->idxNum & FTS5_ORDER_DESC) ? 1 : 0); + int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bDesc); - while( rc==SQLITE_OK && sqlite3Fts5ExprEof(pCsr->pExpr)==0 ){ - i64 ii = sqlite3Fts5ExprRowid(pCsr->pExpr); - if( ii==iRowid ) break; - if( (bDesc && iiiRowid) ){ - *pbSkip = 1; - break; - } - rc = sqlite3Fts5ExprNext(pCsr->pExpr); + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc); + if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ + *pbSkip = 1; } CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK); @@ -681,7 +763,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ */ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - int ePlan = FTS5_PLAN(pCsr->idxNum); + int ePlan = pCsr->ePlan; int bSkip = 0; int rc; @@ -690,7 +772,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ switch( ePlan ){ case FTS5_PLAN_MATCH: case FTS5_PLAN_SOURCE: - rc = sqlite3Fts5ExprNext(pCsr->pExpr); + rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid); if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -777,8 +859,9 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bDesc); - if( sqlite3Fts5ExprEof(pCsr->pExpr) ){ + Fts5Expr *pExpr = pCsr->pExpr; + rc = sqlite3Fts5ExprFirst(pExpr, pTab->pIndex, pCsr->iFirstRowid, bDesc); + if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } fts5CsrNewrow(pCsr); @@ -804,7 +887,7 @@ static int fts5SpecialMatch( for(n=0; z[n] && z[n]!=' '; n++); assert( pTab->base.zErrMsg==0 ); - pCsr->idxNum = FTS5_PLAN_SPECIAL; + pCsr->ePlan = FTS5_PLAN_SPECIAL; if( 0==sqlite3_strnicmp("reads", z, n) ){ pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); @@ -927,6 +1010,16 @@ static int fts5CursorParseRank( return rc; } +static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional @@ -947,14 +1040,17 @@ static int fts5FilterMethod( ){ Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - int bDesc = ((idxNum & FTS5_ORDER_DESC) ? 1 : 0); - int rc = SQLITE_OK; + int rc = SQLITE_OK; /* Error code */ + int iVal = 0; /* Counter for apVal[] */ + int bDesc; /* True if ORDER BY [rank|rowid] DESC */ + int bOrderByRank; /* True if ORDER BY rank */ + sqlite3_value *pMatch = 0; /* MATCH ? expression (or NULL) */ + sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ + sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ + sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ + sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ char **pzErrmsg = pTab->pConfig->pzErrmsg; - assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); - pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; - - assert( nVal<=2 ); assert( pCsr->pStmt==0 ); assert( pCsr->pExpr==0 ); assert( pCsr->csrflags==0 ); @@ -962,6 +1058,38 @@ static int fts5FilterMethod( assert( pCsr->zRank==0 ); assert( pCsr->zRankArgs==0 ); + assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); + pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; + + /* Decode the arguments passed through to this function. + ** + ** Note: The following set of if(...) statements must be in the same + ** order as the corresponding entries in the struct at the top of + ** fts5BestIndexMethod(). */ + if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; + if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; + assert( iVal==nVal ); + bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); + pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); + + /* Set the cursor upper and lower rowid limits. Only some strategies + ** actually use them. This is ok, as the xBestIndex() method leaves the + ** sqlite3_index_constraint.omit flag clear for range constraints + ** on the rowid field. */ + if( pRowidEq ){ + pRowidLe = pRowidGe = pRowidEq; + } + if( bDesc ){ + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + }else{ + pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64); + pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); + } + if( pTab->pSortCsr ){ /* If pSortCsr is non-NULL, then this call is being made as part of ** processing for a "... MATCH ORDER BY rank" query (ePlan is @@ -969,48 +1097,49 @@ static int fts5FilterMethod( ** return results to the user for this query. The current cursor ** (pCursor) is used to execute the query issued by function ** fts5CursorFirstSorted() above. */ - assert( FTS5_PLAN(idxNum)==FTS5_PLAN_SCAN ); - pCsr->idxNum = FTS5_PLAN_SOURCE; + assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); + assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); + assert( pCsr->iLastRowid==LARGEST_INT64 ); + assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); - }else{ - int ePlan = FTS5_PLAN(idxNum); - pCsr->idxNum = idxNum; - if( ePlan==FTS5_PLAN_MATCH || ePlan==FTS5_PLAN_SORTED_MATCH ){ - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); + }else if( pMatch ){ + const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); - rc = fts5CursorParseRank(pTab->pConfig, pCsr, (nVal==2 ? apVal[1] : 0)); - if( rc==SQLITE_OK ){ - if( zExpr[0]=='*' ){ - /* The user has issued a query of the form "MATCH '*...'". This - ** indicates that the MATCH expression is not a full text query, - ** but a request for an internal parameter. */ - rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); - }else{ - char **pzErr = &pTab->base.zErrMsg; - rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); - if( rc==SQLITE_OK ){ - if( ePlan==FTS5_PLAN_MATCH ){ - rc = fts5CursorFirst(pTab, pCsr, bDesc); - }else{ - rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); - } + rc = fts5CursorParseRank(pTab->pConfig, pCsr, pRank); + if( rc==SQLITE_OK ){ + if( zExpr[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); + }else{ + char **pzErr = &pTab->base.zErrMsg; + rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); + if( rc==SQLITE_OK ){ + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); + }else{ + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); } } } - }else{ - /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup - ** by rowid (ePlan==FTS5_PLAN_ROWID). */ - int eStmt = fts5StmtType(idxNum); - rc = sqlite3Fts5StorageStmt( - pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg - ); - if( rc==SQLITE_OK ){ - if( ePlan==FTS5_PLAN_ROWID ){ - sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - } - rc = fts5NextMethod(pCursor); + } + }else{ + /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup + ** by rowid (ePlan==FTS5_PLAN_ROWID). */ + pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); + rc = sqlite3Fts5StorageStmt( + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg + ); + if( rc==SQLITE_OK ){ + if( pCsr->ePlan==FTS5_PLAN_ROWID ){ + sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); } + rc = fts5NextMethod(pCursor); } } @@ -1031,9 +1160,9 @@ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ ** Return the rowid that the cursor currently points to. */ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ - assert( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH - || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH - || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE + assert( pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + || pCsr->ePlan==FTS5_PLAN_SOURCE ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; @@ -1050,7 +1179,7 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ */ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; - int ePlan = FTS5_PLAN(pCsr->idxNum); + int ePlan = pCsr->ePlan; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); switch( ePlan ){ @@ -1082,7 +1211,7 @@ static int fts5SeekCursor(Fts5Cursor *pCsr){ /* If the cursor does not yet have a statement handle, obtain one now. */ if( pCsr->pStmt==0 ){ Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - int eStmt = fts5StmtType(pCsr->idxNum); + int eStmt = fts5StmtType(pCsr); rc = sqlite3Fts5StorageStmt( pTab->pStorage, eStmt, &pCsr->pStmt, &pTab->base.zErrMsg ); @@ -1613,7 +1742,9 @@ static int fts5ApiQueryPhrase( rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew); if( rc==SQLITE_OK ){ Fts5Config *pConf = pTab->pConfig; - pNew->idxNum = FTS5_PLAN_MATCH; + pNew->ePlan = FTS5_PLAN_MATCH; + pNew->iFirstRowid = SMALLEST_INT64; + pNew->iLastRowid = LARGEST_INT64; pNew->base.pVtab = (sqlite3_vtab*)pTab; rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr); } @@ -1761,7 +1892,7 @@ static int fts5ColumnMethod( assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); - if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ + if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ if( iCol==pConfig->nCol ){ sqlite3_result_int64(pCtx, pCsr->iSpecial); } @@ -1776,11 +1907,11 @@ static int fts5ColumnMethod( }else if( iCol==pConfig->nCol+1 ){ /* The value of the "rank" column. */ - if( FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SOURCE ){ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); }else if( - FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_MATCH - || FTS5_PLAN(pCsr->idxNum)==FTS5_PLAN_SORTED_MATCH + pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH ){ if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){ fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 6684c2ba8a..54c23df629 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -566,8 +566,8 @@ int sqlite3Fts5ExprNew( ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr); ** } */ -int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, int bDesc); -int sqlite3Fts5ExprNext(Fts5Expr*); +int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); +int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); int sqlite3Fts5ExprEof(Fts5Expr*); i64 sqlite3Fts5ExprRowid(Fts5Expr*); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 6af3b84f31..9707e517aa 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1193,14 +1193,20 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ /* ** Begin iterating through the set of documents in index pIdx matched by -** the MATCH expression passed as the first argument. If the "bDesc" parameter -** is passed a non-zero value, iteration is in descending rowid order. Or, -** if it is zero, in ascending order. +** the MATCH expression passed as the first argument. If the "bDesc" +** parameter is passed a non-zero value, iteration is in descending rowid +** order. Or, if it is zero, in ascending order. +** +** If iterating in ascending rowid order (bDesc==0), the first document +** visited is that with the smallest rowid that is larger than or equal +** to parameter iFirst. Or, if iterating in ascending order (bDesc==1), +** then the first document visited must have a rowid smaller than or +** equal to iFirst. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bDesc){ +int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bDesc){ Fts5ExprNode *pRoot = p->pRoot; int rc = SQLITE_OK; if( pRoot ){ @@ -1208,6 +1214,13 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bDesc){ p->bDesc = bDesc; rc = fts5ExprNodeFirst(p, pRoot); + /* If not at EOF but the current rowid occurs earlier than iFirst in + ** the iteration order, move to document iFirst or later. */ + if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ + rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); + } + + /* If the iterator is not at a real match, skip forward until it is. */ while( pRoot->bNomatch && rc==SQLITE_OK && pRoot->bEof==0 ){ rc = fts5ExprNodeNext(p, pRoot, 0, 0); } @@ -1221,12 +1234,15 @@ int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, int bDesc){ ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. */ -int sqlite3Fts5ExprNext(Fts5Expr *p){ +int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){ int rc; Fts5ExprNode *pRoot = p->pRoot; do { rc = fts5ExprNodeNext(p, pRoot, 0, 0); }while( pRoot->bNomatch && pRoot->bEof==0 && rc==SQLITE_OK ); + if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){ + pRoot->bEof = 1; + } return rc; } diff --git a/ext/fts5/test/fts5ah.test b/ext/fts5/test/fts5ah.test index ed2940763d..3c8ad253d1 100644 --- a/ext/fts5/test/fts5ah.test +++ b/ext/fts5/test/fts5ah.test @@ -104,6 +104,46 @@ foreach {tn q res} " do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res] } +#------------------------------------------------------------------------- +# Now test that adding range constraints on the rowid field reduces the +# number of pages loaded from disk. +# +foreach {tn fraction tail cnt} { + 1 0.6 {rowid > 5000} 5000 + 2 0.2 {rowid > 9000} 1000 + 3 0.2 {rowid < 1000} 999 + 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 + 5 0.6 {rowid >= 5000} 5001 + 6 0.2 {rowid >= 9000} 1001 + 7 0.2 {rowid <= 1000} 1000 + 8 0.6 {rowid > '5000'} 5000 + 9 0.2 {rowid > '9000'} 1000 + 10 0.1 {rowid = 444} 1 +} { + set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" + set n [execsql_reads $q] + set ret [llength [execsql $q]] + + do_test "1.7.$tn.asc.(n=$n ret=$ret)" { + expr {$n < ($fraction*$nReadX) && $ret==$cnt} + } {1} + + set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC" + set n [execsql_reads $q] + set ret [llength [execsql $q]] + do_test "1.7.$tn.desc.(n=$n ret=$ret)" { + expr {$n < 2*$fraction*$nReadX && $ret==$cnt} + } {1} +} + +do_execsql_test 1.8.1 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text'; +} {10000} +do_execsql_test 1.8.2 { + SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; +} {10000} + + #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} finish_test diff --git a/ext/fts5/test/fts5plan.test b/ext/fts5/test/fts5plan.test index 1670f89faa..72fdc60de3 100644 --- a/ext/fts5/test/fts5plan.test +++ b/ext/fts5/test/fts5plan.test @@ -24,34 +24,34 @@ do_eqp_test 1.1 { SELECT * FROM t1, f1 WHERE f1 MATCH t1.x } { 0 0 0 {SCAN TABLE t1} - 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} + 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} } do_eqp_test 1.2 { SELECT * FROM t1, f1 WHERE f1 > t1.x } { - 0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:} 0 1 0 {SCAN TABLE t1} } do_eqp_test 1.3 { SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff } { - 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_eqp_test 1.4 { SELECT * FROM f1 ORDER BY rank } { - 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:} 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } do_eqp_test 1.5 { SELECT * FROM f1 WHERE rank MATCH ? } { - 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:} + 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} } diff --git a/manifest b/manifest index b7b3fe17f8..f34e791c6c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sfts5\sproblem\sin\sextracting\scolumns\sfrom\sposition\slists\scontaining\slarge\svarints. -D 2015-06-03T11:23:30.476 +C Make\suse\sof\srange\sconstraints\son\sthe\srowid\sfield\sof\san\sfts5\stable\sin\sfull-text\squeries. +D 2015-06-05T19:05:57.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d272f8755b464f20e02dd7799bfe16794c9574c4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -105,13 +105,13 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl ed0534dd51efce39878bce33944c6073d37a1e20 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a -F ext/fts5/fts5.c 34e5098e85ed14cc120004c5622536b77ddf4976 +F ext/fts5/fts5.c f5800895e4d24b7351d44a3858c3a1611bb68dac F ext/fts5/fts5.h 4266c6231094005b051dbfc8dd85d2bc57243d34 -F ext/fts5/fts5Int.h 4c677f3b797acde90ba1b7730eca6a32e7def742 +F ext/fts5/fts5Int.h 3de83c9639bd8332eb84a13c1eb2387e83e128bf F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971 F ext/fts5/fts5_buffer.c 9ec57c75c81e81dca118568876b1caead0aadadf F ext/fts5/fts5_config.c 11f969ed711a0a8b611d47431d74c372ad78c713 -F ext/fts5/fts5_expr.c 78a498ba149fbcfbd95c9630054c27955253309d +F ext/fts5/fts5_expr.c 549bda1f7edcf10365fbfbc002bdea1be3c287bb F ext/fts5/fts5_hash.c c1cfdb2cae0fad00b06fae38a40eaf9261563ccc F ext/fts5/fts5_index.c 7cea402924cd3d8cd5943a7f9514c9153696571b F ext/fts5/fts5_storage.c 04e6717656b78eb230a1c730cac3b935eb94889b @@ -130,7 +130,7 @@ F ext/fts5/test/fts5ad.test 312f3c8ed9592533499c5b94d2059ae6382913a0 F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 -F ext/fts5/test/fts5ah.test dbc37d736886e1e38cfa5cd523812db1ad8d0a31 +F ext/fts5/test/fts5ah.test b9e78fa986a7bd564ebadfb244de02c84d7ac3ae F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592 @@ -160,7 +160,7 @@ F ext/fts5/test/fts5integrity.test b45f633381a85dc000e41d68c96ab510985ca35e F ext/fts5/test/fts5merge.test 8077454f2975a63f35761f4b8a718b3a808b7c9c F ext/fts5/test/fts5near.test d2e3343e62d438f2efd96ebcd83a0d30a16ea6dc F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54 -F ext/fts5/test/fts5plan.test 89783f70dab89ff936ed6f21d88959b49c853a47 +F ext/fts5/test/fts5plan.test 7f38179220c9385f88e1470aae6cba134a308b40 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e F ext/fts5/test/fts5porter2.test c534385e88e685b354c2b2020acc0c4920042c8e F ext/fts5/test/fts5prefix.test 7eba86fc270b110ba2b83ba286a1fd4b3b17955e @@ -1357,7 +1357,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ab85a6fc4f7580278fc9d1f0090fdcf0a90d065b -R a504500377faead9f927e5b8c0ebee20 +P 4ea015ab983300d420ef104cca550b22a6395866 +R 375eda4e39d4e39f9d43e9373e13d973 U dan -Z 68530452f05ae148227ba366408c8cfc +Z 9b2a7bdd6b256eb0f1a2b017a5425890 diff --git a/manifest.uuid b/manifest.uuid index 8efcca17c3..d997303f04 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ea015ab983300d420ef104cca550b22a6395866 \ No newline at end of file +32cbc0ed3699cc21302f0b6a159493117ad4bd4f \ No newline at end of file