From: dan Date: Thu, 10 Jul 2014 20:21:12 +0000 (+0000) Subject: Support "ORDER BY rowid ASC". X-Git-Tag: version-3.8.11~114^2~167 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=48eecfb8b9b4b7c0cc6dd4281fda4d50e206c88d;p=thirdparty%2Fsqlite.git Support "ORDER BY rowid ASC". FossilOrigin-Name: b96b5e166990e4ec363b24f66e04cfa5f00f6342 --- diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 37bf84e1c6..4618e7d491 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -414,21 +414,25 @@ static int fts5ExprNearAdvanceAll( */ static int fts5ExprAdvanceto( Fts5IndexIter *pIter, /* Iterator to advance */ - i64 *piMin, /* IN/OUT: Minimum rowid seen so far */ + int bAsc, /* True if iterator is "rowid ASC" */ + i64 *piLast, /* IN/OUT: Lastest rowid seen so far */ int *pRc, /* OUT: Error code */ int *pbEof /* OUT: Set to true if EOF */ ){ - i64 iMin = *piMin; + i64 iLast = *piLast; i64 iRowid; - while( (iRowid = sqlite3Fts5IterRowid(pIter))>iMin ){ + while( 1 ){ + iRowid = sqlite3Fts5IterRowid(pIter); + if( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) ) break; sqlite3Fts5IterNext(pIter, 0); if( sqlite3Fts5IterEof(pIter) ){ *pbEof = 1; return 1; } } - if( iRowidiLast) ); + *piLast = iRowid; } return 0; @@ -452,10 +456,15 @@ static int fts5ExprNearNextRowidMatch( Fts5ExprNearset *pNear = pNode->pNear; int rc = SQLITE_OK; int i, j; /* Phrase and token index, respectively */ - i64 iMin; /* Smallest rowid any iterator points to */ - int bMatch; + i64 iLast; /* Lastest rowid any iterator points to */ + int bMatch; /* True if all terms are at the same rowid */ + + /* Set iLast, the lastest rowid any iterator points to. If the iterator + ** skips through rowids in the default descending order, this means the + ** minimum rowid. Or, if the iterator is "ORDER BY rowid ASC", then it + ** means the maximum rowid. */ + iLast = sqlite3Fts5IterRowid(pNear->apPhrase[0]->aTerm[0].pIter); - iMin = sqlite3Fts5IterRowid(pNear->apPhrase[0]->aTerm[0].pIter); do { bMatch = 1; for(i=0; inPhrase; i++){ @@ -463,13 +472,15 @@ static int fts5ExprNearNextRowidMatch( for(j=0; jnTerm; j++){ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter; i64 iRowid = sqlite3Fts5IterRowid(pIter); - if( iRowid!=iMin ) bMatch = 0; - if( fts5ExprAdvanceto(pIter, &iMin, &rc, &pNode->bEof) ) return rc; + if( iRowid!=iLast ) bMatch = 0; + if( fts5ExprAdvanceto(pIter, pExpr->bAsc, &iLast, &rc, &pNode->bEof) ){ + return rc; + } } } }while( bMatch==0 ); - pNode->iRowid = iMin; + pNode->iRowid = iLast; return rc; } @@ -555,7 +566,7 @@ static int fts5ExprNearInitAll( (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0) ); - if( sqlite3Fts5IterEof(pTerm->pIter) ){ + if( pTerm->pIter && sqlite3Fts5IterEof(pTerm->pIter) ){ pNode->bEof = 1; return SQLITE_OK; } @@ -569,16 +580,31 @@ static int fts5ExprNearInitAll( static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*); /* -** Nodes at EOF are considered larger than all other nodes. A node that -** points to a *smaller* rowid is considered larger. -** +** Compare the values currently indicated by the two nodes as follows: +** ** res = (*p1) - (*p2) +** +** Nodes that point to values that come later in the iteration order are +** considered to be larger. Nodes at EOF are the largest of all. +** +** This means that if the iteration order is ASC, then numerically larger +** rowids are considered larger. Or if it is the default DESC, numerically +** smaller rowids are larger. */ -static int fts5NodeCompare(Fts5ExprNode *p1, Fts5ExprNode *p2){ +static int fts5NodeCompare( + Fts5Expr *pExpr, + Fts5ExprNode *p1, + Fts5ExprNode *p2 +){ if( p2->bEof ) return -1; if( p1->bEof ) return +1; - if( p1->iRowid>p2->iRowid ) return -1; - return (p1->iRowid < p2->iRowid); + if( pExpr->bAsc ){ + if( p1->iRowidiRowid ) return -1; + return (p1->iRowid > p2->iRowid); + }else{ + if( p1->iRowid>p2->iRowid ) return -1; + return (p1->iRowid < p2->iRowid); + } } static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){ @@ -600,7 +626,7 @@ static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){ case FTS5_OR: { Fts5ExprNode *p1 = pNode->pLeft; Fts5ExprNode *p2 = pNode->pRight; - int cmp = fts5NodeCompare(p1, p2); + int cmp = fts5NodeCompare(pExpr, p1, p2); if( cmp==0 ){ rc = fts5ExprNodeNext(pExpr, p1); @@ -644,7 +670,12 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){ Fts5ExprNode *p2 = pNode->pRight; while( p1->bEof==0 && p2->bEof==0 && p2->iRowid!=p1->iRowid ){ - Fts5ExprNode *pAdv = (p1->iRowid > p2->iRowid) ? p1 : p2; + Fts5ExprNode *pAdv; + if( pExpr->bAsc ){ + pAdv = (p1->iRowid < p2->iRowid) ? p1 : p2; + }else{ + pAdv = (p1->iRowid > p2->iRowid) ? p1 : p2; + } rc = fts5ExprNodeNext(pExpr, pAdv); if( rc!=SQLITE_OK ) break; } @@ -656,7 +687,7 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){ case FTS5_OR: { Fts5ExprNode *p1 = pNode->pLeft; Fts5ExprNode *p2 = pNode->pRight; - Fts5ExprNode *pNext = (fts5NodeCompare(p1, p2) > 0 ? p2 : p1); + Fts5ExprNode *pNext = (fts5NodeCompare(pExpr, p1, p2) > 0 ? p2 : p1); pNode->bEof = pNext->bEof; pNode->iRowid = pNext->iRowid; break; @@ -667,7 +698,7 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){ Fts5ExprNode *p2 = pNode->pRight; while( rc==SQLITE_OK ){ int cmp; - while( rc==SQLITE_OK && (cmp = fts5NodeCompare(p1, p2))>0 ){ + while( rc==SQLITE_OK && (cmp = fts5NodeCompare(pExpr, p1, p2))>0 ){ rc = fts5ExprNodeNext(pExpr, p2); } if( rc || cmp ) break; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index bfd6afa18e..e6ea440f5f 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -299,6 +299,7 @@ struct Fts5Index { }; struct Fts5DoclistIter { + int bAsc; u8 *a; int n; int i; @@ -410,6 +411,7 @@ struct Fts5SegWriter { */ struct Fts5MultiSegIter { int nSeg; /* Size of aSeg[] array */ + int bRev; /* True to iterate in reverse order */ Fts5SegIter *aSeg; /* Array of segment iterators */ u16 *aFirst; /* Current merge state (see above) */ }; @@ -443,6 +445,18 @@ struct Fts5MultiSegIter { ** FTS5_SEGITER_ONETERM: ** If set, set the iterator to point to EOF after the current doclist ** has been exhausted. Do not proceed to the next term in the segment. +** +** FTS5_SEGITER_REVERSE: +** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If +** it is set, iterate through docids in ascending order instead of the +** default descending order. +** +** iRowidOffset/nRowidOffset/aRowidOffset: +** These are used if the FTS5_SEGITER_REVERSE flag is set. +** +** Each time a new page is loaded, the iterator is set to point to the +** final rowid. Additionally, the aRowidOffset[] array is populated +** with the byte offsets of all relevant rowid fields on the page. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -452,15 +466,23 @@ struct Fts5SegIter { Fts5Data *pLeaf; /* Current leaf data */ int iLeafOffset; /* Byte offset within current leaf */ + /* The page and offset from which the current term was read. The offset + ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; + /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */ + int iRowidOffset; /* Current entry in aRowidOffset[] */ + int nRowidOffset; /* Allocated size of aRowidOffset[] array */ + int *aRowidOffset; /* Array of offset to rowid fields */ + /* Variables populated based on current entry. */ Fts5Buffer term; /* Current term */ i64 iRowid; /* Current rowid */ }; #define FTS5_SEGITER_ONETERM 0x01 +#define FTS5_SEGITER_REVERSE 0x02 /* @@ -1076,6 +1098,55 @@ static void fts5SegIterInit( } } +static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){ + *piRowid = (int)fts5GetU16(&pLeaf->p[0]); + *piTerm = (int)fts5GetU16(&pLeaf->p[2]); +} + +/* +** This function is only ever called on iterators created by calls to +** Fts5IndexQuery() with the FTS5INDEX_QUERY_ASC flag set. +** +** When this function is called, iterator pIter points to the first rowid +** on the current leaf associated with the term being queried. This function +** advances it to point to the last such rowid and, if necessary, initializes +** the aRowidOffset[] and iRowidOffset variables. +*/ +static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ + int n = pIter->pLeaf->n; + int i = pIter->iLeafOffset; + u8 *a = pIter->pLeaf->p; + int iRowidOffset = 0; + + while( p->rc==SQLITE_OK && i=n ) break; + i += getVarint(&a[i], (u64*)&iDelta); + if( iDelta==0 ) break; + pIter->iRowid -= iDelta; + + if( iRowidOffset>=pIter->nRowidOffset ){ + int nNew = pIter->nRowidOffset + 8; + int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + pIter->aRowidOffset = aNew; + pIter->nRowidOffset = nNew; + } + + pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset; + pIter->iLeafOffset = i; + } + pIter->iRowidOffset = iRowidOffset; +} + + /* ** Advance iterator pIter to the next entry. ** @@ -1088,66 +1159,196 @@ static void fts5SegIterNext( Fts5SegIter *pIter /* Iterator to advance */ ){ if( p->rc==SQLITE_OK ){ - Fts5Data *pLeaf = pIter->pLeaf; - int iOff; - int bNewTerm = 0; - int nKeep = 0; + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + if( pIter->iRowidOffset>0 ){ + u8 *a = pIter->pLeaf->p; + int iOff; + int nPos; + i64 iDelta; + pIter->iRowidOffset--; + + pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; + iOff += getVarint32(&a[iOff], nPos); + iOff += nPos; + getVarint(&a[iOff], (u64*)&iDelta); + pIter->iRowid += iDelta; + }else{ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ + Fts5Data *pNew; + pIter->iLeafPgno--; + pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( + pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno + )); + if( pNew ){ + if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ + if( pIter->iTermLeafOffsetn ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; + } + }else{ + int iRowidOff, dummy; + fts5LeafHeader(pNew, &iRowidOff, &dummy); + if( iRowidOff ){ + pIter->pLeaf = pNew; + pIter->iLeafOffset = iRowidOff; + } + } + + if( pIter->pLeaf ){ + u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; + pIter->iLeafOffset += getVarint(a, (u64*)&pIter->iRowid); + break; + }else{ + fts5DataRelease(pNew); + } + } + } + + if( pIter->pLeaf ){ + fts5SegIterReverseInitPage(p, pIter); + } + } + }else{ + Fts5Data *pLeaf = pIter->pLeaf; + int iOff; + int bNewTerm = 0; + int nKeep = 0; - /* Search for the end of the position list within the current page. */ - u8 *a = pLeaf->p; - int n = pLeaf->n; + /* Search for the end of the position list within the current page. */ + u8 *a = pLeaf->p; + int n = pLeaf->n; - iOff = pIter->iLeafOffset; - if( iOffiLeafOffset; + if( iOffiLeafOffset = iOff; - if( iDelta==0 ){ - bNewTerm = 1; - if( iOff>=n ){ - fts5SegIterNextPage(p, pIter); - pIter->iLeafOffset = 4; - }else if( iOff!=fts5GetU16(&a[2]) ){ - pIter->iLeafOffset += getVarint32(&a[iOff], nKeep); + if( iOffiLeafOffset = iOff; + if( iDelta==0 ){ + bNewTerm = 1; + if( iOff>=n ){ + fts5SegIterNextPage(p, pIter); + pIter->iLeafOffset = 4; + }else if( iOff!=fts5GetU16(&a[2]) ){ + pIter->iLeafOffset += getVarint32(&a[iOff], nKeep); + } + }else{ + pIter->iRowid -= iDelta; } }else{ - pIter->iRowid -= iDelta; - } - }else{ - iOff = 0; - /* Next entry is not on the current page */ - while( iOff==0 ){ - fts5SegIterNextPage(p, pIter); - pLeaf = pIter->pLeaf; - if( pLeaf==0 ) break; - if( (iOff = fts5GetU16(&pLeaf->p[0])) ){ - iOff += sqlite3GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; + iOff = 0; + /* Next entry is not on the current page */ + while( iOff==0 ){ + fts5SegIterNextPage(p, pIter); + pLeaf = pIter->pLeaf; + if( pLeaf==0 ) break; + if( (iOff = fts5GetU16(&pLeaf->p[0])) ){ + iOff += sqlite3GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + } + else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){ + pIter->iLeafOffset = iOff; + bNewTerm = 1; + } } - else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){ - pIter->iLeafOffset = iOff; - bNewTerm = 1; + } + + /* Check if the iterator is now at EOF. If so, return early. */ + if( pIter->pLeaf && bNewTerm ){ + if( pIter->flags & FTS5_SEGITER_ONETERM ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + fts5SegIterLoadTerm(p, pIter, nKeep); } } } + } +} - /* Check if the iterator is now at EOF. If so, return early. */ - if( pIter->pLeaf && bNewTerm ){ - if( pIter->flags & FTS5_SEGITER_ONETERM ){ - fts5DataRelease(pIter->pLeaf); - pIter->pLeaf = 0; - }else{ - fts5SegIterLoadTerm(p, pIter, nKeep); +/* +** Iterator pIter currently points to the first rowid in a doclist. This +** function sets the iterator up so that iterates in reverse order through +** the doclist. +*/ +static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ + Fts5Data *pLeaf; /* Current leaf data */ + int iOff = pIter->iLeafOffset; /* Byte offset within current leaf */ + Fts5Data *pLast = 0; + int pgnoLast = 0; + + /* Move to the page that contains the last rowid in this doclist. */ + pLeaf = pIter->pLeaf; + + while( iOffn ){ + int nPos; + i64 iDelta; + + /* Position list size in bytes */ + iOff += getVarint32(&pLeaf->p[iOff], nPos); + iOff += nPos; + if( iOff>=pLeaf->n ) break; + + /* Rowid delta. Or, if 0x00, the end of doclist marker. */ + nPos = getVarint(&pLeaf->p[iOff], (u64*)&iDelta); + if( iDelta==0 ) break; + iOff += nPos; + } + + if( iOff>=pLeaf->n ){ + Fts5StructureSegment *pSeg = pIter->pSeg; + i64 iAbs = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pIter->iLeafPgno); + i64 iLast = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pSeg->pgnoLast); + + /* The last rowid in the doclist may not be on the current page. Search + ** forward to find the page containing the last rowid. */ + for(iAbs++; p->rc==SQLITE_OK && iAbs<=iLast; iAbs++){ + Fts5Data *pNew = fts5DataRead(p, iAbs); + if( pNew ){ + int iRowid, iTerm; + fts5LeafHeader(pNew, &iRowid, &iTerm); + if( iRowid ){ + Fts5Data *pTmp = pLast; + pLast = pNew; + pNew = pTmp; + pgnoLast = iAbs & (((i64)1 << FTS5_DATA_PAGE_B) - 1); + } + if( iTerm ){ + iAbs = iLast; + } + fts5DataRelease(pNew); } } } + + /* If pLast is NULL at this point, then the last rowid for this doclist + ** lies on the page currently indicated by the iterator. In this case + ** iLastOff is set to the value that pIter->iLeafOffset will take when + ** the iterator points to that rowid. + ** + ** Or, if pLast is non-NULL, then it is the page that contains the last + ** rowid. + */ + if( pLast ){ + int dummy; + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = pLast; + pIter->iLeafPgno = pgnoLast; + fts5LeafHeader(pLast, &iOff, &dummy); + iOff += getVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + } + + fts5SegIterReverseInitPage(p, pIter); + pIter->flags |= FTS5_SEGITER_REVERSE; } /* @@ -1162,13 +1363,15 @@ static void fts5SegIterSeekInit( Fts5Index *p, /* FTS5 backend */ int iIdx, /* Config.aHash[] index of FTS index */ const u8 *pTerm, int nTerm, /* Term to seek to */ - int bGe, /* If true seek for >=. If false, == */ + int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int h; + int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0); + assert( bGe==0 || (flags & FTS5INDEX_QUERY_ASC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; @@ -1220,7 +1423,12 @@ static void fts5SegIterSeekInit( } } - if( bGe==0 ) pIter->flags |= FTS5_SEGITER_ONETERM; + if( bGe==0 ){ + pIter->flags |= FTS5_SEGITER_ONETERM; + if( pIter->pLeaf && (flags & FTS5INDEX_QUERY_ASC) ){ + fts5SegIterReverse(p, iIdx, pIter); + } + } } /* @@ -1229,6 +1437,7 @@ static void fts5SegIterSeekInit( static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); fts5DataRelease(pIter->pLeaf); + sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); } @@ -1248,6 +1457,7 @@ static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){ Fts5SegIter *p2; /* Right-hand Fts5SegIter */ assert( iOutnSeg && iOut>0 ); + assert( pIter->bRev==0 || pIter->bRev==1 ); if( iOut>=(pIter->nSeg/2) ){ i1 = (iOut - pIter->nSeg/2) * 2; @@ -1269,7 +1479,7 @@ static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){ assert( i2>i1 ); assert( i2!=0 ); if( p1->iRowid==p2->iRowid ) return i2; - res = (p1->iRowid > p2->iRowid) ? -1 : +1; + res = ((p1->iRowid < p2->iRowid)==pIter->bRev) ? -1 : +1; } assert( res!=0 ); if( res<0 ){ @@ -1342,7 +1552,7 @@ static void fts5MultiIterNew( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Structure *pStruct, /* Structure of specific index */ int iIdx, /* Config.aHash[] index of FTS index */ - int bGe, /* True for >= */ + int flags, /* True for >= */ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ @@ -1373,6 +1583,7 @@ static void fts5MultiIterNew( pNew->nSeg = nSlot; pNew->aSeg = (Fts5SegIter*)&pNew[1]; pNew->aFirst = (u16*)&pNew->aSeg[nSlot]; + pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_ASC)); /* Initialize each of the component segment iterators. */ if( iLevel<0 ){ @@ -1384,7 +1595,7 @@ static void fts5MultiIterNew( if( pTerm==0 ){ fts5SegIterInit(p, iIdx, pSeg, pIter); }else{ - fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, bGe, pSeg, pIter); + fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, flags, pSeg, pIter); } } } @@ -3074,7 +3285,11 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ if( pIter->i ){ i64 iDelta; pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta); - pIter->iRowid -= iDelta; + if( pIter->bAsc ){ + pIter->iRowid += iDelta; + }else{ + pIter->iRowid -= iDelta; + } }else{ pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid); } @@ -3086,10 +3301,15 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ } } -static void fts5DoclistIterInit(Fts5Buffer *pBuf, Fts5DoclistIter *pIter){ +static void fts5DoclistIterInit( + Fts5Buffer *pBuf, + int bAsc, + Fts5DoclistIter *pIter +){ memset(pIter, 0, sizeof(*pIter)); pIter->a = pBuf->p; pIter->n = pBuf->n; + pIter->bAsc = bAsc; fts5DoclistIterNext(pIter); } @@ -3098,14 +3318,17 @@ static void fts5DoclistIterInit(Fts5Buffer *pBuf, Fts5DoclistIter *pIter){ */ static void fts5MergeAppendDocid( int *pRc, /* IN/OUT: Error code */ + int bAsc, Fts5Buffer *pBuf, /* Buffer to write to */ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ i64 iRowid /* Rowid to append */ ){ if( pBuf->n==0 ){ fts5BufferAppendVarint(pRc, pBuf, iRowid); - }else{ + }else if( bAsc==0 ){ fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid); + }else{ + fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid); } *piLastRowid = iRowid; } @@ -3120,6 +3343,7 @@ static void fts5MergeAppendDocid( */ static void fts5MergePrefixLists( Fts5Index *p, /* FTS5 backend object */ + int bAsc, Fts5Buffer *p1, /* First list to merge */ Fts5Buffer *p2 /* Second list to merge */ ){ @@ -3132,19 +3356,21 @@ static void fts5MergePrefixLists( memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); - fts5DoclistIterInit(p1, &i1); - fts5DoclistIterInit(p2, &i2); + fts5DoclistIterInit(p1, bAsc, &i1); + fts5DoclistIterInit(p2, bAsc, &i2); while( i1.aPoslist!=0 || i2.aPoslist!=0 ){ - if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid>i2.iRowid) ){ + if( i2.aPoslist==0 || (i1.aPoslist && + ( (!bAsc && i1.iRowid>i2.iRowid) || (bAsc && i1.iRowidrc, &out, &iLastRowid, i1.iRowid); + fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i1.iRowid); fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist); fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist); fts5DoclistIterNext(&i1); } - else if( i1.aPoslist==0 || i2.iRowid>i1.iRowid ){ + else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid); fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist); fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist); fts5DoclistIterNext(&i2); @@ -3157,7 +3383,7 @@ static void fts5MergePrefixLists( memset(&writer, 0, sizeof(writer)); /* Merge the two position lists. */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid); fts5BufferZero(&tmp); sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1); sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2); @@ -3195,6 +3421,7 @@ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ + int bAsc, /* True for "ORDER BY rowid ASC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5IndexIter *pIter /* Populate this object */ @@ -3203,7 +3430,6 @@ static void fts5SetupPrefixIter( Fts5Buffer *aBuf; const int nBuf = 32; - aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p, 0); @@ -3225,29 +3451,34 @@ static void fts5SetupPrefixIter( assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( nTerm0 && iRowid>=iLastRowid ){ + if( doclist.n>0 + && ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid)) + ){ + for(i=0; doclist.n && p->rc==SQLITE_OK; i++){ assert( irc, &doclist, iRowid); - }else{ + }else if( bAsc==0 ){ fts5BufferAppendVarint(&p->rc, &doclist, iLastRowid - iRowid); + }else{ + fts5BufferAppendVarint(&p->rc, &doclist, iRowid - iLastRowid); } iLastRowid = iRowid; fts5MultiIterPoslist(p, p1, 1, &doclist); } for(i=0; ipDoclist = pDoclist; - fts5DoclistIterInit(&doclist, pIter->pDoclist); + fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist); } } @@ -3296,11 +3527,12 @@ Fts5IndexIter *sqlite3Fts5IndexQuery( pRet->pStruct = fts5StructureRead(p, iIdx); if( pRet->pStruct ){ fts5MultiIterNew(p, pRet->pStruct, - iIdx, 0, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti + iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti ); } }else{ - fts5SetupPrefixIter(p, (const u8*)pToken, nToken, pRet); + int bAsc = (flags & FTS5INDEX_QUERY_ASC)!=0; + fts5SetupPrefixIter(p, bAsc, (const u8*)pToken, nToken, pRet); } } @@ -3362,6 +3594,7 @@ const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){ Fts5Index *p = pIter->pIndex; fts5BufferZero(&pIter->poslist); fts5MultiIterPoslist(p, pIter->pMulti, 0, &pIter->poslist); + assert( p->rc==SQLITE_OK ); if( p->rc ) return 0; *pn = pIter->poslist.n; return pIter->poslist.p; diff --git a/manifest b/manifest index 4480043dec..f9998628d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sprefix\squeries\sto\sfts5. -D 2014-07-08T16:27:37.120 +C Support\s"ORDER\sBY\srowid\sASC". +D 2014-07-10T20:21:12.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -107,8 +107,8 @@ F ext/fts5/fts5.c 1af3184dd9c0e5c1686f71202d6b6cac8f225f05 F ext/fts5/fts5Int.h bb716a6e6a376a7c8211e55e5577c6c020d176c2 F ext/fts5/fts5_buffer.c 83b463a179ad4348fa87796fce78b0e4ef6b898a F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef -F ext/fts5/fts5_expr.c 21351cdd256f8e561a57a38490d27f7922247696 -F ext/fts5/fts5_index.c a3084168a384a9d43f7fb045511b386ccb6e55e8 +F ext/fts5/fts5_expr.c 0dc31b06d444cad097bec05699797590729d2638 +F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36 F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb @@ -593,9 +593,9 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343 -F test/fts5ab.test 4db86a9473ee2a8c2cb30e0d81df21c6022f99b6 -F test/fts5ac.test d3aeb7a079d40093b34ac8053fc5e4c0ed7e88dc -F test/fts5ad.test a4d2f344c86a45ee53b424512585b3900ccb8cf3 +F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397 +F test/fts5ac.test 28203ba2334030514d7a6271c5fb1ba3cbc219b1 +F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07 F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef @@ -763,7 +763,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 43a4c2397b5e8a45c41fac20c7a8a2d4094f470f +F test/permutations.test 0b5333e5dcdeffba0ecbe5ee8dc7577029ffab6c F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1191,7 +1191,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8682b87e794767cefcaa080fd53c8973c24c556a -R dba83c3d230dbf413439715289f715cf +P 75ebd3cd5904a4f89f7f3a9b25d32b2a42a31310 +R 5a76d2f2fc0d7fcaa9a60fabc7fdb146 U dan -Z 876bab147b2ac69a43a21b2ef49df211 +Z 870004bd588f44c77d8063239acbea69 diff --git a/manifest.uuid b/manifest.uuid index 9182ec19d5..e8c5df2055 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75ebd3cd5904a4f89f7f3a9b25d32b2a42a31310 \ No newline at end of file +b96b5e166990e4ec363b24f66e04cfa5f00f6342 \ No newline at end of file diff --git a/test/fts5ab.test b/test/fts5ab.test index 88a876d3b2..1f6b7171da 100644 --- a/test/fts5ab.test +++ b/test/fts5ab.test @@ -124,7 +124,6 @@ foreach {tn expr res} { 3 {abase + abash} {1} 4 {abash + abase} {9} 5 {abaft + abashing} {8 5} - 6 {abandon + abandoning} {10} 7 {"abashing abases abasement abaft abashing"} {8} } { @@ -133,10 +132,23 @@ foreach {tn expr res} { } $res } -breakpoint do_execsql_test 3.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)' } {6} +foreach {tn expr res} { + 1 {abash} {1 3 5 9} + 2 {abase} {1 3 4 9} + 3 {abase + abash} {1} + 4 {abash + abase} {9} + 5 {abaft + abashing} {5 8} + 6 {abandon + abandoning} {10} + 7 {"abashing abases abasement abaft abashing"} {8} +} { + do_execsql_test 3.4.$tn { + SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC + } $res +} + finish_test diff --git a/test/fts5ac.test b/test/fts5ac.test index 849ea52e5e..ddd27481a1 100644 --- a/test/fts5ac.test +++ b/test/fts5ac.test @@ -208,18 +208,24 @@ proc nearset {aCol args} { return $bMatch } -proc matchdata {expr {print 0}} { +proc matchdata {expr {bAsc 0}} { set tclexpr [db one {SELECT fts5_expr_tcl($expr, 'nearset $cols', 'x', 'y')}] set res [list] foreach {id x y} $::data { set cols [list $x $y] if $tclexpr { - set res [concat $id $res] + lappend res $id } } - if {$print} { - puts $tclexpr + + # puts $tclexpr + + if {$bAsc} { + set res [lsort -integer -increasing $res] + } else { + set res [lsort -integer -decreasing $res] } + return $res } @@ -263,32 +269,37 @@ foreach {tn expr tclexpr} { #------------------------------------------------------------------------- # -foreach {tn expr} { - 1 { NEAR(r c) } - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } - 10 { x : "r c" } - 11 { y : "r c" } - 12 { a AND b } - 13 { a AND b AND c } - 14a { a } - 14b { a OR b } - 15 { a OR b AND c } - 16 { c AND b OR a } - 17 { c AND (b OR a) } - 18 { c NOT (b OR a) } - 19 { c NOT b OR a AND d } +foreach {bAsc sql} { + 0 {SELECT rowid FROM xx WHERE xx MATCH $expr} + 1 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid ASC} } { - set res [matchdata $expr] - do_execsql_test 4.$tn.[llength $res] { - SELECT rowid FROM xx WHERE xx match $expr - } $res + foreach {tn expr} { + 0.1 x + + 1 { NEAR(r c) } + 2 { NEAR(r c, 5) } + 3 { NEAR(r c, 3) } + 4 { NEAR(r c, 2) } + 5 { NEAR(r c, 0) } + 6 { NEAR(a b c) } + 7 { NEAR(a b c, 8) } + 8 { x : NEAR(r c) } + 9 { y : NEAR(r c) } + 10 { x : "r c" } + 11 { y : "r c" } + 12 { a AND b } + 13 { a AND b AND c } + 14a { a } + 14b { a OR b } + 15 { a OR b AND c } + 16 { c AND b OR a } + 17 { c AND (b OR a) } + 18 { c NOT (b OR a) } + 19 { c NOT b OR a AND d } + } { + set res [matchdata $expr $bAsc] + do_execsql_test 4.$bAsc.$tn.[llength $res] $sql $res + } } diff --git a/test/fts5ad.test b/test/fts5ad.test index 3898b4c89c..70349388ee 100644 --- a/test/fts5ad.test +++ b/test/fts5ad.test @@ -41,6 +41,17 @@ foreach {tn match res} { } $res } +foreach {tn match res} { + 5 {c*} {1} + 6 {i*} {2 3} + 7 {t*} {1 3} + 8 {r*} {1 3} +} { + do_execsql_test 1.$tn { + SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC + } $res +} + foreach {T create} { 2 { CREATE VIRTUAL TABLE t1 USING fts5(a, b); @@ -178,17 +189,25 @@ foreach {T create} { return $ret } - foreach {tn prefix} { - 1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} - 6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*} - 11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*} - 16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*} - 21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*} - 27 {x*} + foreach {bAsc sql} { + 0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} + 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid ASC} } { - set res [prefix_query $prefix] - set n [llength $res] - do_execsql_test $T.$tn.$n {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} $res + foreach {tn prefix} { + 1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} + 6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*} + 11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*} + 16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*} + 21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*} + 27 {x*} + } { + set res [prefix_query $prefix] + if {$bAsc} { + set res [lsort -integer -increasing $res] + } + set n [llength $res] + do_execsql_test $T.$bAsc.$tn.$n $sql $res + } } } diff --git a/test/permutations.test b/test/permutations.test index d03895e8e6..c75cdbfd43 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -225,7 +225,7 @@ test_suite "fts3" -prefix "" -description { test_suite "fts5" -prefix "" -description { All FTS5 tests. } -files { - fts5aa.test fts5ab.test fts5ea.test + fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ea.test } test_suite "nofaultsim" -prefix "" -description {