From 9f44deed93ea3b94e661d9fcafff41d5d8afa0de Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Dec 2015 19:55:00 +0000 Subject: [PATCH] Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. FossilOrigin-Name: 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d --- ext/fts5/fts5Int.h | 10 ++- ext/fts5/fts5_config.c | 45 ++++++++-- ext/fts5/fts5_expr.c | 87 ++++++++++++++++++- ext/fts5/fts5_hash.c | 20 +++-- ext/fts5/fts5_index.c | 33 +++---- ext/fts5/fts5_main.c | 77 +++++++++++----- ext/fts5/fts5_storage.c | 2 +- ext/fts5/test/fts5_common.tcl | 18 +++- ext/fts5/test/fts5ac.test | 80 ++++++++++++----- ext/fts5/test/fts5ad.test | 6 +- .../{fts5offsets.test => fts5detail.test} | 85 ++++++++++++++---- manifest | 32 +++---- manifest.uuid | 2 +- 13 files changed, 378 insertions(+), 119 deletions(-) rename ext/fts5/test/{fts5offsets.test => fts5detail.test} (76%) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 3016f4be04..a0f69bee21 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -151,7 +151,7 @@ struct Fts5Config { char *zContent; /* content table */ char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ - int bOffsets; /* "offsets=" option value (dflt==1) */ + int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; @@ -180,6 +180,9 @@ struct Fts5Config { #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 @@ -637,6 +640,11 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*); int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int +); + int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); /******************************************* diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index b72fd9b745..fd6da7f11e 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -194,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){ } } + +struct Fts5Enum { + const char *zName; + int eVal; +}; +typedef struct Fts5Enum Fts5Enum; + +static int fts5ConfigSetEnum( + const Fts5Enum *aEnum, + const char *zEnum, + int *peVal +){ + int nEnum = strlen(zEnum); + int i; + int iVal = -1; + + for(i=0; aEnum[i].zName; i++){ + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ + if( iVal>=0 ) return SQLITE_ERROR; + iVal = aEnum[i].eVal; + } + } + + *peVal = iVal; + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; +} + /* ** Parse a "special" CREATE VIRTUAL TABLE directive and update ** configuration object pConfig as appropriate. @@ -344,12 +371,16 @@ static int fts5ConfigParseSpecial( return rc; } - if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){ - if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ - *pzErr = sqlite3_mprintf("malformed offsets=... directive"); - rc = SQLITE_ERROR; - }else{ - pConfig->bOffsets = (zArg[0]=='1'); + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ + const Fts5Enum aDetail[] = { + { "none", FTS5_DETAIL_NONE }, + { "full", FTS5_DETAIL_FULL }, + { "columns", FTS5_DETAIL_COLUMNS }, + { 0, 0 } + }; + + if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){ + *pzErr = sqlite3_mprintf("malformed detail=... directive"); } return rc; } @@ -509,7 +540,7 @@ int sqlite3Fts5ConfigParse( pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; - pRet->bOffsets = 1; + pRet->eDetail = FTS5_DETAIL_FULL; #ifdef SQLITE_DEBUG pRet->bPrefixIndex = 1; #endif diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index b7bc13b289..5a5a0652ee 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1744,6 +1744,15 @@ void sqlite3Fts5ParseSetColset( Fts5ExprNearset *pNear, Fts5Colset *pColset ){ + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + pParse->rc = SQLITE_ERROR; + pParse->zErr = sqlite3_mprintf( + "fts5: column queries are not supported (detail=none)" + ); + sqlite3_free(pColset); + return; + } + if( pNear ){ pNear->pColset = pColset; }else{ @@ -1809,12 +1818,12 @@ Fts5ExprNode *sqlite3Fts5ParseNode( if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ pRet->eType = FTS5_TERM; } - }else if( pParse->pConfig->bOffsets==0 ){ + }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ assert( pParse->rc==SQLITE_OK ); pParse->rc = SQLITE_ERROR; assert( pParse->zErr==0 ); pParse->zErr = sqlite3_mprintf( - "fts5: %s queries are not supported (offsets=0)", + "fts5: %s queries are not supported (detail!=full)", pNear->nPhrase==1 ? "phrase": "NEAR" ); sqlite3_free(pRet); @@ -1932,6 +1941,9 @@ static char *fts5ExprPrintTcl( for(iTerm=0; zRet && iTermnTerm; iTerm++){ char *zTerm = pPhrase->aTerm[iTerm].zTerm; zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + if( pPhrase->aTerm[iTerm].bPrefix ){ + zRet = fts5PrintfAppend(zRet, "*"); + } } if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); @@ -2244,3 +2256,74 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){ } return nRet; } + +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ + int i; + Fts5PoslistWriter *pRet; + for(i=0; inPhrase; i++){ + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; + assert( pExpr->apExprPhrase[i]->nTerm==1 ); + pBuf->n = 0; + } + pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase); + if( pRet ){ + memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase); + } + return pRet; +} + +struct Fts5ExprCtx { + Fts5Expr *pExpr; + Fts5PoslistWriter *aWriter; + i64 iOff; +}; +typedef struct Fts5ExprCtx Fts5ExprCtx; + +static int fts5ExprPopulatePoslistsCb( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ +){ + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; + Fts5Expr *pExpr = p->pExpr; + int i; + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; + for(i=0; inPhrase; i++){ + Fts5ExprTerm *pTerm; + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ + int nTerm = strlen(pTerm->zTerm); + if( (nTerm==nToken || (nTermbPrefix)) + && memcmp(pTerm->zTerm, pToken, nTerm)==0 + ){ + int rc = sqlite3Fts5PoslistWriterAppend( + &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff + ); + if( rc ) return rc; + break; + } + } + } + return SQLITE_OK; +} + +int sqlite3Fts5ExprPopulatePoslists( + Fts5Config *pConfig, + Fts5Expr *pExpr, + Fts5PoslistWriter *aWriter, + int iCol, + const char *z, int n +){ + Fts5ExprCtx sCtx; + sCtx.pExpr = pExpr; + sCtx.aWriter = aWriter; + sCtx.iOff = (((i64)iCol) << 32) - 1; + + return sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb + ); +} + diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index dc5adadc8e..a9f1c13ede 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -26,7 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry; struct Fts5Hash { - int bOffsets; /* Copy of Fts5Config.bOffsets */ + int eDetail; /* Copy of Fts5Config.eDetail */ int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ @@ -91,7 +91,7 @@ int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ int nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; - pNew->bOffsets = pConfig->bOffsets; + pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; @@ -216,7 +216,9 @@ int sqlite3Fts5HashWrite( Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ - int bNew = pHash->bOffsets; /* If non-delete entry should be written */ + int bNew; /* If non-delete entry should be written */ + + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); @@ -253,7 +255,7 @@ int sqlite3Fts5HashWrite( p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; - p->iCol = (pHash->bOffsets-1); + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -290,7 +292,7 @@ int sqlite3Fts5HashWrite( p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); p->iSzPoslist = p->nData; p->nData += 1; - p->iCol = (pHash->bOffsets-1); + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); p->iPos = 0; p->iRowid = iRowid; bNew = 1; @@ -300,14 +302,14 @@ int sqlite3Fts5HashWrite( /* Append a new column value, if necessary */ assert( iCol>=p->iCol ); if( iCol!=p->iCol ){ - if( pHash->bOffsets==0 ){ - bNew = 1; - p->iCol = iPos = iCol; - }else{ + if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); p->iCol = iCol; p->iPos = 0; + }else{ + bNew = 1; + p->iCol = iPos = iCol; } } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index bcd81843b7..836a6d2c99 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -4109,19 +4109,19 @@ static void fts5SegiterPoslist( if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ - if( p->pConfig->bOffsets==0 ){ - PoslistOffsetsCtx sCtx; - memset(&sCtx, 0, sizeof(sCtx)); - sCtx.pBuf = pBuf; - sCtx.pColset = pColset; - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); - }else{ + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ PoslistCallbackCtx sCtx; sCtx.pBuf = pBuf; sCtx.pColset = pColset; sCtx.eState = fts5IndexColsetTest(pColset, 0); assert( sCtx.eState==0 || sCtx.eState==1 ); fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + }else{ + PoslistOffsetsCtx sCtx; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); } } } @@ -4192,7 +4192,7 @@ static int fts5AppendPoslist( assert( fts5MultiIterEof(p, pMulti)==0 ); assert( pSeg->nPos>0 ); if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ - if( p->pConfig->bOffsets + if( p->pConfig->eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf && (pColset==0 || pColset->nCol==1) ){ @@ -4850,9 +4850,11 @@ int sqlite3Fts5IterPoslist( i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; + int eDetail = pIter->pIndex->pConfig->eDetail; + assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - if( pIter->pIndex->pConfig->bOffsets + if( eDetail==FTS5_DETAIL_FULL && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; @@ -4871,7 +4873,9 @@ int sqlite3Fts5IterPoslist( }else{ fts5BufferZero(&pIter->poslist); fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); - *pp = pIter->poslist.p; + if( eDetail==FTS5_DETAIL_FULL ){ + *pp = pIter->poslist.p; + } *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); @@ -5062,17 +5066,15 @@ static int fts5QueryCksum( ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; + Fts5Buffer buf = {0, 0, 0}; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ - i64 dummy; - const u8 *pPos; - int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ @@ -5084,6 +5086,7 @@ static int fts5QueryCksum( } } sqlite3Fts5IterClose(pIdxIter); + fts5BufferFree(&buf); *pCksum = cksum; return rc; diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 1d0b4c219e..51c3f57cff 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -226,6 +226,7 @@ struct Fts5Cursor { #define FTS5CSR_EOF 0x08 #define FTS5CSR_FREE_ZRANK 0x10 #define FTS5CSR_REQUIRE_RESEEK 0x20 +#define FTS5CSR_REQUIRE_POSLIST 0x40 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) #define BitFlagTest(x,y) (((x) & (y))!=0) @@ -313,7 +314,7 @@ static int fts5IsContentless(Fts5Table *pTab){ ** Return true if pTab is an offsetless table. */ static int fts5IsOffsetless(Fts5Table *pTab){ - return pTab->pConfig->bOffsets==0; + return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL; } /* @@ -646,6 +647,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){ FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST ); } @@ -1603,6 +1605,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ return rc; } +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**); + static void *fts5ApiUserData(Fts5Context *pCtx){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; return pCsr->pAux->pUserData; @@ -1652,8 +1656,53 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +static int fts5ApiColumnText( + Fts5Context *pCtx, + int iCol, + const char **pz, + int *pn +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ + *pz = 0; + *pn = 0; + }else{ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + } + } + return rc; +} + static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ int n; + int rc = SQLITE_OK; + + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5PoslistWriter *aWriter; + int i; +assert( pCsr->pSorter==0 ); /* fixme */ + aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); + if( aWriter==0 ) rc = SQLITE_NOMEM; + for(i=0; inCol && rc==SQLITE_OK; i++){ + int n; const char *z; + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprPopulatePoslists( + pConfig, pCsr->pExpr, aWriter, i, z, n + ); + } + } + sqlite3_free(aWriter); + } + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); + } + if( pCsr->pSorter ){ Fts5Sorter *pSorter = pCsr->pSorter; int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); @@ -1756,10 +1805,12 @@ static int fts5ApiInst( ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; +#if 0 }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 2]; *piOff = -1; +#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -1773,27 +1824,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ return fts5CursorRowid((Fts5Cursor*)pCtx); } -static int fts5ApiColumnText( - Fts5Context *pCtx, - int iCol, - const char **pz, - int *pn -){ - int rc = SQLITE_OK; - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ - *pz = 0; - *pn = 0; - }else{ - rc = fts5SeekCursor(pCsr, 0); - if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); - } - } - return rc; -} - static int fts5ColumnSizeCb( void *pContext, /* Pointer to int */ int tflags, @@ -1925,11 +1955,13 @@ static void fts5ApiPhraseNext( if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; +#if 0 }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); *piCol += (iVal-2); *piOff = -1; +#endif }else{ int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); @@ -1982,7 +2014,6 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, }; - /* ** Implementation of API function xQueryPhrase(). */ diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c index 65429edbd9..fe3ce829a1 100644 --- a/ext/fts5/fts5_storage.c +++ b/ext/fts5/fts5_storage.c @@ -916,7 +916,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ if( pConfig->abUnindexed[i] ) continue; ctx.iCol = i; ctx.szCol = 0; - if( pConfig->bOffsets==0 ){ + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } if( rc==SQLITE_OK ){ diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl index 74e84ab234..30b191e299 100644 --- a/ext/fts5/test/fts5_common.tcl +++ b/ext/fts5/test/fts5_common.tcl @@ -239,7 +239,21 @@ proc nearset {aCol args} { set iFirst [expr $iFL - $O(-near) - [llength $p]] for {set i $iFirst} {$i <= $iFL} {incr i} { - if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i } + set lCand [lrange $col $i [expr $i+$nPm1]] + + set bMatch 1 + foreach tok $p term $lCand { + if {[string match $tok $term]==0} { + #puts "$tok $term failed" + set bMatch 0 + } + } + if {$bMatch} { + #puts "match at $i" + lappend B($iPhrase) $i + } + + #if {$lCand == $p} { lappend B($iPhrase) $i } } if {[llength $B($iPhrase)] == 0} break } @@ -265,7 +279,7 @@ proc nearset {aCol args} { incr counter } - #puts $res + #puts "$aCol -> $res" sort_poslist $res } diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 0de4848145..cc10cece09 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -130,7 +130,7 @@ set data { # # CREATE VIRTUAL TABLE xy USING fts5(x, y); # -# Assuming the table contains the same records as stored int the global +# Assuming the table contains the same records as stored in the global # $::data array (see above), this function returns a list containing one # element for each match in the dataset. The elements are themselves lists # formatted as follows: @@ -175,30 +175,22 @@ proc matchdata {bPos expr {bAsc 1}} { # End of test code #------------------------------------------------------------------------- -proc fts5_test_poslist {cmd} { - set res [list] - for {set i 0} {$i < [$cmd xInstCount]} {incr i} { - lappend res [string map {{ } .} [$cmd xInst $i]] - } - set res -} - foreach {tn2 sql} { 1 {} 2 {BEGIN} } { reset_db - sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist + fts5_aux_test_functions db - do_execsql_test 1.0 { + do_execsql_test 1.$tn2.0 { CREATE VIRTUAL TABLE xx USING fts5(x,y); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } execsql $sql - do_test $tn2.1.1 { + do_test 1.$tn2.1.1 { foreach {id x y} $data { execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } } @@ -224,7 +216,7 @@ foreach {tn2 sql} { set expr "\"$phrase\"" set res [matchdata 1 $expr] - do_execsql_test $tn2.1.2.$tn.[llength $res] { + do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -246,7 +238,7 @@ foreach {tn2 sql} { 3.1 { a AND b AND c } } { set res [matchdata 1 $expr] - do_execsql_test $tn2.2.$tn.[llength $res] { + do_execsql_test 1.$tn2.2.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -275,7 +267,7 @@ foreach {tn2 sql} { 4.4 {{"y" y}:b} } { set res [matchdata 1 $expr] - do_execsql_test $tn2.3.$tn.[llength $res] { + do_execsql_test 1.$tn2.3.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -296,7 +288,7 @@ foreach {tn2 sql} { 9 { y : NEAR(r c) } } { set res [matchdata 1 $expr] - do_execsql_test $tn2.4.1.$tn.[llength $res] { + do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr } $res } @@ -307,14 +299,14 @@ foreach {tn2 sql} { foreach {tn expr tclexpr} { 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} } { - do_execsql_test $tn2.5.$tn { + do_execsql_test 1.$tn2.5.$tn { SELECT fts5_expr_tcl($expr, 'N $x') } [list $tclexpr] } #------------------------------------------------------------------------- # - do_execsql_test $tn2.6.integrity { + do_execsql_test 1.$tn2.6.integrity { INSERT INTO xx(xx) VALUES('integrity-check'); } #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} @@ -346,14 +338,62 @@ foreach {tn2 sql} { 19 { c NOT b OR a AND d } } { set res [matchdata 0 $expr $bAsc] - do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res + do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res } } } -do_execsql_test 3.1 { +do_execsql_test 2.1 { SELECT fts5_expr_tcl('a AND b'); } {{AND [nearset -- {a}] [nearset -- {b}]}} +# Some tests for detail=col tables and detail=none. +# +foreach {tn2 sql} { + 1 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); + } + 2 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); + BEGIN; + } + 3 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); + BEGIN; + } + 4 { + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); + } +} { + reset_db + fts5_aux_test_functions db + + execsql $sql + + do_execsql_test 3.$tn2.0 { + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); + } + + + do_test 3.$tn2.1.1 { + foreach {id x y} $data { + execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } + } + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } + } {} + + foreach {tn q} { + 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" + 11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d" + } { + set res [matchdata 1 $q] + + do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q + } $res + } + +} + finish_test diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index a9fc568133..cc86a6b2eb 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -75,17 +75,17 @@ foreach {T create} { } 6 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 7 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 8 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); BEGIN; } diff --git a/ext/fts5/test/fts5offsets.test b/ext/fts5/test/fts5detail.test similarity index 76% rename from ext/fts5/test/fts5offsets.test rename to ext/fts5/test/fts5detail.test index 2f2674fabe..59f4b0e3c3 100644 --- a/ext/fts5/test/fts5offsets.test +++ b/ext/fts5/test/fts5detail.test @@ -13,7 +13,7 @@ # source [file join [file dirname [info script]] fts5_common.tcl] -set testprefix fts5offsets +set testprefix fts5detail # If SQLITE_ENABLE_FTS5 is not defined, omit this file. ifcapable !fts5 { @@ -21,11 +21,13 @@ ifcapable !fts5 { return } +fts5_aux_test_functions db + #-------------------------------------------------------------------------- # Simple tests. # do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0); + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 @@ -58,17 +60,18 @@ foreach {tn match res} { do_catchsql_test 1.3.1 { SELECT rowid FROM t1('h + d'); -} {1 {fts5: phrase queries are not supported (offsets=0)}} +} {1 {fts5: phrase queries are not supported (detail!=full)}} do_catchsql_test 1.3.2 { SELECT rowid FROM t1('NEAR(h d)'); -} {1 {fts5: NEAR queries are not supported (offsets=0)}} +} {1 {fts5: NEAR queries are not supported (detail!=full)}} + #------------------------------------------------------------------------- -# integrity-check with both offsets= and prefix= options. +# integrity-check with both detail= and prefix= options. # do_execsql_test 2.0 { - CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1"); + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); INSERT INTO t2(a) VALUES('aa ab'); } @@ -78,9 +81,17 @@ do_execsql_test 2.1 { INSERT INTO t2(t2) VALUES('integrity-check'); } +do_execsql_test 2.2 { + SELECT fts5_test_poslist(t2) FROM t2('aa'); +} {0.0.0} + +set ::pc 0 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] +#exit + #------------------------------------------------------------------------- # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs -# work with offsets=0 tables. +# work with detail=col tables. # set data { 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} @@ -124,9 +135,42 @@ set data { 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} } + +set data { + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} +} + +proc matchdata {expr {bAsc 1}} { + + set tclexpr [db one { + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') + }] + set res [list] + + #puts "$expr -> $tclexpr" + foreach {id x y z} $::data { + set cols [list $x $y $z] + set ::pc 0 + #set hits [lsort -command instcompare [eval $tclexpr]] + set hits [eval $tclexpr] + if {[llength $hits]>0} { + lappend res [list $id $hits] + } + } + + if {$bAsc} { + set res [lsort -integer -increasing -index 0 $res] + } else { + set res [lsort -integer -decreasing -index 0 $res] + } + + return [concat {*}$res] +} + foreach {tn tbl} { - 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) } + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } } { +#break reset_db fts5_aux_test_functions db execsql $tbl @@ -139,17 +183,7 @@ foreach {tn tbl} { 9 a* 10 b* 11 c* } { - set res [list] - foreach {id x y z} $data { - if {[lsearch [concat $x $y $z] $expr]>=0} { - lappend res $id - set inst [list] - if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 } - if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 } - if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 } - lappend res $inst - } - } + set res [matchdata $expr] do_execsql_test 3.$tn.$tn2.1 { SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) @@ -159,8 +193,21 @@ foreach {tn tbl} { SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) } $res } +} +#------------------------------------------------------------------------- +# Simple tests for detail=none tables. +# +breakpoint +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); } +do_catchsql_test 4.1 { + SELECT * FROM t4('a:a') +} {1 {fts5: column queries are not supported (detail=none)}} + finish_test diff --git a/manifest b/manifest index d02d515584..050615992a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sso\sthat\sfts5\sAPI\sfunctions\sxInst,\sxPhraseFirst\sand\sxPhraseNext\swork\swith\sthe\soffsets=0\soption. -D 2015-12-22T18:54:16.763 +C Change\sthe\sname\sof\sthe\soffsets=0\soption\sto\s"detail=column".\sHave\sthe\sxInst,\sxPhraseFirst\sand\sother\sAPI\sfunctions\swork\sby\sparsing\sthe\soriginal\stext\sfor\sdetail=column\stables. +D 2015-12-28T19:55:00.739 F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751 @@ -97,15 +97,15 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 7d6785c75afe23239dad9cbaffc6fc55803feb4b -F ext/fts5/fts5Int.h e71739ed0b816758f1f77baaccca1b4a9064c4dc +F ext/fts5/fts5Int.h 7328cfa7b0151e34bb3715d404d732c6d49d5361 F ext/fts5/fts5_aux.c 1f384972d606375b8fa078319f25ab4b5feb1b35 F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1 -F ext/fts5/fts5_config.c ba5248a05c28ec6a6fdf2599a86e9fd67e5c61e2 -F ext/fts5/fts5_expr.c 3b2c7ac54e6c03e732751a6a4bf9ced8f408e2a2 -F ext/fts5/fts5_hash.c d4a6b52faca0134cc7bcc880f03a257a0dec2636 -F ext/fts5/fts5_index.c 1d1939afbc434907993d7e9a0c631be630d5b0dc -F ext/fts5/fts5_main.c b7d6540c63f347908d8ee937ca8a7987130c84b3 -F ext/fts5/fts5_storage.c 14e0bb6549a66da54adf4fa1564edbf24647cb22 +F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6 +F ext/fts5/fts5_expr.c c9522ae99f862ddb318ea69998a02147d2a00c8d +F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40 +F ext/fts5/fts5_index.c 24e81d8f2ce92f50eb19c15bffd5ada4ec91d8de +F ext/fts5/fts5_main.c a71db65929e150031c5a195e3c782e773deb55a6 +F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179 F ext/fts5/fts5_tcl.c c3cfff5f727b3d02f73c5717413d637b18b960f2 F ext/fts5/fts5_test_mi.c c9c8cf455c49ded156a234e0de1c8ba9be85e5c3 F ext/fts5/fts5_tokenize.c 618efe033bceb80c521b1e9ddfd9fee85fb5946e @@ -114,11 +114,11 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1 F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2 F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba -F ext/fts5/test/fts5_common.tcl 4a23a75f31dbb96c03c2840fae1bf85b4f64f062 +F ext/fts5/test/fts5_common.tcl 27f941596e241e25b4bd64c2446ae91dd4cb138c F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad -F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c -F ext/fts5/test/fts5ad.test 21d87b12c7ec83b4ec48816d24503443dffb10a1 +F ext/fts5/test/fts5ac.test 5dea9e119ed39e0923b423dc2b3eaaef2ec3abd3 +F ext/fts5/test/fts5ad.test 0fd1a5bcb5dcc76a639bd26f27e2641034a508a3 F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 @@ -139,6 +139,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599 +F ext/fts5/test/fts5detail.test 71babe633c68c6c5153b2fe1be81a97e953ca485 w ext/fts5/test/fts5offsets.test F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 @@ -156,7 +157,6 @@ F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65 F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2 F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367 F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc -F ext/fts5/test/fts5offsets.test 7d32df518f6513131c14ce985006c3ac5873e79f F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5 F ext/fts5/test/fts5phrase.test f6d1d464da5beb25dc56277aa4f1d6102f0d9a2f @@ -1406,7 +1406,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 609a0bc7f34e6dae74ce756aff920f3df78fe828 -R 3d59533357e77f6a30434e9a4ea67f85 +P 69bffc1632c8a8f3bfe5bf92607e64fed982e48c +R bce3f221a6a9e036b503a3a5522049ea U dan -Z ae770209a83bcb452cb39fda88b8ef75 +Z 73bb0a8c75a5e4585189e64532bdf9a9 diff --git a/manifest.uuid b/manifest.uuid index 07c12c4bf0..a1ac4a70e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69bffc1632c8a8f3bfe5bf92607e64fed982e48c \ No newline at end of file +228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d \ No newline at end of file -- 2.47.2