** Close the cursor. For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
sqlite3_finalize(pCsr->pStmt);
sqlite3Fts3ExprFree(pCsr->pExpr);
sqlite3Fts3FreeDeferredTokens(pCsr);
}
+static int fts3ScanInteriorNode(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piFirst, /* OUT: Selected child node */
+ sqlite3_int64 *piLast /* OUT: Selected child node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+
+ int isFirstTerm = 1; /* True when processing first term on page */
+ int dummy;
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
+ int nBlock; /* Size of child node in bytes */
+
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &dummy);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+
+ while( zCsr<zEnd && (piFirst || piLast) ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ sqlite3_free(zBuffer);
+ return SQLITE_NOMEM;
+ }
+ zBuffer = zNew;
+ }
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){
+ *piFirst = iChild;
+ piFirst = 0;
+ }
+
+ if( piLast && cmp<0 ){
+ *piLast = iChild;
+ piLast = 0;
+ }
+
+ iChild++;
+ };
+
+ if( piFirst ) *piFirst = iChild;
+ if( piLast ) *piLast = iChild;
+
+ sqlite3_free(zBuffer);
+ return rc;
+}
/*
int nTerm, /* Size of term zTerm in bytes */
const char *zNode, /* Buffer containing segment interior node */
int nNode, /* Size of buffer at zNode */
- sqlite3_int64 *piLeaf /* Selected leaf node */
+ sqlite3_int64 *piLeaf, /* Selected leaf node */
+ sqlite3_int64 *piLeaf2 /* Selected leaf node */
){
- int rc = SQLITE_OK; /* Return code */
- const char *zCsr = zNode; /* Cursor to iterate through node */
- const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
- char *zBuffer = 0; /* Buffer to load terms into */
- int nAlloc = 0; /* Size of allocated buffer */
-
- while( 1 ){
- int isFirstTerm = 1; /* True when processing first term on page */
- int iHeight; /* Height of this node in tree */
- sqlite3_int64 iChild; /* Block id of child node to descend to */
- int nBlock; /* Size of child node in bytes */
+ int rc; /* Return code */
+ int iHeight; /* Height of this node in tree */
- zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
- zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
-
- while( zCsr<zEnd ){
- int cmp; /* memcmp() result */
- int nSuffix; /* Size of term suffix */
- int nPrefix = 0; /* Size of term prefix */
- int nBuffer; /* Total term size */
+ sqlite3Fts3GetVarint32(zNode, &iHeight);
+ rc = fts3ScanInteriorNode(p, zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2);
- /* Load the next term on the node into zBuffer */
- if( !isFirstTerm ){
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
- }
- isFirstTerm = 0;
- zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
- if( nPrefix+nSuffix>nAlloc ){
- char *zNew;
- nAlloc = (nPrefix+nSuffix) * 2;
- zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
- if( !zNew ){
- sqlite3_free(zBuffer);
- return SQLITE_NOMEM;
- }
- zBuffer = zNew;
- }
- memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
- nBuffer = nPrefix + nSuffix;
- zCsr += nSuffix;
-
- /* Compare the term we are searching for with the term just loaded from
- ** the interior node. If the specified term is greater than or equal
- ** to the term from the interior node, then all terms on the sub-tree
- ** headed by node iChild are smaller than zTerm. No need to search
- ** iChild.
- **
- ** If the interior node term is larger than the specified term, then
- ** the tree headed by iChild may contain the specified term.
- */
- cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
- if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
- iChild++;
- };
+ if( rc==SQLITE_OK && iHeight>1 ){
+ const char *zBlob;
+ int nBlob;
- /* If (iHeight==1), the children of this interior node are leaves. The
- ** specified term may be present on leaf node iChild.
- */
- if( iHeight==1 ){
- *piLeaf = iChild;
- break;
+ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){
+ rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0);
+ }
+ piLeaf = 0;
}
- /* Descend to interior node iChild. */
- rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
- if( rc!=SQLITE_OK ) break;
- zEnd = &zCsr[nBlock];
+ rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob);
+ if( rc==SQLITE_OK ){
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2);
+ }
}
- sqlite3_free(zBuffer);
+
return rc;
}
char *a1, /* Buffer containing first doclist */
int n1, /* Size of buffer a1 */
char *a2, /* Buffer containing second doclist */
- int n2 /* Size of buffer a2 */
+ int n2, /* Size of buffer a2 */
+ int *pnDoc /* OUT: Number of docids in output */
){
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
char *p2 = a2;
char *pEnd1 = &a1[n1];
char *pEnd2 = &a2[n2];
+ int nDoc = 0;
assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
|| mergetype==MERGE_AND || mergetype==MERGE_NOT
fts3PutDeltaVarint(&p, &iPrev, i1);
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ nDoc++;
}else if( i1<i2 ){
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
}else{
if( 0==fts3PoslistPhraseMerge(ppPos, nParam1, 0, 1, &p1, &p2) ){
p = pSave;
iPrev = iPrevSave;
+ }else{
+ nDoc++;
}
fts3GetDeltaVarint2(&p1, pEnd1, &i1);
fts3GetDeltaVarint2(&p2, pEnd2, &i2);
}
}
+ if( pnDoc ) *pnDoc = nDoc;
*pnBuffer = (int)(p-aBuffer);
return SQLITE_OK;
}
return SQLITE_NOMEM;
}
fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut
+ aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0
);
sqlite3_free(pTS->aaOutput[i]);
sqlite3_free(aOut);
}
return SQLITE_NOMEM;
}
- fts3DoclistMerge(mergetype, 0, 0,
- aNew, &nNew, pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge
+ fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew,
+ pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0
);
if( iOut>0 ) sqlite3_free(aMerge);
rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
}else{
int rc2; /* Return value of sqlite3Fts3ReadBlock() */
- sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */
- rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
+ sqlite3_int64 i1; /* First leaf that may contain zTerm */
+ sqlite3_int64 i2; /* Last leaf that may contain zTerm */
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0));
+ if( isPrefix==0 ) i2 = i1;
if( rc==SQLITE_OK ){
- sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
}
return rc;
}
+/*
+** This function counts the total number of docids in the doclist stored
+** in buffer aList[], size nList bytes.
+**
+** If the isPoslist argument is true, then it is assumed that the doclist
+** contains a position-list following each docid. Otherwise, it is assumed
+** that the doclist is simply a list of docids stored as delta encoded
+** varints.
+*/
static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){
int nDoc = 0; /* Return value */
+
if( aList ){
char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */
char *p = aList; /* Cursor */
- sqlite3_int64 dummy; /* For Fts3GetVarint() */
-
- while( p<aEnd ){
- nDoc++;
- p += sqlite3Fts3GetVarint(p, &dummy);
- if( isPoslist ) fts3PoslistCopy(0, &p);
+ if( !isPoslist ){
+ /* The number of docids in the list is the same as the number of
+ ** varints. In FTS3 a varint consists of a single byte with the 0x80
+ ** bit cleared and zero or more bytes with the 0x80 bit set. So to
+ ** count the varints in the buffer, just count the number of bytes
+ ** with the 0x80 bit clear. */
+ while( p<aEnd ) nDoc += (((*p++)&0x80)==0);
+ }else{
+ while( p<aEnd ){
+ nDoc++;
+ while( (*p++)&0x80 ); /* Skip docid varint */
+ fts3PoslistCopy(0, &p); /* Skip over position list */
+ }
}
}
if( ii==0 ){
pOut = pList;
nOut = nList;
+ if( pCsr->doDeferred==0 && pPhrase->nToken>1 ){
+ nDoc = fts3DoclistCountDocids(1, pOut, nOut);
+ }
}else{
/* Merge the new term list and the current output. */
char *aLeft, *aRight;
nDist = iPrevTok-iTok;
}
pOut = aRight;
-
- fts3DoclistMerge(mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight);
+ fts3DoclistMerge(
+ mt, nDist, 0, pOut, &nOut, aLeft, nLeft, aRight, nRight, &nDoc
+ );
sqlite3_free(aLeft);
}
assert( nOut==0 || pOut!=0 );
iPrevTok = iTok;
- nDoc = fts3DoclistCountDocids(ii<(pPhrase->nToken-1), pOut, nOut);
}
if( rc==SQLITE_OK ){
rc = SQLITE_NOMEM;
}else{
rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft,
- aOut, pnOut, aLeft, nLeft, aRight, nRight
+ aOut, pnOut, aLeft, nLeft, aRight, nRight, 0
);
if( rc!=SQLITE_OK ){
sqlite3_free(aOut);
if( ii==0 ){
aRet = aNew;
nRet = nNew;
+ if( nExpr>1 ){
+ nDoc = fts3DoclistCountDocids(0, aRet, nRet);
+ }
}else{
fts3DoclistMerge(
- MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew
+ MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc
);
sqlite3_free(aNew);
}
- nDoc = fts3DoclistCountDocids(0, aRet, nRet);
}
}
}
*/
char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aBuffer;
sqlite3_free(aLeft);
default: {
assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
- aLeft, nLeft, aRight, nRight
+ aLeft, nLeft, aRight, nRight, 0
);
*paOut = aLeft;
break;
return rc;
}
-
/*
** This is the xFilter interface for the virtual table. See
** the virtual table xFilter method documentation for additional
assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
assert( nVal==0 || nVal==1 );
assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+ assert( p->pSegments==0 );
/* In case the cursor has been used before, clear it now. */
sqlite3_finalize(pCsr->pStmt);
if( rc!=SQLITE_OK ) return rc;
rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0);
+ sqlite3Fts3SegmentsClose(p);
if( rc!=SQLITE_OK ) return rc;
pCsr->pNextId = pCsr->aDoclist;
pCsr->iPrevId = 0;
** hash-table to the database.
*/
static int fts3SyncMethod(sqlite3_vtab *pVtab){
- return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+ sqlite3Fts3SegmentsClose((Fts3Table *)pVtab);
+ return rc;
}
/*
/* xDisconnect */ fts3DisconnectMethod,
/* xDestroy */ fts3DestroyMethod,
/* xOpen */ fts3OpenMethod,
- /* xClose */ fulltextClose,
+ /* xClose */ fts3CloseMethod,
/* xFilter */ fts3FilterMethod,
/* xNext */ fts3NextMethod,
/* xEof */ fts3EofMethod,
u8 bHasContent; /* True if %_content table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
+ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
+
/* The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
** pending data, including hash table overhead, but not malloc overhead.
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
+void sqlite3Fts3SegmentsClose(Fts3Table *);
+
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
}
if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
if( pnToken ) *pnToken = sCtx.nToken;
+ sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab);
return rc;
}
** The %_segments table is declared as follows:
**
** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB)
-**
-** This function opens a read-only blob handle on the "block" column of
-** row iSegment of the %_segments table associated with FTS3 table p.
-**
-** If all goes well, SQLITE_OK is returned and *ppBlob set to the
-** read-only blob handle. It is the responsibility of the caller to call
-** sqlite3_blob_close() on the blob handle. Or, if an error occurs, an
-** SQLite error code is returned and *ppBlob is either not modified or
-** set to 0.
-*/
-static int fts3OpenSegmentsBlob(
- Fts3Table *p, /* FTS3 table handle */
- sqlite3_int64 iSegment, /* Rowid in %_segments table */
- sqlite3_blob **ppBlob /* OUT: Read-only blob handle */
+*/
+static int fts3SegmentsBlob(
+ Fts3Table *p,
+ sqlite3_int64 iSegment,
+ char **paBlob,
+ int *pnBlob
){
- if( 0==p->zSegmentsTbl
- && 0==(p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName))
- ) {
- return SQLITE_NOMEM;
+ int rc;
+
+ if( p->pSegments ){
+ rc = sqlite3_blob_reopen(p->pSegments, iSegment);
+ }else{
+ if( 0==p->zSegmentsTbl ){
+ p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName);
+ if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM;
+ }
+ rc = sqlite3_blob_open(
+ p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, &p->pSegments
+ );
}
- return sqlite3_blob_open(
- p->db, p->zDb, p->zSegmentsTbl, "block", iSegment, 0, ppBlob
- );
+
+ if( rc==SQLITE_OK ){
+ int nByte = sqlite3_blob_bytes(p->pSegments);
+ if( paBlob ){
+ char *aByte = sqlite3_malloc(nByte);
+ if( !aByte ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0);
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aByte);
+ aByte = 0;
+ }
+ }
+ *paBlob = aByte;
+ }
+ *pnBlob = nByte;
+ }
+
+ return rc;
+}
+
+void sqlite3Fts3SegmentsClose(Fts3Table *p){
+ sqlite3_blob_close(p->pSegments);
+ p->pSegments = 0;
}
return SQLITE_OK;
}
- rc = fts3OpenSegmentsBlob(p, ++pReader->iCurrentBlock, &pBlob);
- if( rc==SQLITE_OK ){
- pReader->nNode = sqlite3_blob_bytes(pBlob);
- pReader->aNode = (char *)sqlite3_malloc(pReader->nNode);
- if( pReader->aNode ){
- rc = sqlite3_blob_read(pBlob, pReader->aNode, pReader->nNode, 0);
- }else{
- rc = SQLITE_NOMEM;
- }
- sqlite3_blob_close(pBlob);
- }
+ rc = fts3SegmentsBlob(
+ p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode
+ );
if( rc!=SQLITE_OK ){
return rc;
&& !fts3SegReaderIsPending(pReader)
&& !fts3SegReaderIsRootOnly(pReader)
){
- sqlite3_blob *pBlob = 0;
+ int nBlob = 0;
+ sqlite3_int64 iBlock;
if( pCsr->nRowAvg==0 ){
/* The average document size, which is required to calculate the cost
if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc;
}
- rc = fts3OpenSegmentsBlob(p, pReader->iStartBlock, &pBlob);
- if( rc==SQLITE_OK ){
- /* Assume that a blob flows over onto overflow pages if it is larger
- ** than (pgsz-35) bytes in size (the file-format documentation
- ** confirms this).
- */
- int nBlob = sqlite3_blob_bytes(pBlob);
+ /* Assume that a blob flows over onto overflow pages if it is larger
+ ** than (pgsz-35) bytes in size (the file-format documentation
+ ** confirms this).
+ */
+ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){
+ rc = fts3SegmentsBlob(p, iBlock, 0, &nBlob);
+ if( rc!=SQLITE_OK ) break;
if( (nBlob+35)>pgsz ){
int nOvfl = (nBlob + 34)/pgsz;
nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg);
}
}
- assert( rc==SQLITE_OK || pBlob==0 );
- sqlite3_blob_close(pBlob);
}
*pnCost += nCost;
Fts3SegReader *pReader; /* Newly allocated SegReader object */
int nExtra = 0; /* Bytes to allocate segment root node */
+ assert( iStartLeaf<=iEndLeaf );
if( iStartLeaf==0 ){
nExtra = nRoot;
}
rc = SQLITE_ERROR;
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
u32 *aSzDel; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
+ assert( p->pSegments==0 );
/* Allocate space to hold the change in document sizes */
aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 );
}
sqlite3_free(aSzIns);
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
}
}
+ sqlite3Fts3SegmentsClose(p);
return rc;
}
set VOCAB_SIZE 2000
set DOC_SIZE 100
-set NUM_INSERTS 1000
+set NUM_INSERTS 100000
set NUM_SELECTS 1000
# Force everything in this script to be deterministic.
sql "CREATE VIRTUAL TABLE t1 USING fts4;"
for {set i 0} {$i < $nInsert} {incr i} {
set doc [select_doc $::DOC_SIZE]
- #sql "INSERT INTO t1 VALUES('$doc');"
- sql "\"$doc\""
+ sql "INSERT INTO t1 VALUES('$doc');"
}
}
-C Experimental\schanges\sto\sfts4\sto\stry\sto\sselectively\savoid\sloading\svery\slarge\sdoclists.
-D 2010-10-19T14:08:00
+C Updates\sto\sFTS4\sto\simprove\sperformance\sand\smake\smore\saccurate\scost\sestimates\sfor\sprefix\sterms.
+D 2010-10-20T18:56:04
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b01fdfcfecf8a0716c29867a67959f6148b79961
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 9d4ccf3b7bbfbeeef03dba91377c4d72b757dcb9
+F ext/fts3/fts3.c ce7bcd1f42e74912149fe6201fc63a6ac0db42a8
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h a640e4fbdb2fcab1457f87993ca3f4ceaa31e776
+F ext/fts3/fts3Int.h 9fbe422f7d0e005371702acaa3cd44283a67c389
F ext/fts3/fts3_expr.c a5aee50edde20e5c9116199bd58be869a3a22c9f
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba
-F ext/fts3/fts3_snippet.c 474c11e718610cade73e6009f75ffc173d4c42c5
+F ext/fts3/fts3_snippet.c ca60a2a47de5e7abb22a804ccd1a743f81c2fe3e
F ext/fts3/fts3_tokenizer.c b4f2d01c24573852755bc92864816785dae39318
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
-F ext/fts3/fts3_write.c 29b63a98de55d4eb34b7fc6fd90b3224d6cdc7ff
-F ext/fts3/fts3speed.tcl 71b9cdc8f499822124a9eef42003e31a88f26f16
+F ext/fts3/fts3_write.c be47d30cf80bc91e050ece18e2de7e207432be1a
+F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
F ext/icu/icu.c 850e9a36567bbcce6bd85a4b68243cad8e3c2de2
F src/backup.c d5b0137bc20327af08c14772227cc35134839c30
F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff
-F src/btree.c 8a1b0267a4f1914aedbaa93d3fcf4f2e42141ea8
+F src/btree.c 3edab36d03d86c200cb9551467410f975d510aa9
F src/btree.h 2d1a83ad509047e8cc314fda7e054f99ff52414d
F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0
F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d
F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
F src/select.c 6a5c72fb0e8dc7f6133f5a9d7a747130ef0a00ea
F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056
-F src/sqlite.h.in 13f219b9ab78f22603019fd193f09d5c8913795a
+F src/sqlite.h.in 460599b35c035deb339d1c9933089ef32187ecc6
F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754
F src/sqliteInt.h c63b0340dfdfde18ff255ddccf004edd2d073288
F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44
F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
F src/tclsqlite.c e91019fb6787166abca23a81b16c07fecc2ed751
-F src/test1.c cbedc6ea7905b1361db054fbf7fcd0dafb6d844e
+F src/test1.c f6e39615c8315e03798217a360810e4c59595627
F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31
F src/test3.c 056093cfef69ff4227a6bdb9108564dc7f45e4bc
F src/test4.c 0528360b5025688002a5feb6be906ddce52eaaee
F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4
F src/vdbeapi.c 5368714fa750270cf6430160287c21adff44582d
F src/vdbeaux.c de0b06b11a25293e820a49159eca9f1c51a64716
-F src/vdbeblob.c 258a6010ba7a82b72b327fb24c55790655689256
+F src/vdbeblob.c c8cbe6ce28cc8bf806ea0818b5167dd9a27c48a3
F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c 6c90e3e65b2f026fc54703a8f3c917155f419d87
F test/fts3ae.test ce32a13b34b0260928e4213b4481acf801533bda
F test/fts3af.test d394978c534eabf22dd0837e718b913fd66b499c
F test/fts3ag.test 0b7d303f61ae5d620c4efb5e825713ea34ff9441
-F test/fts3ah.test 3c5a1bd49979d7b5b5ed9fdbcdd14a7bfe5a5ff9
+F test/fts3ah.test dc9f66c32c296f1bc8bcc4535126bddfeca62894
F test/fts3ai.test d29cee6ed653e30de478066881cec8aa766531b2
F test/fts3aj.test 584facbc9ac4381a7ec624bfde677340ffc2a5a4
F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
F test/fts3cov.test 6f1ff88ff6b5abcfff6979098cb9d0c68a69202e
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
-F test/fts3defer.test a9f81bba6e1132dd6a2ad3cf11e4628733975c8c
+F test/fts3defer.test cf66bf69afcc2fb8373d3aed31c55399409e83f2
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P c0ee614fd988f445c4884a37f494479bdd669185
-R 0c4aebbf44d624104504e37bec992917
-T *bgcolor * #c0ffc0
-T *branch * experimental
-T *sym-experimental *
-T -sym-trunk *
+P 5ae0ba447a561e3b6637b52f9b83a9fc683d2572
+R acd75bd5094c2beea7f56f4633b47ea2
U dan
-Z c2b99a58ccad27405ff8b0fcedef5c33
+Z 910cba31e6572b93e93d39c91f91b9da
-5ae0ba447a561e3b6637b52f9b83a9fc683d2572
\ No newline at end of file
+d0a450ce78e99f55c862f26f9332786660007a0a
\ No newline at end of file
void sqlite3BtreeCacheOverflow(BtCursor *pCur){
assert( cursorHoldsMutex(pCur) );
assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) );
- assert(!pCur->isIncrblobHandle);
- assert(!pCur->aOverflow);
+ invalidateOverflowCache(pCur);
pCur->isIncrblobHandle = 1;
}
#endif
sqlite3_blob **ppBlob
);
+/*
+** CAPI3REF: Move a BLOB Handle
+*/
+SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64);
+
/*
** CAPI3REF: Close A BLOB Handle
**
return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
}
+
+static int test_blob_reopen(
+ ClientData clientData, /* Not used */
+ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */
+ int objc, /* Number of arguments */
+ Tcl_Obj *CONST objv[] /* Command arguments */
+){
+ Tcl_WideInt iRowid;
+ Tcl_Channel channel;
+ ClientData instanceData;
+ sqlite3_blob *pBlob;
+ int notUsed;
+ int rc;
+
+ unsigned char *zBuf;
+ int nBuf;
+
+ if( objc!=3 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL ROWID");
+ return TCL_ERROR;
+ }
+
+ channel = Tcl_GetChannel(interp, Tcl_GetString(objv[1]), ¬Used);
+ if( !channel || TCL_OK!=Tcl_GetWideIntFromObj(interp, objv[2], &iRowid) ){
+ return TCL_ERROR;
+ }
+
+ if( TCL_OK!=(rc = Tcl_Flush(channel)) ){
+ return rc;
+ }
+ if( TCL_OK!=(rc = Tcl_Seek(channel, 0, SEEK_SET)) ){
+ return rc;
+ }
+
+ instanceData = Tcl_GetChannelInstanceData(channel);
+ pBlob = *((sqlite3_blob **)instanceData);
+
+ rc = sqlite3_blob_reopen(pBlob, iRowid);
+ if( rc!=SQLITE_OK ){
+ Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE);
+ }
+
+ return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
+}
+
#endif
/*
#ifndef SQLITE_OMIT_INCRBLOB
{ "sqlite3_blob_read", test_blob_read, 0 },
{ "sqlite3_blob_write", test_blob_write, 0 },
+ { "sqlite3_blob_reopen", test_blob_reopen, 0 },
#endif
{ "pcache_stats", test_pcache_stats, 0 },
#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
int flags; /* Copy of "flags" passed to sqlite3_blob_open() */
int nByte; /* Size of open blob, in bytes */
int iOffset; /* Byte offset of blob in cursor data */
+ int iCol; /* Table column this handle is open on */
BtCursor *pCsr; /* Cursor pointing at blob row */
sqlite3_stmt *pStmt; /* Statement holding cursor open */
sqlite3 *db; /* The associated database */
};
+
+static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){
+ int rc; /* Error code */
+ char *zErr = 0; /* Error message */
+ Vdbe *v = (Vdbe *)p->pStmt;
+
+ v->aVar[0].u.i = iRow;
+ rc = sqlite3_step(p->pStmt);
+
+ if( rc==SQLITE_ROW ){
+ Vdbe *v = (Vdbe *)p->pStmt;
+ u32 type = v->apCsr[0]->aType[p->iCol];
+ if( type<12 ){
+ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s",
+ type==0?"null": type==7?"real": "integer"
+ );
+ rc = SQLITE_ERROR;
+ sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ }else{
+ p->iOffset = v->apCsr[0]->aOffset[p->iCol];
+ p->nByte = sqlite3VdbeSerialTypeLen(type);
+ p->pCsr = v->apCsr[0]->pCursor;
+ sqlite3BtreeEnterCursor(p->pCsr);
+ sqlite3BtreeCacheOverflow(p->pCsr);
+ sqlite3BtreeLeaveCursor(p->pCsr);
+ }
+ }
+
+ if( rc==SQLITE_ROW ){
+ rc = SQLITE_OK;
+ }else if( p->pStmt ){
+ rc = sqlite3_finalize(p->pStmt);
+ p->pStmt = 0;
+ if( rc==SQLITE_OK ){
+ zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow);
+ rc = SQLITE_ERROR;
+ }else{
+ zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db));
+ }
+ }
+
+ assert( rc!=SQLITE_OK || zErr==0 );
+ assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE );
+
+ *pzErr = zErr;
+ return rc;
+}
+
/*
** Open a blob handle.
*/
{OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */
{OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */
- {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */
+ {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */
{OP_Column, 0, 0, 1}, /* 7 */
{OP_ResultRow, 1, 0, 0}, /* 8 */
- {OP_Close, 0, 0, 0}, /* 9 */
- {OP_Halt, 0, 0, 0}, /* 10 */
+ {OP_Goto, 0, 5, 0}, /* 9 */
+ {OP_Close, 0, 0, 0}, /* 10 */
+ {OP_Halt, 0, 0, 0}, /* 11 */
};
Vdbe *v = 0;
char *zErr = 0;
Table *pTab;
Parse *pParse;
+ Incrblob *pBlob;
+ flags = !!flags; /* flags = (flags ? 1 : 0); */
*ppBlob = 0;
+
sqlite3_mutex_enter(db->mutex);
+
+ pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
pParse = sqlite3StackAllocRaw(db, sizeof(*pParse));
- if( pParse==0 ){
- rc = SQLITE_NOMEM;
+ if( pParse==0 || pBlob==0 ){
+ assert( db->mallocFailed );
goto blob_open_out;
}
+
do {
memset(pParse, 0, sizeof(Parse));
pParse->db = db;
if( v ){
int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob);
- flags = !!flags; /* flags = (flags ? 1 : 0); */
/* Configure the OP_Transaction */
sqlite3VdbeChangeP1(v, 0, iDb);
}
}
+ pBlob->flags = flags;
+ pBlob->pStmt = (sqlite3_stmt *)v;
+ pBlob->iCol = iCol;
+ pBlob->db = db;
sqlite3BtreeLeaveAll(db);
+ v = 0;
if( db->mallocFailed ){
goto blob_open_out;
}
- sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow);
- rc = sqlite3_step((sqlite3_stmt *)v);
- if( rc!=SQLITE_ROW ){
- nAttempt++;
- rc = sqlite3_finalize((sqlite3_stmt *)v);
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, sqlite3_errmsg(db));
- v = 0;
- }
- } while( nAttempt<5 && rc==SQLITE_SCHEMA );
-
- if( rc==SQLITE_ROW ){
- /* The row-record has been opened successfully. Check that the
- ** column in question contains text or a blob. If it contains
- ** text, it is up to the caller to get the encoding right.
- */
- Incrblob *pBlob;
- u32 type = v->apCsr[0]->aType[iCol];
+ sqlite3_bind_int64(pBlob->pStmt, 1, iRow);
+ rc = blobSeekToRow(pBlob, iRow, &zErr);
+ } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA );
- if( type<12 ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "cannot open value of type %s",
- type==0?"null": type==7?"real": "integer"
- );
- rc = SQLITE_ERROR;
- goto blob_open_out;
- }
- pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob));
- if( db->mallocFailed ){
- sqlite3DbFree(db, pBlob);
- goto blob_open_out;
- }
- pBlob->flags = flags;
- pBlob->pCsr = v->apCsr[0]->pCursor;
- sqlite3BtreeEnterCursor(pBlob->pCsr);
- sqlite3BtreeCacheOverflow(pBlob->pCsr);
- sqlite3BtreeLeaveCursor(pBlob->pCsr);
- pBlob->pStmt = (sqlite3_stmt *)v;
- pBlob->iOffset = v->apCsr[0]->aOffset[iCol];
- pBlob->nByte = sqlite3VdbeSerialTypeLen(type);
- pBlob->db = db;
+blob_open_out:
+ if( rc==SQLITE_OK && db->mallocFailed==0 ){
*ppBlob = (sqlite3_blob *)pBlob;
- rc = SQLITE_OK;
- }else if( rc==SQLITE_OK ){
- sqlite3DbFree(db, zErr);
- zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow);
- rc = SQLITE_ERROR;
+ }else{
+ if( v ) sqlite3VdbeFinalize(v);
+ if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
+ sqlite3DbFree(db, pBlob);
}
-blob_open_out:
- if( v && (rc!=SQLITE_OK || db->mallocFailed) ){
- sqlite3VdbeFinalize(v);
- }
- sqlite3Error(db, rc, zErr);
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
sqlite3DbFree(db, zErr);
sqlite3StackFree(db, pParse);
rc = sqlite3ApiExit(db, rc);
/* Request is out of range. Return a transient error. */
rc = SQLITE_ERROR;
sqlite3Error(db, SQLITE_ERROR, 0);
- } else if( v==0 ){
+ }else if( v==0 ){
/* If there is no statement handle, then the blob-handle has
** already been invalidated. Return SQLITE_ABORT in this case.
*/
return p ? p->nByte : 0;
}
+/*
+** Move an existing blob handle to point to a different row of the same
+** database table.
+**
+** If an error occurs, or if the specified row does not exist or does not
+** contain a blob or text value, then an error code is returned and the
+** database handle error code and message set. If this happens, then all
+** subsequent calls to sqlite3_blob_xxx() functions (except blob_close())
+** immediately return SQLITE_ABORT.
+*/
+int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
+ int rc;
+ Incrblob *p = (Incrblob *)pBlob;
+ sqlite3 *db;
+
+ if( p==0 ) return SQLITE_MISUSE_BKPT;
+ db = p->db;
+ sqlite3_mutex_enter(db->mutex);
+
+ if( p->pStmt==0 ){
+ /* If there is no statement handle, then the blob-handle has
+ ** already been invalidated. Return SQLITE_ABORT in this case.
+ */
+ rc = SQLITE_ABORT;
+ }else{
+ char *zErr;
+ rc = blobSeekToRow(p, iRow, &zErr);
+ if( rc!=SQLITE_OK ){
+ sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr);
+ sqlite3DbFree(db, zErr);
+ }
+ assert( rc!=SQLITE_SCHEMA );
+ }
+
+ rc = sqlite3ApiExit(db, rc);
+ sqlite3_mutex_leave(db->mutex);
+ return rc;
+}
+
#endif /* #ifndef SQLITE_OMIT_INCRBLOB */
execsql {SELECT rowid FROM t1 WHERE t1 MATCH $aterm}
} {1 3}
-do_test fts3ah-1.2 {
+do_test fts3ah-1.3 {
execsql {SELECT rowid FROM t1 WHERE t1 MATCH $xterm}
} {}
-do_test fts3ah-1.3 {
+do_test fts3ah-1.4 {
execsql "SELECT rowid FROM t1 WHERE t1 MATCH '$aterm -$xterm'"
} {1 3}
-do_test fts3ah-1.4 {
+do_test fts3ah-1.5 {
execsql "SELECT rowid FROM t1 WHERE t1 MATCH '\"$aterm $bterm\"'"
} {1}
return
}
+set sqlite_fts3_enable_parentheses 1
+
set ::testprefix fts3defer
#--------------------------------------------------------------------------
"srwwnezqk csjqxhgj rbwzuf nvfasfh jcpiwj xldlpy nvfasfh jk vgsld wjybxmieki"
}
+#set e [list]
+#foreach d $data {set e [concat $e $d]}
+#puts [lsort -unique $e]
+#exit
+set zero_long_doclists {
+ UPDATE t1_segments SET block=zeroblob(length(block)) WHERE length(block)>10000
+}
foreach {tn setup} {
1 {
set dmt_modes {0 1 2}
execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
- execsql {
- UPDATE t1_segments
- SET block = zeroblob(length(block))
- WHERE length(block)>10000;
- }
+ execsql $zero_long_doclists
+ }
+ 4 {
+ set dmt_modes 0
+ execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
+ foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
+ execsql "INSERT INTO t1(t1) VALUES('optimize')"
+ execsql $zero_long_doclists
}
} {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm jk vgsld"'
} {13 17}
+ do_select_test 2.8 {
+ SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
+ } {10 13 17 31 35 51 58 88 89 90 93 100}
+ do_select_test 2.9 {
+ SELECT rowid FROM t1
+ WHERE t1 MATCH '(
+ zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
+ zk OR zkhdvkw OR zm OR zsmhnf
+ ) vgsld'
+ } {10 13 17 31 35 51 58 88 89 90 93 100}
+
do_select_test 3.1 {
SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
} {
projects / thirdparty / sqlite.git / commitdiff
