** 'xyz' becomes xyz
** [pqr] becomes pqr
** `mno` becomes mno
+**
*/
void sqlite3Fts3Dequote(char *z){
- char quote;
- int i, j;
+ char quote; /* Quote character (if any ) */
quote = z[0];
- switch( quote ){
- case '\'': break;
- case '"': break;
- case '`': break; /* For MySQL compatibility */
- case '[': quote = ']'; break; /* For MS SqlServer compatibility */
- default: return;
- }
-
- i = 1;
- j = 0;
- while( ALWAYS(z[i]) ){
- if( z[i]==quote ){
- if( z[i+1]==quote ){
- z[j++] = quote;
- i += 2;
+ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
+ int iIn = 1; /* Index of next byte to read from input */
+ int iOut = 0; /* Index of next byte to write to output */
+
+ /* If the first byte was a '[', then the close-quote character is a ']' */
+ if( quote=='[' ) quote = ']';
+
+ while( ALWAYS(z[iIn]) ){
+ if( z[iIn]==quote ){
+ if( z[iIn+1]!=quote ) break;
+ z[iOut++] = quote;
+ iIn += 2;
}else{
- break;
+ z[iOut++] = z[iIn++];
}
- }else{
- z[j++] = z[i++];
}
+ z[iOut] = '\0';
}
- z[j] = 0;
}
static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
int i;
TermSelect tsc;
Fts3SegFilter filter; /* Segment term filter configuration */
- Fts3SegReader **apSegment = 0; /* Array of segments to read data from */
+ Fts3SegReader **apSegment; /* Array of segments to read data from */
int nSegment = 0; /* Size of apSegment array */
- int nAlloc = 0; /* Allocated size of segment array */
+ int nAlloc = 16; /* Allocated size of segment array */
int rc; /* Return code */
sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */
int iAge = 0; /* Used to assign ages to segments */
+ apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc);
+ if( !apSegment ) return SQLITE_NOMEM;
+ rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]);
+ if( rc!=SQLITE_OK ) return rc;
+ if( apSegment[0] ){
+ nSegment = 1;
+ }
+
/* Loop through the entire %_segdir table. For each segment, create a
** Fts3SegReader to iterate through the subset of the segment leaves
** that may contain a term that matches zTerm/nTerm. For non-prefix
int iCol = pPhrase->iColumn;
int isTermPos = (pPhrase->nToken>1 || isReqPos);
- assert( p->nPendingData==0 );
-
for(ii=0; ii<pPhrase->nToken; ii++){
struct PhraseToken *pTok = &pPhrase->aToken[ii];
char *z = pTok->z; /* Next token of the phrase */
if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
return SQLITE_NOMEM;
}
- rc = sqlite3Fts3PendingTermsFlush(p);
- if( rc!=SQLITE_OK ) return rc;
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
iCol, zQuery, -1, &pCsr->pExpr
/* The column value supplied by SQLite must be in range. */
assert( iCol>=0 && iCol<=p->nColumn+1 );
- rc = fts3CursorSeek(pCsr);
- if( rc==SQLITE_OK ){
- if( iCol==p->nColumn+1 ){
- /* This call is a request for the "docid" column. Since "docid" is an
- ** alias for "rowid", use the xRowid() method to obtain the value.
- */
- sqlite3_int64 iRowid;
- rc = fts3RowidMethod(pCursor, &iRowid);
- sqlite3_result_int64(pContext, iRowid);
- }else if( iCol==p->nColumn ){
- /* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor.
- */
- sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
- }else{
+ if( iCol==p->nColumn+1 ){
+ /* This call is a request for the "docid" column. Since "docid" is an
+ ** alias for "rowid", use the xRowid() method to obtain the value.
+ */
+ sqlite3_int64 iRowid;
+ rc = fts3RowidMethod(pCursor, &iRowid);
+ sqlite3_result_int64(pContext, iRowid);
+ }else if( iCol==p->nColumn ){
+ /* The extra column whose name is the same as the table.
+ ** Return a blob which is a pointer to the cursor.
+ */
+ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
+ rc = SQLITE_OK;
+ }else{
+ rc = fts3CursorSeek(pCsr);
+ if( rc==SQLITE_OK ){
sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1));
}
}
int sqlite3Fts3Optimize(Fts3Table *);
int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
+int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
}
}
-/* Attempt to locate an element of the hash table pH with a key
-** that matches pKey,nKey. Return the data for this element if it is
-** found, or NULL if there is no match.
-*/
-void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
- int h; /* A hash on key */
- Fts3HashElem *elem; /* The element that matches key */
+Fts3HashElem *sqlite3Fts3HashFindElem(
+ const Fts3Hash *pH,
+ const void *pKey,
+ int nKey
+){
+ int h; /* A hash on key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
assert( xHash!=0 );
h = (*xHash)(pKey,nKey);
assert( (pH->htsize & (pH->htsize-1))==0 );
- elem = fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
- return elem ? elem->data : 0;
+ return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1));
+}
+
+/*
+** Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey. Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
+ Fts3HashElem *pElem; /* The element that matches key (if any) */
+
+ pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey);
+ return pElem ? pElem->data : 0;
}
/* Insert an element into the hash table pH. The key is pKey,nKey
void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
void sqlite3Fts3HashClear(Fts3Hash*);
+Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int);
/*
** Shorthand for the functions above
*/
-#define fts3HashInit sqlite3Fts3HashInit
-#define fts3HashInsert sqlite3Fts3HashInsert
-#define fts3HashFind sqlite3Fts3HashFind
-#define fts3HashClear sqlite3Fts3HashClear
+#define fts3HashInit sqlite3Fts3HashInit
+#define fts3HashInsert sqlite3Fts3HashInsert
+#define fts3HashFind sqlite3Fts3HashFind
+#define fts3HashClear sqlite3Fts3HashClear
+#define fts3HashFindElem sqlite3Fts3HashFindElem
/*
** Macros for looping over all elements of a hash table. The idiom is
/*
** Data structure used while accumulating terms in the pending-terms hash
-** table. The hash table entry maps from term (a string) to a malloced
+** table. The hash table entry maps from term (a string) to a malloc'd
** instance of this structure.
*/
struct PendingList {
** sqlite3Fts3SegReaderNew()
** sqlite3Fts3SegReaderFree()
** sqlite3Fts3SegReaderIterate()
+**
+** Methods used to manipulate Fts3SegReader structures:
+**
+** fts3SegReaderNext()
+** fts3SegReaderFirstDocid()
+** fts3SegReaderNextDocid()
*/
struct Fts3SegReader {
- int iIdx; /* Index within level */
+ int iIdx; /* Index within level, or 0x7FFFFFFF for PT */
sqlite3_int64 iStartBlock;
sqlite3_int64 iEndBlock;
sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
char *aNode; /* Pointer to node data (or NULL) */
int nNode; /* Size of buffer at aNode (or 0) */
int nTermAlloc; /* Allocated size of zTerm buffer */
+ Fts3HashElem **ppNextElem;
/* Variables set by fts3SegReaderNext(). These may be read directly
** by the caller. They are valid from the time SegmentReaderNew() returns
sqlite3_int64 iDocid;
};
+#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0)
+
/*
** An instance of this structure is used to create a segment b-tree in the
** database. The internal details of this type are only accessed by the
if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
int rc;
+ if( fts3SegReaderIsPending(pReader) ){
+ Fts3HashElem *pElem = *(pReader->ppNextElem);
+ if( pElem==0 ){
+ pReader->aNode = 0;
+ }else{
+ PendingList *pList = (PendingList *)fts3HashData(pElem);
+ pReader->zTerm = (char *)fts3HashKey(pElem);
+ pReader->nTerm = fts3HashKeysize(pElem);
+ pReader->nNode = pReader->nDoclist = pList->nData;
+ pReader->aNode = pReader->aDoclist = pList->aData;
+ pReader->ppNextElem++;
+ }
+ return SQLITE_OK;
+ }
if( !pReader->pStmt ){
pReader->aNode = 0;
return SQLITE_OK;
sqlite3_reset(pReader->pStmt);
p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
}
- sqlite3_free(pReader->zTerm);
+ if( !fts3SegReaderIsPending(pReader) ){
+ sqlite3_free(pReader->zTerm);
+ }
sqlite3_free(pReader);
}
}
return rc;
}
+/*
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
+*/
+static int fts3CompareElemByTerm(const void *lhs, const void *rhs){
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
+
+ int n = (n1<n2 ? n1 : n2);
+ int c = memcmp(z1, z2, n);
+ if( c==0 ){
+ c = n1 - n2;
+ }
+ return c;
+}
+
+/*
+** This function is used to allocate an Fts3SegReader that iterates through
+** a subset of the terms stored in the Fts3Table.pendingTerms array.
+*/
+int sqlite3Fts3SegReaderPending(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to search for */
+ int nTerm, /* Size of buffer zTerm */
+ int isPrefix, /* True for a term-prefix query */
+ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */
+){
+ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */
+ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */
+ int nElem = 0; /* Size of array at aElem */
+ int rc = SQLITE_OK; /* Return Code */
+
+ if( isPrefix ){
+ Fts3HashElem *pE = 0; /* Iterator variable */
+
+ for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){
+ char *zKey = (char *)fts3HashKey(pE);
+ int nKey = fts3HashKeysize(pE);
+ if( nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm) ){
+ int nByte = (1+nElem * sizeof(Fts3HashElem *));
+ Fts3HashElem **aElem2 = (Fts3HashElem **)sqlite3_realloc(aElem, nByte);
+ if( !aElem2 ){
+ rc = SQLITE_NOMEM;
+ nElem = 0;
+ break;
+ }
+ aElem = aElem2;
+ aElem[nElem++] = pE;
+ }
+ }
+
+ /* If more than one term matches the prefix, sort the Fts3HashElem
+ ** objects in term order using qsort(). This uses the same comparison
+ ** callback as is used when flushing terms to disk.
+ */
+ if( nElem>1 ){
+ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
+ }
+
+ }else{
+ Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm);
+ if( pE ){
+ aElem = &pE;
+ nElem = 1;
+ }
+ }
+
+ if( nElem>0 ){
+ int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *);
+ pReader = (Fts3SegReader *)sqlite3_malloc(nByte);
+ if( !pReader ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pReader, 0, nByte);
+ pReader->iIdx = 0x7FFFFFFF;
+ pReader->ppNextElem = (Fts3HashElem **)&pReader[1];
+ memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *));
+ fts3SegReaderNext(pReader);
+ }
+ }
+
+ if( isPrefix ){
+ sqlite3_free(aElem);
+ }
+ *ppReader = pReader;
+ return rc;
+}
+
/*
** The second argument to this function is expected to be a statement of
**
** 1) EOF is greater than not EOF.
**
-** 2) The current terms (if any) are compared with memcmp(). If one
+** 2) The current terms (if any) are compared using memcmp(). If one
** term is a prefix of another, the longer term is considered the
** larger.
**
Fts3SegReader *pSeg = apSegment[i];
while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
rc = fts3SegReaderNext(pSeg);
- if( rc!=SQLITE_OK ) goto finished;
- }
+ if( rc!=SQLITE_OK ) goto finished; }
}
}
sqlite3_int64 iDocid = apSegment[0]->iDocid;
fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
j = 1;
- while( j<nMerge
- && apSegment[j]->pOffsetList
- && apSegment[j]->iDocid==iDocid
+ while( j<nMerge
+ && apSegment[j]->pOffsetList
+ && apSegment[j]->iDocid==iDocid
){
fts3SegReaderNextDocid(apSegment[j], 0, 0);
j++;
return rc;
}
-/*
-** This is a comparison function used as a qsort() callback when sorting
-** an array of pending terms by term. This occurs as part of flushing
-** the contents of the pending-terms hash table to the database.
-*/
-static int qsortCompare(const void *lhs, const void *rhs){
- char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
- char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
- int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
- int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
-
- int n = (n1<n2 ? n1 : n2);
- int c = memcmp(z1, z2, n);
- if( c==0 ){
- c = n1 - n2;
- }
- return c;
-}
-
/*
** Flush the contents of pendingTerms to a level 0 segment.
** Also, should we be using qsort()?
*/
if( nElem>1 ){
- qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare);
+ qsort(apElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm);
}
------BEGIN PGP SIGNED MESSAGE-----
-Hash: SHA1
-
-C Only\sdeclare\sthe\ssqlite3_mutex_held()\sand\ssqlite3_mutex_notheld()\sinterfaces\nin\sthe\sheader\sfile\sif\sNDEBUG\sis\snot\sdefined.
-D 2009-12-10T01:17:29
+C Changes\sto\sfts3\sto\savoid\sflushing\sdata\sto\sdisk\swithin\sa\sSELECT\sstatement.
+D 2009-12-10T16:04:26
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 6e24afa93ecb3395b0e9467ca2b44fa7f66f4fdc
+F ext/fts3/fts3.c f72d7fdb5cca933a9311037e63610a182d9369b4
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 36f8e6d6fafa6d71a48e1810095e1e58fd55b199
+F ext/fts3/fts3Int.h 1419e2973b44ee78f0ae8f7e03abfa2bdaf14f54
F ext/fts3/fts3_expr.c fcf6812dbfd9cb9a2cabaf50e741411794f83e7e
-F ext/fts3/fts3_hash.c 18feef38fca216992725e9eae775a0c7735e6724
-F ext/fts3/fts3_hash.h d410ff2c93c81a56b927fcf07b2099ccbfa7a479
+F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
+F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c a651e287e02b49b565a6ccf9441959d434489156
F ext/fts3/fts3_snippet.c 6c2eb6d872d66b2a9aa5663f2662e993f18a6496
F ext/fts3/fts3_tokenizer.c 3dc76eaea6b58ecfbe50135b8473aa668d712dcd
F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
F ext/fts3/fts3_tokenizer1.c 11a604a53cff5e8c28882727bf794e5252e5227b
-F ext/fts3/fts3_write.c c7d549ac5a2733d1d3004a784ad8387f5c65fc75
+F ext/fts3/fts3_write.c ee50b8feb757bf0cddc522223ebd49f91985a1ad
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
F test/fts3.test ae0433b09b12def08105640e57693726c4949338
-F test/fts3_common.tcl 8f75c15fa4507d923a1c21152e2716d77dc74661
+F test/fts3_common.tcl 363b6b215cb88ad9a59c5fd7c9d4dd0581446497
F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
F test/fts3ac.test 356280144a2c92aa7b11474afadfe62a437fcd69
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F test/fts3malloc.test d02ee86b21edd2b43044e0d6dfdcd26cb6efddcb
F test/fts3near.test dc196dd17b4606f440c580d45b3d23aa975fd077
-F test/fts3rnd.test 44fa7209327deabc3ca22e2ff7a293b0bd46c37f
+F test/fts3rnd.test d7fe25493aa76f5010df0a6dbfa4dfa14f537c26
F test/func.test af106ed834001738246d276659406823e35cde7b
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/fuzz.test a4174c3009a3e2c2e14b31b364ebf7ddb49de2c9
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 54b955c36ba5c139a63c6031855305b764d3fa6c
-R af7bd7d961d62f8a3ad909680a657a87
-U drh
-Z 29173053da939d420a44ec000f9044f8
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFLIEwtoxKgR168RlERArN7AJwJu6zTK/fKFR2tbMFA0GU+gGv4YACffJG4
-rSGfNuCvvW2w3AtoGqVeahA=
-=A3lG
------END PGP SIGNATURE-----
+P ee9b1c05a7f12d3d668b804bd11ae0def984b66e
+R 58b4a041207a70975c8de2329badf743
+U dan
+Z 97d832b78b48585e6dc269202723d83a
-ee9b1c05a7f12d3d668b804bd11ae0def984b66e
\ No newline at end of file
+48c0db0eb2d134bb302bb5eca6beb0ec46736257
\ No newline at end of file
set modes [list 100000 transient 1 persistent]
} else {
set answers [list $catchres]
- set modes [list 0 nofail]
+ set modes [list 0 ""]
}
set str [join $answers " OR "]
foreach {nRepeat zName} $modes {
- for {set iFail 1} 1 {incr iFail} {
+ for {set iFail 48} 1 {incr iFail} {
if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
set res [uplevel [list catchsql $sql]]
if {[lsearch -exact $answers $res]>=0} {
set res $str
}
- do_test $name.$zName.$iFail [list set {} $res] $str
+ set testname "$name.$zName.$iFail"
+ if {$zName == ""} { set testname $name }
+ do_test $testname [list set {} $res] $str
set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
if {$nFail==0} break
}
set nVocab 100
set lVocab [list]
+expr srand(0)
+
+
# Generate a vocabulary of nVocab words. Each word is 3 characters long.
#
set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z}
}
for {set iTest 0} {$iTest <= 100} {incr iTest} {
+ catchsql COMMIT
set DO_MALLOC_TEST 0
set nRep 10
while {[info exists ::t1($iInsert)]} {
set iInsert [expr {int(rand()*1000000)}]
}
- db transaction {
+ execsql BEGIN
insert_row $iInsert
update_row $iUpdate
delete_row $iDelete
- }
+ if {0==($iTest%2)} { execsql COMMIT }
# Pick 10 terms from the vocabulary. Check that the results of querying
# the database for the set of documents containing each of these terms
]
}
}
+
+ catchsql COMMIT
}
}
projects / thirdparty / sqlite.git / commitdiff
