int iSzPoslist; /* Offset of space for 4-byte poslist size */
int nData; /* Total bytes of data (incl. structure) */
u8 bDel; /* Set delete-flag @ iSzPoslist */
+ u8 bContent; /* Set content-flag (detail=none mode) */
int iCol; /* Column of last value written */
int iPos; /* Position of last value written */
return SQLITE_OK;
}
-static void fts5HashAddPoslistSize(Fts5HashEntry *p){
+static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
if( p->iSzPoslist ){
u8 *pPtr = (u8*)p;
- int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
- int nPos = nSz*2 + p->bDel; /* Value of nPos field */
-
- assert( p->bDel==0 || p->bDel==1 );
- if( nPos<=127 ){
- pPtr[p->iSzPoslist] = (u8)nPos;
+ if( pHash->eDetail==FTS5_DETAIL_NONE ){
+ assert( p->nData==p->iSzPoslist );
+ if( p->bDel ){
+ pPtr[p->nData++] = 0x00;
+ if( p->bContent ){
+ pPtr[p->nData++] = 0x00;
+ }
+ }
}else{
- int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
- memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
- sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
- p->nData += (nByte-1);
+ int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
+ int nPos = nSz*2 + p->bDel; /* Value of nPos field */
+
+ assert( p->bDel==0 || p->bDel==1 );
+ if( nPos<=127 ){
+ pPtr[p->iSzPoslist] = (u8)nPos;
+ }else{
+ int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
+ memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
+ sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
+ p->nData += (nByte-1);
+ }
}
- p->bDel = 0;
+
p->iSzPoslist = 0;
+ p->bDel = 0;
+ p->bContent = 0;
}
}
+/*
+** Add an entry to the in-memory hash table. The key is the concatenation
+** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
+**
+** (bByte || pToken) -> (iRowid,iCol,iPos)
+**
+** Or, if iCol is negative, then the value is a delete marker.
+*/
int sqlite3Fts5HashWrite(
Fts5Hash *pHash,
i64 iRowid, /* Rowid for this entry */
/* If an existing hash entry cannot be found, create a new one. */
if( p==0 ){
+ /* Figure out how much space to allocate */
int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
if( nByte<128 ) nByte = 128;
+ /* Grow the Fts5Hash.aSlot[] array if necessary. */
if( (pHash->nEntry*2)>=pHash->nSlot ){
int rc = fts5HashResize(pHash);
if( rc!=SQLITE_OK ) return rc;
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
}
+ /* Allocate new Fts5HashEntry and add it to the hash table. */
p = (Fts5HashEntry*)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, FTS5_HASHENTRYSIZE);
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
p->zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
- p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
- p->iSzPoslist = p->nData;
- p->nData += 1;
- p->iRowid = iRowid;
- p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
+
+ /* Add the first rowid field to the hash-entry */
+ p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
+ p->iRowid = iRowid;
+
+ p->iSzPoslist = p->nData;
+ if( pHash->eDetail!=FTS5_DETAIL_NONE ){
+ p->nData += 1;
+ p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
+ }
+
nIncr += p->nData;
- }
+ }else{
- /* Check there is enough space to append a new entry. Worst case scenario
- ** is:
- **
- ** + 9 bytes for a new rowid,
- ** + 4 byte reserved for the "poslist size" varint.
- ** + 1 byte for a "new column" byte,
- ** + 3 bytes for a new column number (16-bit max) as a varint,
- ** + 5 bytes for the new position offset (32-bit max).
- */
- if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
- int nNew = p->nAlloc * 2;
- Fts5HashEntry *pNew;
- Fts5HashEntry **pp;
- pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
- if( pNew==0 ) return SQLITE_NOMEM;
- pNew->nAlloc = nNew;
- for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
- *pp = pNew;
- p = pNew;
+ /* Appending to an existing hash-entry. Check that there is enough
+ ** space to append the largest possible new entry. Worst case scenario
+ ** is:
+ **
+ ** + 9 bytes for a new rowid,
+ ** + 4 byte reserved for the "poslist size" varint.
+ ** + 1 byte for a "new column" byte,
+ ** + 3 bytes for a new column number (16-bit max) as a varint,
+ ** + 5 bytes for the new position offset (32-bit max).
+ */
+ if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
+ int nNew = p->nAlloc * 2;
+ Fts5HashEntry *pNew;
+ Fts5HashEntry **pp;
+ pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
+ if( pNew==0 ) return SQLITE_NOMEM;
+ pNew->nAlloc = nNew;
+ for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
+ *pp = pNew;
+ p = pNew;
+ }
+ nIncr -= p->nData;
}
+ assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
+
pPtr = (u8*)p;
- nIncr -= p->nData;
/* If this is a new rowid, append the 4-byte size field for the previous
** entry, and the new rowid for this entry. */
if( iRowid!=p->iRowid ){
- fts5HashAddPoslistSize(p);
+ fts5HashAddPoslistSize(pHash, p);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
- p->iSzPoslist = p->nData;
- p->nData += 1;
- p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
- p->iPos = 0;
p->iRowid = iRowid;
bNew = 1;
+ p->iSzPoslist = p->nData;
+ if( pHash->eDetail!=FTS5_DETAIL_NONE ){
+ p->nData += 1;
+ p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
+ p->iPos = 0;
+ }
}
if( iCol>=0 ){
- /* Append a new column value, if necessary */
- assert( iCol>=p->iCol );
- if( iCol!=p->iCol ){
- if( pHash->eDetail==FTS5_DETAIL_FULL ){
- pPtr[p->nData++] = 0x01;
- p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
- p->iCol = iCol;
- p->iPos = 0;
- }else{
- bNew = 1;
- p->iCol = iPos = iCol;
+ if( pHash->eDetail==FTS5_DETAIL_NONE ){
+ p->bContent = 1;
+ }else{
+ /* Append a new column value, if necessary */
+ assert( iCol>=p->iCol );
+ if( iCol!=p->iCol ){
+ if( pHash->eDetail==FTS5_DETAIL_FULL ){
+ pPtr[p->nData++] = 0x01;
+ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
+ p->iCol = iCol;
+ p->iPos = 0;
+ }else{
+ bNew = 1;
+ p->iCol = iPos = iCol;
+ }
}
- }
- /* Append the new position offset, if necessary */
- if( bNew ){
- p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
- p->iPos = iPos;
+ /* Append the new position offset, if necessary */
+ if( bNew ){
+ p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
+ p->iPos = iPos;
+ }
}
}else{
/* This is a delete. Set the delete flag. */
p->bDel = 1;
}
- nIncr += p->nData;
+ nIncr += p->nData;
*pHash->pnByte += nIncr;
return SQLITE_OK;
}
}
if( p ){
- fts5HashAddPoslistSize(p);
+ fts5HashAddPoslistSize(pHash, p);
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
}else{
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
int nTerm = (int)strlen(p->zKey);
- fts5HashAddPoslistSize(p);
+ fts5HashAddPoslistSize(pHash, p);
*pzTerm = p->zKey;
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
Fts5Buffer term; /* Current term */
i64 iRowid; /* Current rowid */
int nPos; /* Number of bytes in current position list */
- int bDel; /* True if the delete flag is set */
+ u8 bDel; /* True if the delete flag is set */
+ // u8 bContent; /* True if has content (detail=none mode) */
};
/*
#define FTS5_SEGITER_ONETERM 0x01
#define FTS5_SEGITER_REVERSE 0x02
-
/*
** Argument is a pointer to an Fts5Data structure that contains a leaf
** page. This macro evaluates to true if the leaf contains no terms, or
static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
if( p->rc==SQLITE_OK ){
int iOff = pIter->iLeafOffset; /* Offset to read at */
- int nSz;
ASSERT_SZLEAF_OK(pIter->pLeaf);
- fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
- pIter->bDel = (nSz & 0x0001);
- pIter->nPos = nSz>>1;
+ if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
+ pIter->bDel = 0;
+ pIter->nPos = 1;
+ if( iOff<pIter->pLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){
+ pIter->bDel = 1;
+ iOff++;
+ if( iOff<pIter->pLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){
+ pIter->nPos = 1;
+ iOff++;
+ }else{
+ pIter->nPos = 0;
+ }
+ }
+ }else{
+ int nSz;
+ fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
+ pIter->bDel = (nSz & 0x0001);
+ pIter->nPos = nSz>>1;
+ assert_nc( pIter->nPos>=0 );
+ }
pIter->iLeafOffset = iOff;
- assert_nc( pIter->nPos>=0 );
}
}
int n = pLeaf->szLeaf;
ASSERT_SZLEAF_OK(pLeaf);
- iOff = pIter->iLeafOffset + pIter->nPos;
+ if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
+ iOff = pIter->iLeafOffset;
+ }else{
+ iOff = pIter->iLeafOffset + pIter->nPos;
+ }
if( iOff<n ){
/* The next entry is on the current page. */
}else{
/* The following could be done by calling fts5SegIterLoadNPos(). But
** this block is particularly performance critical, so equivalent
- ** code is inlined. */
+ ** code is inlined.
+ **
+ ** Later: Switched back to fts5SegIterLoadNPos() because it supports
+ ** detail=none mode. Not ideal.
+ */
+#if 0
int nSz;
assert( p->rc==SQLITE_OK );
fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
pIter->bDel = (nSz & 0x0001);
pIter->nPos = nSz>>1;
assert_nc( pIter->nPos>=0 );
+#endif
+ fts5SegIterLoadNPos(p, pIter);
}
}
}
return p - (*pa);
}
+static int fts5AppendRowid(
+ Fts5Index *p,
+ i64 iDelta,
+ Fts5IndexIter *pMulti,
+ Fts5Colset *pColset,
+ Fts5Buffer *pBuf
+){
+ fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
+ return 0;
+}
/*
** Iterator pMulti currently points to a valid entry (not EOF). This
(iLastRowid) = (iRowid); \
}
+/*
+** Swap the contents of buffer *p1 with that of *p2.
+*/
+static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
+ Fts5Buffer tmp = *p1;
+ *p1 = *p2;
+ *p2 = tmp;
+}
+
+static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
+ int i = *piOff;
+ if( i>=pBuf->n ){
+ *piOff = -1;
+ }else{
+ u64 iVal;
+ *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
+ *piRowid += iVal;
+ }
+}
+
+/*
+** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
+** In this case the buffers consist of a delta-encoded list of rowids only.
+*/
+static void fts5MergeRowidLists(
+ Fts5Index *p, /* FTS5 backend object */
+ Fts5Buffer *p1, /* First list to merge */
+ Fts5Buffer *p2 /* Second list to merge */
+){
+ int i1 = 0;
+ int i2 = 0;
+ i64 iRowid1 = 0;
+ i64 iRowid2 = 0;
+ i64 iOut = 0;
+
+ Fts5Buffer out;
+ memset(&out, 0, sizeof(out));
+ sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
+ if( p->rc ) return;
+
+ fts5NextRowid(p1, &i1, &iRowid1);
+ fts5NextRowid(p1, &i2, &iRowid2);
+ while( i1>=0 || i2>=0 ){
+ if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){
+ fts5BufferSafeAppendVarint(&out, iRowid1 - iOut);
+ iOut = iRowid1;
+ fts5NextRowid(p1, &i1, &iRowid1);
+ }else{
+ fts5BufferSafeAppendVarint(&out, iRowid2 - iOut);
+ iOut = iRowid2;
+ fts5NextRowid(p2, &i2, &iRowid2);
+ if( i1>=0 && iRowid1==iRowid2 ){
+ fts5NextRowid(p1, &i1, &iRowid1);
+ }
+ }
+ }
+
+ fts5BufferSwap(&out, p1);
+ fts5BufferFree(&out);
+}
+
/*
** Buffers p1 and p2 contain doclists. This function merges the content
** of the two doclists together and sets buffer p1 to the result before
}
}
-/*
-** Swap the contents of buffer *p1 with that of *p2.
-*/
-static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
- Fts5Buffer tmp = *p1;
- *p1 = *p2;
- *p2 = tmp;
-}
-
static void fts5SetupPrefixIter(
Fts5Index *p, /* Index to read from */
int bDesc, /* True for "ORDER BY rowid DESC" */
Fts5Buffer *aBuf;
const int nBuf = 32;
+ void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
+ int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
+ if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
+ xMerge = fts5MergeRowidLists;
+ xAppend = fts5AppendRowid;
+ }else{
+ xMerge = fts5MergePrefixLists;
+ xAppend = fts5AppendPoslist;
+ }
+
aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
pStruct = fts5StructureRead(p);
fts5BufferSwap(&doclist, &aBuf[i]);
fts5BufferZero(&doclist);
}else{
- fts5MergePrefixLists(p, &doclist, &aBuf[i]);
+ xMerge(p, &doclist, &aBuf[i]);
fts5BufferZero(&aBuf[i]);
}
}
iLastRowid = 0;
}
- if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
+ if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
iLastRowid = iRowid;
}
}
for(i=0; i<nBuf; i++){
if( p->rc==SQLITE_OK ){
- fts5MergePrefixLists(p, &doclist, &aBuf[i]);
+ xMerge(p, &doclist, &aBuf[i]);
}
fts5BufferFree(&aBuf[i]);
}
assert( pIter->pIndex->rc==SQLITE_OK );
*piRowid = pSeg->iRowid;
+ if( eDetail==FTS5_DETAIL_NONE ){
+ *pn = pSeg->nPos;
+ }else
if( eDetail==FTS5_DETAIL_FULL
&& pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf
){
--- /dev/null
+# 2015 September 05
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5simple2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+ INSERT INTO t1 VALUES('a b c');
+}
+do_execsql_test 1.1 {
+ SELECT rowid FROM t1('c a b')
+} {1}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+ BEGIN;
+ INSERT INTO t1 VALUES('b c d');
+ INSERT INTO t1 VALUES('b c d');
+ COMMIT;
+}
+do_execsql_test 2.1 {
+ SELECT rowid FROM t1('b c d')
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+ BEGIN;
+ INSERT INTO t1 VALUES('b c d');
+ INSERT INTO t1 VALUES('b c d');
+}
+do_execsql_test 3.1 {
+ SELECT rowid FROM t1('b c d'); COMMIT;
+} {1 2}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
+ BEGIN;
+ INSERT INTO t1 VALUES('a1 b1 c1');
+ INSERT INTO t1 VALUES('a2 b2 c2');
+ INSERT INTO t1 VALUES('a3 b3 c3');
+ COMMIT;
+}
+breakpoint
+do_execsql_test 4.1 {
+ SELECT rowid FROM t1('b*');
+} {1 2 3}
+
+
+finish_test
+
-C Add\sthe\sxPhraseFirstColumn()\sand\sxPhraseNextColumn()\sAPI\sfunctions\sto\sfts5.\sFor\siterating\sthrough\sthe\sset\sof\scolumns\sthat\scontain\sintances\sof\sa\sphrase.
-D 2015-12-29T19:35:03.765
+C Updates\sto\sfts5\sto\ssupport\sdetail=none\smode.\sAs\sof\sthis\scommit,\smany\scases\sare\sstill\sbroken.
+D 2015-12-30T19:58:57.068
F Makefile.in 28bcd6149e050dff35d4dcfd97e890cd387a499d
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 5fff077fcc46de7714ed6eebb6159a4c00eab751
F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
F ext/fts5/fts5_expr.c 08ee4f342a2b1fd82a5dccd0b9b3cde4921a8be5
-F ext/fts5/fts5_hash.c 8b510868502ec31119409fc7022edc37c27b5c40
-F ext/fts5/fts5_index.c ca8310eaa286e6c1e4c4581a420b7c3a1ec6302c
+F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
+F ext/fts5/fts5_index.c 94dec3c322e9179fb0dd3239d6fe9555b9c621ec
F ext/fts5/fts5_main.c 23f3912ff44172859c771eb55cb57778fd662e89
F ext/fts5/fts5_storage.c 076a3356536a8831eb6e554195171d5c249cd179
F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
F ext/fts5/test/fts5simple.test a599b7577bc3827a9a678add3b43d8b818b93456
+F ext/fts5/test/fts5simple2.test 70b98039c8a73341bb4a632adb3018e1017e3beb
F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d
-R 9b7a210ea600814406fb8e0040363d54
+P 8c30605bcd0a78a5015948171145bc6f640b8358
+R 393cc400b4b232cee04e2e069dffc8fe
U dan
-Z 50bb7fa01e78a4726d7b3d5426b2a1ed
+Z c9f782e6b3f3ae84d42c40bf56b4157b
-8c30605bcd0a78a5015948171145bc6f640b8358
\ No newline at end of file
+ac8f4cf0cede6bcbe47eeefb85d80a27e9278212
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
