#define FTS5_MAX_LEVEL 64
-#define FTS5_MERGE_TOMBSTONE_WEIGHT 5
-
/*
** There are two versions of the format used for the structure record:
**
i64 iWriteRowid; /* Rowid for current doc being written */
int bDelete; /* Current write is a delete */
int nContentlessDelete; /* Number of contentless delete ops */
+ int nPendingRow; /* Number of INSERT in hash table */
/* Error state. */
int rc; /* Current error code */
u64 iOrigin1;
u64 iOrigin2;
int nPgTombstone; /* Number of tombstone hash table pages */
+ i64 nEntryTombstone; /* Number of tombstone entries that "count" */
+ i64 nEntry; /* Number of rows in this segment */
};
struct Fts5StructureLevel {
int nMerge; /* Number of segments in incr-merge */
i += fts5GetVarint(&pData[i], &pSeg->iOrigin1);
i += fts5GetVarint(&pData[i], &pSeg->iOrigin2);
i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone);
+ i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone);
+ i += fts5GetVarint(&pData[i], &pSeg->nEntry);
nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2);
}
if( pSeg->pgnoLast<pSeg->pgnoFirst ){
assert( pLvl->nMerge<=pLvl->nSeg );
for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
+ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast);
if( pStruct->nOriginCntr>0 ){
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin1);
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iOrigin2);
- fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nPgTombstone);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone);
+ fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry);
}
}
}
if( p->pHash ){
sqlite3Fts5HashClear(p->pHash);
p->nPendingData = 0;
+ p->nPendingRow = 0;
}
p->nContentlessDelete = 0;
}
/* Read input from all segments in the input level */
nInput = pLvl->nSeg;
- /* Set the range of origins that will go into the output segment */
+ /* Set the range of origins that will go into the output segment. */
if( pStruct->nOriginCntr>0 ){
pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
int i;
/* Remove the redundant segments from the %_data table */
+ assert( pSeg->nEntry==0 );
for(i=0; i<nInput; i++){
- fts5DataRemoveSegment(p, &pLvl->aSeg[i]);
+ Fts5StructureSegment *pOld = &pLvl->aSeg[i];
+ pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone);
+ fts5DataRemoveSegment(p, pOld);
}
/* Remove the redundant segments from the input level */
if( pnRem ) *pnRem -= writer.nLeafWritten;
}
+/*
+** If this is not a contentless_delete=1 table, or if the 'delete-automerge'
+** configuration option is set to 0, then this function always returns -1.
+** Otherwise, it searches the structure object passed as the second argument
+** for a level suitable for merging due to having a large number of
+** tombstones in the tombstone hash. If one is found, its index is returned.
+** Otherwise, if there is no suitable level, -1.
+*/
+static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
+ Fts5Config *pConfig = p->pConfig;
+ int iRet = -1;
+ if( pConfig->bContentlessDelete && pConfig->nDeleteAutomerge>0 ){
+ int ii;
+ int nBest = 0;
+
+ for(ii=0; ii<pStruct->nLevel; ii++){
+ Fts5StructureLevel *pLvl = &pStruct->aLevel[ii];
+ i64 nEntry = 0;
+ i64 nTomb = 0;
+ int iSeg;
+ for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
+ nEntry += pLvl->aSeg[iSeg].nEntry;
+ nTomb += pLvl->aSeg[iSeg].nEntryTombstone;
+ }
+ assert( nEntry>0 || pLvl->nSeg==0 );
+ if( nEntry>0 ){
+ int nPercent = (nTomb * 100) / nEntry;
+ if( nPercent>=pConfig->nDeleteAutomerge && nPercent>nBest ){
+ iRet = ii;
+ nBest = nPercent;
+ }
+ }
+ }
+ }
+ return iRet;
+}
+
/*
** Do up to nPg pages of automerge work on the index.
**
int iBestLvl = 0; /* Level offering the most input segments */
int nBest = 0; /* Number of input segments on best level */
- /* Set iBestLvl to the level to read input segments from. */
+ /* Set iBestLvl to the level to read input segments from. Or to -1 if
+ ** there is no level suitable to merge segments from. */
assert( pStruct->nLevel>0 );
for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
- int nThisSeg = 0;
if( pLvl->nMerge ){
if( pLvl->nMerge>nBest ){
iBestLvl = iLvl;
- nBest = pLvl->nMerge;
+ nBest = nMin;
}
break;
}
- nThisSeg = pLvl->nSeg;
- if( bTombstone && nThisSeg ){
- int iSeg;
- int nPg = 0;
- int nTomb = 0;
- for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
- Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
- nPg += pSeg->pgnoLast;
- nTomb += pSeg->nPgTombstone;
- }
- nThisSeg += ((nTomb*FTS5_MERGE_TOMBSTONE_WEIGHT) / nPg);
- }
- if( nThisSeg>nBest ){
- nBest = nThisSeg;
+ if( pLvl->nSeg>nBest ){
+ nBest = pLvl->nSeg;
iBestLvl = iLvl;
}
}
-
- /* If nBest is still 0, then the index must be empty. */
-#ifdef SQLITE_DEBUG
- for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){
- assert( pStruct->aLevel[iLvl].nSeg==0 );
+ if( nBest<nMin ){
+ iBestLvl = fts5IndexFindDeleteMerge(p, pStruct);
}
-#endif
- if( nBest<nMin && pStruct->aLevel[iBestLvl].nMerge==0 ){
- if( bTombstone || p->pConfig->bContentlessDelete==0 ){
- break;
- }else{
- bTombstone = 1;
- continue;
- }
- }
+ if( iBestLvl<0 ) break;
bRet = 1;
fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
if( pStruct->nOriginCntr>0 ){
pSeg->iOrigin1 = pStruct->nOriginCntr;
pSeg->iOrigin2 = pStruct->nOriginCntr;
+ pSeg->nEntry = p->nPendingRow;
pStruct->nOriginCntr++;
}
pStruct->nSegment++;
*/
static void fts5IndexFlush(Fts5Index *p){
/* Unless it is empty, flush the hash table to disk */
- if( p->nPendingData || (p->nContentlessDelete && p->pConfig->nAutomerge>0) ){
+ if( p->nPendingData || p->nContentlessDelete ){
assert( p->pHash );
- p->nPendingData = 0;
fts5FlushOneHash(p);
+ p->nPendingData = 0;
+ p->nPendingRow = 0;
}
}
assert( p->rc==SQLITE_OK );
fts5IndexFlush(p);
+ assert( p->nContentlessDelete==0 );
pStruct = fts5StructureRead(p);
fts5StructureInvalidate(p);
** INSERT command.
*/
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
- Fts5Structure *pStruct = fts5StructureRead(p);
+ Fts5Structure *pStruct = 0;
+
+ fts5IndexFlush(p);
+ pStruct = fts5StructureRead(p);
if( pStruct ){
int nMin = p->pConfig->nUsermerge;
fts5StructureInvalidate(p);
p->iWriteRowid = iRowid;
p->bDelete = bDelete;
+ if( bDelete==0 ){
+ p->nPendingRow++;
+ }
return fts5IndexReturn(p);
}
Fts5Structure *pStruct;
pStruct = fts5StructureRead(p);
if( pStruct ){
+ int bFound = 0; /* True after pSeg->nEntryTombstone incr. */
int iLvl;
- for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+ for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
int iSeg;
- for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+ for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
+ if( bFound==0 ){
+ pSeg->nEntryTombstone++;
+ bFound = 1;
+ }
fts5IndexTombstoneAdd(p, pSeg, iRowid);
}
}
rc = sqlite3_declare_vtab(db,
"CREATE TABLE xyz("
"level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
- "npgtombstone, struct HIDDEN);"
+ "npgtombstone, nentrytombstone, nentry, struct HIDDEN);"
);
if( rc==SQLITE_OK ){
pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
pIdxInfo->idxNum = 0;
for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
if( p->usable==0 ) continue;
- if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==9 ){
+ if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){
rc = SQLITE_OK;
pIdxInfo->aConstraintUsage[i].omit = 1;
pIdxInfo->aConstraintUsage[i].argvIndex = 1;
case 5: /* leaf2 */
sqlite3_result_int(ctx, pSeg->pgnoLast);
break;
- case 6: /* loc1 */
- sqlite3_result_int(ctx, pSeg->iOrigin1);
+ case 6: /* origin1 */
+ sqlite3_result_int64(ctx, pSeg->iOrigin1);
break;
- case 7: /* loc2 */
- sqlite3_result_int(ctx, pSeg->iOrigin2);
+ case 7: /* origin2 */
+ sqlite3_result_int64(ctx, pSeg->iOrigin2);
break;
case 8: /* npgtombstone */
sqlite3_result_int(ctx, pSeg->nPgTombstone);
break;
+ case 9: /* nentrytombstone */
+ sqlite3_result_int64(ctx, pSeg->nEntryTombstone);
+ break;
+ case 10: /* nentry */
+ sqlite3_result_int64(ctx, pSeg->nEntry);
+ break;
}
return SQLITE_OK;
}
-C Integrate\scontentless\sdelete\swith\sauto-merge.
-D 2023-07-22T19:47:46.359
+C Add\sthe\sfts5\s'delete-automerge'\sinteger\soption.\sA\slevel\sis\seligible\sfor\sauto-merging\sif\sit\shas\sa\sgreater\sthan\sor\sequal\spercentage\sof\sits\sentries\sdeleted\sby\stombstones\sthan\sthe\s'delete-automerge'\soption.\sDefault\svalue\sis\s10.
+D 2023-07-24T19:13:06.235
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
-F ext/fts5/fts5Int.h f59c14f725ad0fcb8a81b9bf012e5021c6501bf43e73aa00b00d728e2ac7efaf
+F ext/fts5/fts5Int.h 7decc306406187d1826c5eba9b8e8e6661b85580e5da1203760c0c2de9bc4a5e
F ext/fts5/fts5_aux.c 572d5ec92ba7301df2fea3258576332f2f4d2dfd66d8263afd157d9deceac480
F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
-F ext/fts5/fts5_config.c 010fabcc0aaa0dfa76b19146e8bddf7de368933eeac01e294af6607447500caa
+F ext/fts5/fts5_config.c c35f3433586f9152af2f444356020bf5c0f2e1d0fae29e67ab45de81277d07c9
F ext/fts5/fts5_expr.c 2473c13542f463cae4b938c498d6193c90d38ea1a2a4f9849c0479736e50d24d
-F ext/fts5/fts5_hash.c 60224220ccfb2846b741b6dbb1b8872094ec6d87b3118c04244dafc83e6f9c40
-F ext/fts5/fts5_index.c 31b8c8dd6913d76d6d7755342e36816495e5ad177d253f6bf39e0efdb9dc31e0
+F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
+F ext/fts5/fts5_index.c f5d50e218db3d32dd12c83b3f700bf79ed7f24e3f60f43f5b56e62146e2c31b5
F ext/fts5/fts5_main.c 2f87ee44fdb21539c264541149f07f70e065d58f37420063e5ddef80ba0f5ede
F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
F ext/fts5/test/fts5content.test 213506436fb2c87567b8e31f6d43ab30aab99354cec74ed679f22aad0cdbf283
F ext/fts5/test/fts5contentless.test 9a42a86822670792ba632f5c57459addeb774d93b29d5e6ddae08faa64c2b6d9
F ext/fts5/test/fts5contentless2.test 12c778d134a121b8bad000fbf3ae900d53226fee840ce36fe941b92737f1fda7
-F ext/fts5/test/fts5contentless3.test cd3b8332c737d1d6f28e04d6338876c79c22815b8ecd34fb677409a013a45224
-F ext/fts5/test/fts5contentless4.test 3b11ccbbe928d45eb8f985c0137a8fe2c69b70b940b10de31540040de5674311
+F ext/fts5/test/fts5contentless3.test 487dce16b6677f68b44d7cbd158b9b7275d25e2c14d713f9188d9645bb699286
+F ext/fts5/test/fts5contentless4.test 4403cbbbb5021b36b24d914addb22a782699d1a03a98fede705793f7184dbcd8
F ext/fts5/test/fts5corrupt.test 77ae6f41a7eba10620efb921cf7dbe218b0ef232b04519deb43581cb17a57ebe
F ext/fts5/test/fts5corrupt2.test 7453752ba12ce91690c469a6449d412561cc604b1dec994e16ab132952e7805f
F ext/fts5/test/fts5corrupt3.test 7da9895dafa404efd20728f66ff4b94399788bdc042c36fe2689801bba2ccd78
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P e61c9b083f5e0b6b6ee18f9394581ad816f445dbfb72ed1fe954f4182755a576
-R 223fbfa40ff414b8abee5011e5d1533a
+P 85c1589ab1fc69d1eef4bbc1bdefa2b10af5f6b9c08e813130b93829b592f416
+R 067f23316c8bda4fd7b0c9766119547d
U dan
-Z 8cb202caf797f67b895f2a9a207e2618
+Z b4aabcdd1189dc9cd9c674268a107cec
# Remove this line to create a well-formed Fossil manifest.