int iPgIdx = pSeg->pLeaf->szLeaf;
u64 iDelta = 0;
- u64 iNextDelta = 0;
int iNextOff = 0;
int iOff = 0;
int nIdx = 0;
int bLastInDoclist = 0;
int iIdx = 0;
int iStart = 0;
- int iKeyOff = 0;
- int iPrevKeyOff = 0;
int iDelKeyOff = 0; /* Offset of deleted key, if any */
nIdx = nPg-iPgIdx;
** This block sets the following variables:
**
** iStart:
+ ** The offset of the first byte of the rowid or delta-rowid
+ ** value for the doclist entry being removed.
+ **
** iDelta:
+ ** The value of the rowid or delta-rowid value for the doclist
+ ** entry being removed.
+ **
+ ** iNextOff:
+ ** The offset of the next entry following the position list
+ ** for the one being removed. If the position list for this
+ ** entry overflows onto the next leaf page, this value will be
+ ** greater than pLeaf->szLeaf.
*/
{
- int iSOP;
+ int iSOP; /* Start-Of-Position-list */
if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
iStart = pSeg->iTermLeafOffset;
}else{
}
iOff = iStart;
+
+ /* Set variable bLastInDoclist to true if this entry happens to be
+ ** the last rowid in the doclist for its term. */
if( iNextOff>=iPgIdx ){
int pgno = pSeg->iLeafPgno+1;
fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
iNextOff = iPgIdx;
}else{
- /* Set bLastInDoclist to true if the entry being removed is the last
+ /* Loop through the page-footer. If iNextOff (offset of the
+ ** entry following the one we are removing) is equal to the
+ ** offset of a key on this page, then the entry is the last
** in its doclist. */
- for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+ int iKeyOff = 0;
+ for(iIdx=0; iIdx<nIdx; /* no-op */){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
iKeyOff += iVal;
}
}
- if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+ /* If this is (a) the first rowid on a page and (b) is not followed by
+ ** another position list on the same page, set the "first-rowid" field
+ ** of the header to 0. */
+ if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
fts5PutU16(&aPg[0], 0);
}
if( bLastInDoclist==0 ){
if( iNextOff!=iPgIdx ){
+ u64 iNextDelta = 0;
iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
}
}else if(
- iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno
+ pSeg->iLeafPgno==pSeg->iTermLeafPgno
+ && iStart==pSeg->iTermLeafOffset
){
/* The entry being removed was the only position list in its
** doclist. Therefore the term needs to be removed as well. */
int iKey = 0;
- for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+ int iKeyOff = 0;
+
+ /* Set iKeyOff to the offset of the term that will be removed - the
+ ** last offset in the footer that is not greater than iStart. */
+ for(iIdx=0; iIdx<nIdx; iKey++){
u32 iVal = 0;
iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
if( (iKeyOff+iVal)>(u32)iStart ) break;
iKeyOff += iVal;
}
+ assert_nc( iKey>=1 );
+ /* Set iDelKeyOff to the value of the footer entry to remove from
+ ** the page. */
iDelKeyOff = iOff = iKeyOff;
+
if( iNextOff!=iPgIdx ){
+ /* This is the only position-list associated with the term, and there
+ ** is another term following it on this page. So the subsequent term
+ ** needs to be moved to replace the term associated with the entry
+ ** being removed. */
int nPrefix = 0;
int nSuffix = 0;
int nPrefix2 = 0;
}
}
}else if( iStart==4 ){
- int iPgno;
-
- assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
- /* The entry being removed may be the only position list in
- ** its doclist. */
- for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
- Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
- int bEmpty = (pPg && pPg->nn==4);
- fts5DataRelease(pPg);
- if( bEmpty==0 ) break;
- }
-
- if( iPgno==pSeg->iTermLeafPgno ){
- i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
- Fts5Data *pTerm = fts5DataRead(p, iId);
- if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
- u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
- int nTermIdx = pTerm->nn - pTerm->szLeaf;
- int iTermIdx = 0;
- int iTermOff = 0;
-
- while( 1 ){
- u32 iVal = 0;
- int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
- iTermOff += iVal;
- if( (iTermIdx+nByte)>=nTermIdx ) break;
- iTermIdx += nByte;
- }
- nTermIdx = iTermIdx;
+ int iPgno;
+
+ assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+ /* The entry being removed may be the only position list in
+ ** its doclist. */
+ for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+ Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+ int bEmpty = (pPg && pPg->nn==4);
+ fts5DataRelease(pPg);
+ if( bEmpty==0 ) break;
+ }
+
+ if( iPgno==pSeg->iTermLeafPgno ){
+ i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+ Fts5Data *pTerm = fts5DataRead(p, iId);
+ if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+ u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+ int nTermIdx = pTerm->nn - pTerm->szLeaf;
+ int iTermIdx = 0;
+ int iTermOff = 0;
+
+ while( 1 ){
+ u32 iVal = 0;
+ int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+ iTermOff += iVal;
+ if( (iTermIdx+nByte)>=nTermIdx ) break;
+ iTermIdx += nByte;
+ }
+ nTermIdx = iTermIdx;
- memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
- fts5PutU16(&pTerm->p[2], iTermOff);
+ memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+ fts5PutU16(&pTerm->p[2], iTermOff);
- fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
- if( nTermIdx==0 ){
- fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
- }
+ fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+ if( nTermIdx==0 ){
+ fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
}
- fts5DataRelease(pTerm);
}
+ fts5DataRelease(pTerm);
}
+ }
- if( p->rc==SQLITE_OK ){
- const int nMove = nPg - iNextOff;
- int nShift = 0;
-
- memmove(&aPg[iOff], &aPg[iNextOff], nMove);
- iPgIdx -= (iNextOff - iOff);
- nPg = iPgIdx;
- fts5PutU16(&aPg[2], iPgIdx);
-
- nShift = iNextOff - iOff;
- for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
- u32 iVal = 0;
- iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
- iKeyOff += iVal;
- if( iKeyOff!=iDelKeyOff ){
- if( iKeyOff>iOff ){
- iKeyOff -= nShift;
- nShift = 0;
- }
- nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
- iPrevKeyOff = iKeyOff;
- }
- }
+ if( p->rc==SQLITE_OK ){
+ const int nMove = nPg - iNextOff; /* Number of bytes to move */
+ int nShift = iNextOff - iOff; /* Distance to move them */
+ int iKeyOff = 0;
+ int iPrevKeyOff = 0;
- if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
- fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
+ memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+ iPgIdx -= nShift;
+ nPg = iPgIdx;
+ fts5PutU16(&aPg[2], iPgIdx);
+
+ for(iIdx=0; iIdx<nIdx; /* no-op */){
+ u32 iVal = 0;
+ iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+ iKeyOff += iVal;
+ if( iKeyOff!=iDelKeyOff ){
+ if( iKeyOff>iOff ){
+ iKeyOff -= nShift;
+ nShift = 0;
+ }
+ nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
+ iPrevKeyOff = iKeyOff;
}
+ }
- assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
- fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+ if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+ fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
}
- sqlite3_free(aIdx);
+
+ assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
+ fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg);
+ }
+ sqlite3_free(aIdx);
}
/*
--- /dev/null
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure7
+
+
+set NVOCAB 500
+set NDOC [expr 1000]
+
+set NREP 100
+set nDeletePerRep [expr 5]
+
+set VOCAB [list]
+
+proc select_one {list} {
+ set n [llength $list]
+ lindex $list [expr {abs(int(rand()*$n))}]
+}
+
+proc init_vocab {} {
+ set L [split "abcdefghijklmnopqrstuvwxyz" {}]
+ set nL [llength $L]
+ for {set i 0} {$i < $::NVOCAB} {incr i} {
+ set n [expr {6 + int(rand()*8)}]
+ set word ""
+ for {set j 0} {$j < $n} {incr j} {
+ append word [select_one $L]
+ }
+ lappend ::VOCAB $word
+ }
+}
+
+proc get_word {} {
+ select_one $::VOCAB
+}
+
+proc get_document {nWord} {
+ set ret [list]
+ for {set i 0} {$i < $nWord} {incr i} {
+ lappend ret [get_word]
+ }
+ return $ret
+}
+
+init_vocab
+
+db func document [list get_document 12]
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(body);
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 1.1 {
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+ )
+ INSERT INTO t1 SELECT document() FROM s;
+}
+
+for {set iRep 0} {$iRep < $NREP} {incr iRep} {
+ set lRowid [db eval {SELECT rowid FROM t1}]
+ for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
+ set idx [select_one $lRowid]
+ db eval {
+ DELETE FROM t1 WHERE rowid=$idx
+ }
+ }
+ db eval {
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
+ )
+ INSERT INTO t1 SELECT document() FROM s;
+ }
+ do_execsql_test 1.2.$iRep {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+}
+
+reset_db
+db func document [list get_document 12]
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(body);
+ INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
+}
+do_execsql_test 2.1 {
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+ )
+ INSERT INTO t1 SELECT document() FROM s;
+}
+for {set ii 0} {$ii < $NDOC} {incr ii} {
+ set lRowid [db eval {SELECT rowid FROM t1}]
+ set idx [select_one $lRowid]
+ db eval { DELETE FROM t1 WHERE rowid=$idx }
+ do_execsql_test 2.2.$ii {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+ }
+}
+
+finish_test
+
+
-C Do\snot\smake\sassumptions\sabout\sthe\sbyteorder\sof\sPowerPC\sprocessors.
-D 2023-09-04T12:50:17.388
+C Add\stests\sfor,\sand\ssource\scode\scomments\sto,\sfts5.\sNo\schanges\sto\scode.
+D 2023-09-04T16:48:31.747
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081
F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6
F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d
-F ext/fts5/fts5_index.c 77bd70d50cb8397f3d8465cc4894dcdac75aa5e1fb3bbc5a4a5bc15e12fced97
+F ext/fts5/fts5_index.c 47c290589fa38f6a1860b4fc26716ed30d79ddc283b38813d1c8c1d702108ab8
F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445
F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5
F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae
F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610
F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97
F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
-F ext/fts5/test/fts5secure6.test a0a28cfb9bf9721408b65b5d7c7ce369af3d688e273da24d101c25d60cdce05c
+F ext/fts5/test/fts5secure6.test 120feecc8c55b4774f858721e6c62c2094b059ecbcfd7fdc24bde886f55ef6ca
+F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b
F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b
F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 2aef9af3dd493a82ba090ccb8ab1b8974349392ec3f1c35db7a3dbbec41b5c51
-R a87cf75daed42b21bba72eb7f67c969b
-U drh
-Z 72c7bea784c6c2657adbd0596be177aa
+P 4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61
+R e68bce01327834d099d97be2fe3709dd
+U dan
+Z a7ec9ec1221680dfc975891082f06d30
# Remove this line to create a well-formed Fossil manifest.
projects / thirdparty / sqlite.git / commitdiff
