From: dan Date: Mon, 4 Sep 2023 16:48:31 +0000 (+0000) Subject: Add tests for, and source code comments to, fts5. No changes to code. X-Git-Tag: version-3.44.0~229 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=dd24b1c8a14f11587c3ea61142fd76ceaff912bb;p=thirdparty%2Fsqlite.git Add tests for, and source code comments to, fts5. No changes to code. FossilOrigin-Name: b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a --- diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 1b3f0fef99..aa16b0174e 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5048,7 +5048,6 @@ static void fts5DoSecureDelete( int iPgIdx = pSeg->pLeaf->szLeaf; u64 iDelta = 0; - u64 iNextDelta = 0; int iNextOff = 0; int iOff = 0; int nIdx = 0; @@ -5056,8 +5055,6 @@ static void fts5DoSecureDelete( int bLastInDoclist = 0; int iIdx = 0; int iStart = 0; - int iKeyOff = 0; - int iPrevKeyOff = 0; int iDelKeyOff = 0; /* Offset of deleted key, if any */ nIdx = nPg-iPgIdx; @@ -5082,10 +5079,21 @@ static void fts5DoSecureDelete( ** This block sets the following variables: ** ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. */ { - int iSOP; + int iSOP; /* Start-Of-Position-list */ if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ iStart = pSeg->iTermLeafOffset; }else{ @@ -5121,14 +5129,20 @@ static void fts5DoSecureDelete( } iOff = iStart; + + /* Set variable bLastInDoclist to true if this entry happens to be + ** the last rowid in the doclist for its term. */ if( iNextOff>=iPgIdx ){ int pgno = pSeg->iLeafPgno+1; fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); iNextOff = iPgIdx; }else{ - /* Set bLastInDoclist to true if the entry being removed is the last + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last ** in its doclist. */ - for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset ){ /* The entry being removed was the only position list in its ** doclist. Therefore the term needs to be removed as well. */ int iKey = 0; - for(iIdx=0, iKeyOff=0; iIdx(u32)iStart ) break; iKeyOff += iVal; } + assert_nc( iKey>=1 ); + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ int nPrefix = 0; int nSuffix = 0; int nPrefix2 = 0; @@ -5198,80 +5229,81 @@ static void fts5DoSecureDelete( } } }else if( iStart==4 ){ - int iPgno; - - assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); - /* The entry being removed may be the only position list in - ** its doclist. */ - for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ - Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); - int bEmpty = (pPg && pPg->nn==4); - fts5DataRelease(pPg); - if( bEmpty==0 ) break; - } - - if( iPgno==pSeg->iTermLeafPgno ){ - i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); - Fts5Data *pTerm = fts5DataRead(p, iId); - if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ - u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; - int nTermIdx = pTerm->nn - pTerm->szLeaf; - int iTermIdx = 0; - int iTermOff = 0; - - while( 1 ){ - u32 iVal = 0; - int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); - iTermOff += iVal; - if( (iTermIdx+nByte)>=nTermIdx ) break; - iTermIdx += nByte; - } - nTermIdx = iTermIdx; + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; - memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); - fts5PutU16(&pTerm->p[2], iTermOff); + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); - fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); - if( nTermIdx==0 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); - } + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); } - fts5DataRelease(pTerm); } + fts5DataRelease(pTerm); } + } - if( p->rc==SQLITE_OK ){ - const int nMove = nPg - iNextOff; - int nShift = 0; - - memmove(&aPg[iOff], &aPg[iNextOff], nMove); - iPgIdx -= (iNextOff - iOff); - nPg = iPgIdx; - fts5PutU16(&aPg[2], iPgIdx); - - nShift = iNextOff - iOff; - for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ - iKeyOff -= nShift; - nShift = 0; - } - nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); - iPrevKeyOff = iKeyOff; - } - } + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ + int iKeyOff = 0; + int iPrevKeyOff = 0; - if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ - fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + for(iIdx=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; } + } - assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); - fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); } - sqlite3_free(aIdx); + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); } /* diff --git a/ext/fts5/test/fts5secure6.test b/ext/fts5/test/fts5secure6.test index 5ab17c4f32..ffb10cb24a 100644 --- a/ext/fts5/test/fts5secure6.test +++ b/ext/fts5/test/fts5secure6.test @@ -70,5 +70,26 @@ do_execsql_test 2.2 { SELECT rowid FROM t1('def') } {-100000 -99999 9223372036854775800} +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) +} + +do_execsql_test 2.1 { + BEGIN; + INSERT INTO t1(rowid, x) + VALUES(51869, 'when whenever where weress what turn'), + (51871, 'to were'); + COMMIT; +} + +do_execsql_test 3.2 { + DELETE FROM t1 WHERE rowid=51871; + INSERT INTO t1(t1) VALUES('integrity-check'); +} + finish_test diff --git a/ext/fts5/test/fts5secure7.test b/ext/fts5/test/fts5secure7.test new file mode 100644 index 0000000000..16a044f538 --- /dev/null +++ b/ext/fts5/test/fts5secure7.test @@ -0,0 +1,116 @@ +# 2023 Feb 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# TESTRUNNER: slow +# + +source [file join [file dirname [info script]] fts5_common.tcl] +ifcapable !fts5 { finish_test ; return } +set ::testprefix fts5secure7 + + +set NVOCAB 500 +set NDOC [expr 1000] + +set NREP 100 +set nDeletePerRep [expr 5] + +set VOCAB [list] + +proc select_one {list} { + set n [llength $list] + lindex $list [expr {abs(int(rand()*$n))}] +} + +proc init_vocab {} { + set L [split "abcdefghijklmnopqrstuvwxyz" {}] + set nL [llength $L] + for {set i 0} {$i < $::NVOCAB} {incr i} { + set n [expr {6 + int(rand()*8)}] + set word "" + for {set j 0} {$j < $n} {incr j} { + append word [select_one $L] + } + lappend ::VOCAB $word + } +} + +proc get_word {} { + select_one $::VOCAB +} + +proc get_document {nWord} { + set ret [list] + for {set i 0} {$i < $nWord} {incr i} { + lappend ret [get_word] + } + return $ret +} + +init_vocab + +db func document [list get_document 12] + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); +} +do_execsql_test 1.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} + +for {set iRep 0} {$iRep < $NREP} {incr iRep} { + set lRowid [db eval {SELECT rowid FROM t1}] + for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} { + set idx [select_one $lRowid] + db eval { + DELETE FROM t1 WHERE rowid=$idx + } + } + db eval { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep + ) + INSERT INTO t1 SELECT document() FROM s; + } + do_execsql_test 1.2.$iRep { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +reset_db +db func document [list get_document 12] +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE t1 USING fts5(body); + INSERT INTO t1(t1, rank) VALUES('secure-delete', 1); + INSERT INTO t1(t1, rank) VALUES('pgsz', 128); +} +do_execsql_test 2.1 { + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC + ) + INSERT INTO t1 SELECT document() FROM s; +} +for {set ii 0} {$ii < $NDOC} {incr ii} { + set lRowid [db eval {SELECT rowid FROM t1}] + set idx [select_one $lRowid] + db eval { DELETE FROM t1 WHERE rowid=$idx } + do_execsql_test 2.2.$ii { + INSERT INTO t1(t1) VALUES('integrity-check'); + } +} + +finish_test + + diff --git a/manifest b/manifest index e07de16aa1..fa64b06f89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\smake\sassumptions\sabout\sthe\sbyteorder\sof\sPowerPC\sprocessors. -D 2023-09-04T12:50:17.388 +C Add\stests\sfor,\sand\ssource\scode\scomments\sto,\sfts5.\sNo\schanges\sto\scode. +D 2023-09-04T16:48:31.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -94,7 +94,7 @@ F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b7292 F ext/fts5/fts5_config.c 054359543566cbff1ba65a188330660a5457299513ac71c53b3a07d934c7b081 F ext/fts5/fts5_expr.c bd3b81ce669c4104e34ffe66570af1999a317b142c15fccb112de9fb0caa57a6 F ext/fts5/fts5_hash.c 65e7707bc8774706574346d18c20218facf87de3599b995963c3e6d6809f203d -F ext/fts5/fts5_index.c 77bd70d50cb8397f3d8465cc4894dcdac75aa5e1fb3bbc5a4a5bc15e12fced97 +F ext/fts5/fts5_index.c 47c290589fa38f6a1860b4fc26716ed30d79ddc283b38813d1c8c1d702108ab8 F ext/fts5/fts5_main.c 7070031993ba5b5d89b13206ec4ef624895f2f7c0ec72725913d301e4d382445 F ext/fts5/fts5_storage.c 3c9b41fce41b6410f2e8f82eb035c6a29b2560483f773e6dc98cf3cb2e4ddbb5 F ext/fts5/fts5_tcl.c b1445cbe69908c411df8084a10b2485500ac70a9c747cdc8cda175a3da59d8ae @@ -204,7 +204,8 @@ F ext/fts5/test/fts5secure2.test 2e961d7eef939f294c56b5d895cac7f1c3a60b934ee2cfd F ext/fts5/test/fts5secure3.test c7e1080a6912f2a3ac68f2e05b88b72a99de38543509b2bbf427cac5c9c1c610 F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b9846cf7fc8337b67c97 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a -F ext/fts5/test/fts5secure6.test a0a28cfb9bf9721408b65b5d7c7ce369af3d688e273da24d101c25d60cdce05c +F ext/fts5/test/fts5secure6.test 120feecc8c55b4774f858721e6c62c2094b059ecbcfd7fdc24bde886f55ef6ca +F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a F ext/fts5/test/fts5securefault.test dbca2b6a1c16700017f5051138991b705410889933f2a37c57ae8a23b296b10b F ext/fts5/test/fts5simple.test a298670508c1458b88ce6030440f26a30673931884eb5f4094ac1773b3ba217b F ext/fts5/test/fts5simple2.test 258a1b0c590409bfa5271e872c79572b319d2a56554d0585f68f146a0da603f0 @@ -2115,8 +2116,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2aef9af3dd493a82ba090ccb8ab1b8974349392ec3f1c35db7a3dbbec41b5c51 -R a87cf75daed42b21bba72eb7f67c969b -U drh -Z 72c7bea784c6c2657adbd0596be177aa +P 4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61 +R e68bce01327834d099d97be2fe3709dd +U dan +Z a7ec9ec1221680dfc975891082f06d30 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cbc82d7c1b..2fb3beaf05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a2498fed4c5436fbcd4179db85e2741fdab37d42b0eebf12f41ec4573ce2c61 \ No newline at end of file +b12afff4efe4de84388d102060e8c312abd2f9eca8270b3c0f01ac6d1f2e329a \ No newline at end of file