From dafeda754b98da699abbcee87728b67e9b009917 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Sep 2015 19:51:05 +0000 Subject: [PATCH] Improve test coverage of fts5_index.c. FossilOrigin-Name: c1f76686cee3918b1be785a4071d68cb3afda0ef --- ext/fts5/fts5_index.c | 33 ++---- ext/fts5/test/fts5corrupt3.test | 196 ++++++++++++++++++++++++++++++-- ext/fts5/test/fts5dlidx.test | 57 ++++++++++ ext/fts5/test/fts5fault7.test | 48 +++++++- manifest | 20 ++-- manifest.uuid | 2 +- 6 files changed, 313 insertions(+), 43 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e36a5388fd..c9fb6207ec 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -1427,7 +1427,6 @@ static void fts5SegIterNextPage( fts5DataRelease(pIter->pLeaf); pIter->iLeafPgno++; if( pIter->pNextLeaf ){ - assert( pIter->iLeafPgno<=pSeg->pgnoLast ); pIter->pLeaf = pIter->pNextLeaf; pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ @@ -1659,10 +1658,8 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ )); if( pNew ){ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ - if( pIter->iTermLeafOffsetszLeaf ){ - pIter->pLeaf = pNew; - pIter->iLeafOffset = pIter->iTermLeafOffset; - } + pIter->pLeaf = pNew; + pIter->iLeafOffset = pIter->iTermLeafOffset; }else{ int iRowidOff; iRowidOff = fts5LeafFirstRowidOff(pNew); @@ -3168,7 +3165,7 @@ static void fts5WriteAppendTerm( Fts5PageWriter *pPage = &pWriter->writer; Fts5Buffer *pPgidx = &pWriter->writer.pgidx; - if( p->rc ) return; + assert( p->rc==SQLITE_OK ); assert( pPage->buf.n>=4 ); assert( pPage->buf.n>4 || pWriter->bFirstTermInPage ); @@ -3315,16 +3312,12 @@ static void fts5WriteFinish( int i; Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ - if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ - *pnLeaf = 0; - }else{ - if( pLeaf->buf.n>4 ){ - fts5WriteFlushLeaf(p, pWriter); - } - *pnLeaf = pLeaf->pgno-1; - - fts5WriteFlushBtree(p, pWriter); + assert( pLeaf->pgno>=1 ); + if( pLeaf->buf.n>4 ){ + fts5WriteFlushLeaf(p, pWriter); } + *pnLeaf = pLeaf->pgno-1; + fts5WriteFlushBtree(p, pWriter); } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); @@ -3763,9 +3756,8 @@ static void fts5FlushOneHash(Fts5Index *p){ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm); - if( writer.bFirstRowidInPage==0 - && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) - ){ + assert( writer.bFirstRowidInPage==0 ); + if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ @@ -3773,8 +3765,6 @@ static void fts5FlushOneHash(Fts5Index *p){ i64 iDelta = 0; int iOff = 0; - /* writer.bFirstRowidInPage = 0; */ - /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ @@ -4155,7 +4145,7 @@ static void fts5SetupPrefixIter( i64 iRowid = fts5MultiIterRowid(p1); int nTerm; const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); - assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); + assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( nTerm0 && iRowid<=iLastRowid ){ @@ -5290,7 +5280,6 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ if( iOff100"] { + incr i + do_test $tn.$i { + db eval BEGIN + db eval {DELETE FROM t1_data WHERE rowid = $::rowid} + list [ + catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg + ] $msg + } {1 {database disk image is malformed}} + catch { db eval ROLLBACK } + } +} + +do_3_test 3.2 + +do_execsql_test 3.3 { + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); +} {} + +do_3_test 3.4 + +do_test 3.5 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + } + for {set i 0} {$i < 1000} {incr i} { + set rnd [expr int(rand() * 1000)] + set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]] + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.6 + +do_test 3.7 { + execsql { + INSERT INTO t1(t1, rank) VALUES('pgsz', 40); + INSERT INTO t1 SELECT x FROM t1; + INSERT INTO t1(t1) VALUES('optimize'); + } +} {} + +do_3_test 3.8 + +do_test 3.9 { + execsql { + DELETE FROM t1; + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } +} {} + +do_3_test 3.10 + +} + +#------------------------------------------------------------------------- +# Test that segments that end unexpectedly are identified as corruption. +# +reset_db +do_test 4.0 { + execsql { + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + } + for {set i 0} {$i < 100} {incr i} { + set rnd [expr int(rand() * 100)] + set doc "x[string repeat $rnd 20]" + execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) } + } + execsql { INSERT INTO t1(t1) VALUES('optimize') } +} {} + +set nErr 0 +for {set i 1} {1} {incr i} { + set struct [db one {SELECT block FROM t1_data WHERE id=10}] + binary scan $struct c* var + set end [lindex $var end] + if {$end<=$i} break + lset var end [expr $end - $i] + set struct [binary format c* $var] + db eval { + BEGIN; + UPDATE t1_data SET block = $struct WHERE id=10; + } + do_test 4.1.$i { + incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }] + set {} {} + } {} + catch { db eval ROLLBACK } +} +do_test 4.1.x { expr $nErr>45 } 1 + +#------------------------------------------------------------------------- +# + +# The first argument passed to this command must be a binary blob +# containing an FTS5 leaf page. This command returns a copy of this +# blob, with the pgidx of the leaf page replaced by a single varint +# containing value $iVal. +# +proc rewrite_pgidx {blob iVal} { + binary scan $blob SS off1 szLeaf + if {$iVal<0 || $iVal>=128} { + error "$iVal out of range!" + } else { + set pgidx [binary format c $iVal] + } + + binary format a${szLeaf}a* $blob $pgidx +} + +reset_db +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE x1 USING fts5(x); + INSERT INTO x1(x1, rank) VALUES('pgsz', 40); + BEGIN; + INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj'); + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1 SELECT x FROM x1; + INSERT INTO x1(x1) VALUES('optimize'); + COMMIT; +} + +#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b } +# +db func rewrite_pgidx rewrite_pgidx +set i 0 +foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] { + foreach val {2 100} { + do_test 5.2.$val.[incr i] { + catchsql { + BEGIN; + UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid; + SELECT rowid FROM x1 WHERE x1 MATCH 'xa*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xb*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xc*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xd*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xe*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xf*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xg*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xh*'; + SELECT rowid FROM x1 WHERE x1 MATCH 'xi*'; + } + set {} {} + } {} + catch { db eval ROLLBACK } + } +} + + sqlite3_fts5_may_be_corrupt 0 finish_test diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 07d7e2baeb..86fb8bb930 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -26,6 +26,8 @@ if { $tcl_platform(wordSize)<8 } { return } +if 1 { + proc do_fb_test {tn sql res} { set res2 [lsort -integer -decr $res] uplevel [list do_execsql_test $tn.1 $sql $res] @@ -128,5 +130,60 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} { do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] +} + +#-------------------------------------------------------------------- +# +reset_db + +set ::vocab [list \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ + IteratorpItercurrentlypointstothefirstrowidofadoclist \ + Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ + pageIfthecurrenttermisthelasttermonthepageloadthe \ + doclistindexfromdiskandinitializeaniteratoratpIterpDlidx \ +] +proc rnddoc {} { + global vocab + set nVocab [llength $vocab] + set ret [list] + for {set i 0} {$i < 64} {incr i} { + lappend ret [lindex $vocab [expr $i % $nVocab]] + } + set ret +} +db func rnddoc rnddoc + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE abc USING fts5(a); + INSERT INTO abc(abc, rank) VALUES('pgsz', 32); + + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + INSERT INTO abc VALUES ( rnddoc() ); + + INSERT INTO abc SELECT rnddoc() FROM abc; + INSERT INTO abc SELECT rnddoc() FROM abc; +} + +do_execsql_test 3.2 { + INSERT INTO abc(abc) VALUES('integrity-check'); + INSERT INTO abc(abc) VALUES('optimize'); + INSERT INTO abc(abc) VALUES('integrity-check'); +} + +set v [lindex $vocab 0] +set i 0 +foreach v $vocab { + do_execsql_test 3.3.[incr i] { + SELECT rowid FROM abc WHERE abc MATCH $v + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} +} + + finish_test diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test index 381f9ea6ad..9bce1c7caf 100644 --- a/ext/fts5/test/fts5fault7.test +++ b/ext/fts5/test/fts5fault7.test @@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl -set testprefix fts5fault2 +set testprefix fts5fault7 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts5 { @@ -22,6 +22,8 @@ ifcapable !fts5 { return } +if 1 { + #------------------------------------------------------------------------- # Test fault-injection on a query that uses xColumnSize() on columnsize=0 # table. @@ -41,5 +43,49 @@ do_faultsim_test 1 -faults oom* -body { faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} } +} + +#------------------------------------------------------------------------- +# Test fault-injection when a segment is promoted. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t2 USING fts5(a); + INSERT INTO t2(t2, rank) VALUES('automerge', 0); + INSERT INTO t2(t2, rank) VALUES('crisismerge', 4); + INSERT INTO t2(t2, rank) VALUES('pgsz', 40); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); + + INSERT INTO t2 VALUES('a b c'); + INSERT INTO t2 VALUES('d e f'); + INSERT INTO t2 VALUES('f e d'); + INSERT INTO t2 VALUES('c b a'); +} {} + +faultsim_save_and_close +do_faultsim_test 1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + INSERT INTO t2 VALUES('c d c g g f'); + INSERT INTO t2 VALUES('c d g b f d'); + INSERT INTO t2 VALUES('c c f d e d'); + INSERT INTO t2 VALUES('e a f c e f'); + INSERT INTO t2 VALUES('c g f b b d'); + INSERT INTO t2 VALUES('d a g a b b'); + INSERT INTO t2 VALUES('e f a b c e'); + INSERT INTO t2 VALUES('e c a g c d'); + INSERT INTO t2 VALUES('g b d d e b'); + INSERT INTO t2 VALUES('e a d a e d'); + } +} -body { + db eval COMMIT +} -test { + faultsim_test_result {0 {}} +} + finish_test diff --git a/manifest b/manifest index 836476cb10..1f1b10325e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Testability\simprovements\sfor\sthe\sONEPASS_MULTI\senhancement. -D 2015-09-14T19:26:37.988 +C Improve\stest\scoverage\sof\sfts5_index.c. +D 2015-09-14T19:51:05.852 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695 F ext/fts5/fts5_expr.c 667faaf14a69a5683ac383acdc8d942cf32c3f93 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246 -F ext/fts5/fts5_index.c 62a682a70ea2e84fa67c7786ead892b201116ad1 +F ext/fts5/fts5_index.c fae675f4d60cfff93b31f85576fddb6ade7d344e F ext/fts5/fts5_main.c 3fa906f6c0177caf8f82862bc70f37b28bb3305c F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37 @@ -146,8 +146,8 @@ F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1 F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62 F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c -F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3 -F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1 +F ext/fts5/test/fts5corrupt3.test 618a965b4fd2859f9ddb72413c9828a23828704b +F ext/fts5/test/fts5dlidx.test 77259f6d8d671b486113b157bd30107ca9d6b0f6 F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0 F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43 @@ -157,7 +157,7 @@ F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7 F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875 F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423 -F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4 +F ext/fts5/test/fts5fault7.test f62ed4d98f137eb03f1db94d1fa41b17a771d971 F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6 F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145 @@ -1387,7 +1387,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 379455af9fdfb299a857d46f35f0a76ed6daa08a -R 240d53905959924a39a1969f0c2c4fad -U drh -Z db76ee6590838fdbed88f2282d6173bf +P d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2 +R a767de21459f1cfc9798aa9a41d99081 +U dan +Z 579f9e68a86182329093b3386940c228 diff --git a/manifest.uuid b/manifest.uuid index 93b327f2ec..7a9c2af077 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2df93f26fc0cf6fe01acfeaf2946972e9d8cca2 \ No newline at end of file +c1f76686cee3918b1be785a4071d68cb3afda0ef \ No newline at end of file -- 2.47.2