From: dan Date: Fri, 15 May 2015 12:18:39 +0000 (+0000) Subject: Add test cases. And some fixes. X-Git-Tag: version-3.8.11~114^2~43 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=415ae725cb652b24d9630cf4003dbe99322ff154;p=thirdparty%2Fsqlite.git Add test cases. And some fixes. FossilOrigin-Name: adee788586197445672013d434e7ba47ce510b59 --- diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 6df3774ebb..e2796d9c78 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -2071,15 +2071,9 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ int pgnoLast = 0; if( pDlidx ){ - /* If the doclist-iterator is already at EOF, then the current doclist - ** contains no entries except those on the current page. */ - if( fts5DlidxIterEof(p, pDlidx)==0 ){ - int iSegid = pIter->pSeg->iSegid; - pgnoLast = fts5DlidxIterPgno(pDlidx); - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); - }else{ - pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); - } + int iSegid = pIter->pSeg->iSegid; + pgnoLast = fts5DlidxIterPgno(pDlidx); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); }else{ int iOff; /* Byte offset within pLeaf */ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ @@ -3430,8 +3424,7 @@ static void fts5WriteInit( pWriter->iSegid = iSegid; pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter)); - pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter)); - if( pWriter->aDlidx==0 ) return; + if( fts5WriteDlidxGrow(p, pWriter, 1) ) return; pWriter->nWriter = 1; pWriter->nDlidx = 1; pWriter->aWriter[0].pgno = 1; @@ -3902,7 +3895,7 @@ static void fts5FlushOneHash(Fts5Index *p){ /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ - while( iOffrc==SQLITE_OK && iOffrc==SQLITE_OK ){ int nSpace = pgsz - pBuf->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ @@ -3990,7 +3983,7 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->rc==SQLITE_OK && p->nPendingData ){ + if( p->nPendingData ){ assert( p->pHash ); p->nPendingData = 0; fts5FlushOneHash(p); @@ -4337,6 +4330,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); } + /* Flush the hash table to disk if required */ if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ fts5IndexFlush(p); } @@ -4932,7 +4926,6 @@ static void fts5DlidxIterTestReverse( ){ i64 iRowid = fts5DlidxIterRowid(pDlidx); int pgno = fts5DlidxIterPgno(pDlidx); - assert( fts5DlidxIterPgno(pDlidx)>iLeaf ); cksum2 += iRowid + ((i64)pgno<<32); } @@ -4971,23 +4964,31 @@ static void fts5IndexIntegrityCheckSegment( if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal - ** to or larger than the split-key in iter.term. */ + ** to or larger than the split-key in iter.term. Also check that if there + ** is also a rowid pointer within the leaf page header, it points to a + ** location before the term. */ iOff = fts5GetU16(&pLeaf->p[2]); if( iOff==0 ){ p->rc = FTS5_CORRUPT; }else{ + int iRowidOff; int nTerm; /* Size of term on leaf in bytes */ int res; /* Comparison of term and split-key */ - iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); - res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n)); - if( res==0 ) res = nTerm - iter.term.n; - if( res<0 ){ + + iRowidOff = fts5GetU16(&pLeaf->p[0]); + if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); + res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n)); + if( res==0 ) res = nTerm - iter.term.n; + if( res<0 ) p->rc = FTS5_CORRUPT; } } fts5DataRelease(pLeaf); if( p->rc ) break; + /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){ diff --git a/ext/fts5/fts5_vocab.c b/ext/fts5/fts5_vocab.c index 23a063296a..ee2a316291 100644 --- a/ext/fts5/fts5_vocab.c +++ b/ext/fts5/fts5_vocab.c @@ -344,36 +344,36 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->iCol = 0; assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); - while( 1 ){ + while( rc==SQLITE_OK ){ const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos); - if( rc!=SQLITE_OK ) break; - - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aVal[1]++; - } - pCsr->aVal[0]++; - }else{ - int iCol = -1; - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; - if( iCol!=ii ){ - pCsr->aDoc[ii]++; - iCol = ii; + if( rc==SQLITE_OK ){ + if( pTab->eType==FTS5_VOCAB_ROW ){ + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + pCsr->aVal[1]++; + } + pCsr->aVal[0]++; + }else{ + int iCol = -1; + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ + int ii = FTS5_POS2COLUMN(iPos); + pCsr->aCnt[ii]++; + if( iCol!=ii ){ + pCsr->aDoc[ii]++; + iCol = ii; + } } } + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + } + if( rc==SQLITE_OK ){ + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break; + if( sqlite3Fts5IterEof(pCsr->pIter) ) break; } - - rc = sqlite3Fts5IterNextScan(pCsr->pIter); - if( rc!=SQLITE_OK ) break; - zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); - if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break; - if( sqlite3Fts5IterEof(pCsr->pIter) ) break; } } } diff --git a/ext/fts5/test/fts5corrupt2.test b/ext/fts5/test/fts5corrupt2.test index 15868d1de1..df22f61b93 100644 --- a/ext/fts5/test/fts5corrupt2.test +++ b/ext/fts5/test/fts5corrupt2.test @@ -31,6 +31,7 @@ do_execsql_test 1.0 { } set mask [expr 31 << 31] +if 1 { # Test 1: # @@ -114,6 +115,8 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { } {} } +} + #------------------------------------------------------------------------- # Test that corruption in leaf page headers is detected by queries that use # doclist-indexes. @@ -130,17 +133,21 @@ do_execsql_test 3.0 { foreach {tn hdr} { 1 "\x00\x00\x00\x00" 2 "\xFF\xFF\xFF\xFF" + 3 "\x44\x45" } { set tn2 0 set nCorrupt 0 + set nCorrupt2 0 foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] { if {$rowid & $mask} continue incr tn2 - do_test 3.$tn.$tn2 { + do_test 3.$tn.$tn2.1 { execsql BEGIN set fd [db incrblob main x3_data block $rowid] fconfigure $fd -encoding binary -translation binary + set existing [read $fd [string length $hdr]] + seek $fd 0 puts -nonewline $fd $hdr close $fd @@ -149,6 +156,12 @@ foreach {tn hdr} { set {} 1 } {1} + if {($tn2 % 10)==0 && $existing != $hdr} { + do_test 3.$tn.$tn2.2 { + catchsql { INSERT INTO x3(x3) VALUES('integrity-check') } + } {1 {database disk image is malformed}} + } + execsql ROLLBACK } diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test index 76c05e8fd0..013de1b4a0 100644 --- a/ext/fts5/test/fts5dlidx.test +++ b/ext/fts5/test/fts5dlidx.test @@ -26,11 +26,6 @@ proc do_fb_test {tn sql res} { uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] } -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE t1 USING fts5(x); - INSERT INTO t1(t1, rank) VALUES('pgsz', 32); -} - # This test populates the FTS5 table containing $nEntry entries. Rows are # numbered from 0 to ($nEntry-1). The rowid for row $i is: # @@ -61,9 +56,9 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} { } execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) } } - breakpoint execsql COMMIT + breakpoint do_test $tn.1 { execsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {} @@ -81,12 +76,51 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} { } -do_dlidx_test1 1.1 10 100 10000 0 1000 -do_dlidx_test1 1.2 10 10 10000 0 128 -do_dlidx_test1 1.3 10 10 66 0 36028797018963970 -do_dlidx_test1 1.4 10 10 50 0 150000000000000000 +foreach {tn pgsz} { + 1 32 + 2 200 +} { + do_execsql_test $tn.0 { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); + } + + do_dlidx_test1 1.$tn.1 10 100 10000 0 1000 + do_dlidx_test1 1.$tn.2 10 10 10000 0 128 + do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970 + do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000 + do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55] + do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58] +} + +proc do_dlidx_test2 {tn nEntry iFirst nStep} { + set str [string repeat "a " 500] + execsql { + BEGIN; + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING fts5(x); + INSERT INTO t1(t1, rank) VALUES('pgsz', 64); + INSERT INTO t1 VALUES('b a'); + + WITH iii(ii, i) AS ( + SELECT 1, $iFirst UNION ALL + SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry + ) + INSERT INTO t1(rowid,x) SELECT i, $str FROM iii; + COMMIT; + } + do_execsql_test $tn.1 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' + } {1} + breakpoint + do_execsql_test $tn.2 { + SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC + } {1} +} +do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] finish_test diff --git a/ext/fts5/test/fts5fault4.test b/ext/fts5/test/fts5fault4.test index df2112c63f..6bfc825320 100644 --- a/ext/fts5/test/fts5fault4.test +++ b/ext/fts5/test/fts5fault4.test @@ -317,14 +317,15 @@ do_execsql_test 10.0 { COMMIT; } -} - do_faultsim_test 10.1 -faults oom-t* -body { db one { SELECT fts5_expr('a AND b NEAR(a b)') } } -test { faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} } +} + + #do_faultsim_test 10.2 -faults oom-t* -body { # db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') } #} -test { @@ -333,6 +334,5 @@ do_faultsim_test 10.1 -faults oom-t* -body { #} - finish_test diff --git a/ext/fts5/test/fts5fault5.test b/ext/fts5/test/fts5fault5.test new file mode 100644 index 0000000000..21a4e5d6b4 --- /dev/null +++ b/ext/fts5/test/fts5fault5.test @@ -0,0 +1,96 @@ +# 2014 June 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file is focused on OOM errors. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +source $testdir/malloc_common.tcl +set testprefix fts5fault5 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +#------------------------------------------------------------------------- +# OOM while creating an FTS5 table. +# +do_faultsim_test 1.1 -faults oom-t* -prep { + db eval { DROP TABLE IF EXISTS abc } +} -body { + db eval { CREATE VIRTUAL TABLE abc USING fts5(x,y) } +} -test { + faultsim_test_result {0 {}} +} + + +#------------------------------------------------------------------------- +# OOM while writing a multi-tier doclist-index. And while running +# integrity-check on the same. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -faults oom-t* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM tt } +} -body { + set str [string repeat "abc " 50] + db eval { + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) + INSERT INTO tt(rowid, x) SELECT i, $str FROM ii; + } +} -test { + faultsim_test_result {0 {}} +} + +do_faultsim_test 2.2 -faults oom-t* -body { + db eval { INSERT INTO tt(tt) VALUES('integrity-check') } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# OOM while scanning an fts5vocab table. +# +reset_db +do_test 3.0 { + execsql { + CREATE VIRTUAL TABLE tt USING fts5(x); + CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row'); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + BEGIN; + } + for {set i 0} {$i < 20} {incr i} { + set str [string repeat "$i " 20] + execsql { INSERT INTO tt VALUES($str) } + } + execsql COMMIT +} {} + +do_faultsim_test 2.1 -faults oom-t* -body { + db eval { + SELECT term FROM tv; + } +} -test { + faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}} +} + + + +finish_test + diff --git a/manifest b/manifest index 2af27f7119..46ffb625b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2015-05-13T18:12:58.028 +C Add\stest\scases.\sAnd\ssome\sfixes. +D 2015-05-15T12:18:39.221 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in edfc69769e613a6359c42c06ea1d42c3bece1736 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,12 +112,12 @@ F ext/fts5/fts5_buffer.c 70b971e13503566f1e257941c60817ba0920a16b F ext/fts5/fts5_config.c 05811f0bd80c396afcf3ceea68da16149a9a3258 F ext/fts5/fts5_expr.c 3fe1170453d6a322d2de8a3fd0aed3edff7b8b09 F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304 -F ext/fts5/fts5_index.c b9a3382af3027f5c9717d90613fda5f29f7d57fa +F ext/fts5/fts5_index.c 71d5ce47464f176e8708c7ec02d18613eb5eebda F ext/fts5/fts5_storage.c cb8b585bfb7870a36101f1a8fa0b0777f4d1b68d F ext/fts5/fts5_tcl.c f18eeb125d733f4e815a11679b971fa61cd7ec77 F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d -F ext/fts5/fts5_vocab.c 2e37ea9b4d4d5460bc778f2adb872c6a869601e7 +F ext/fts5/fts5_vocab.c e532f38a62d27fa662138a6cf33890b314225506 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32 @@ -138,15 +138,16 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f F ext/fts5/test/fts5corrupt.test 35bfdbbb3cdcea46ae7385f6432e9b5c574e70a1 -F ext/fts5/test/fts5corrupt2.test bdad9241f73076917512f5298501d50d9e9d7dc7 -F ext/fts5/test/fts5dlidx.test 74c3c8c33dfde594c0d8a22b9358d82fe56c8c7a +F ext/fts5/test/fts5corrupt2.test 88942d27ed581314f2867ef37352c72372c543df +F ext/fts5/test/fts5dlidx.test 070531bd45685e545e3e6021deb543f730a4011b F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5 F ext/fts5/test/fts5ea.test ed163ed820fd503354bd7dcf9d3b0e3801ade962 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e F ext/fts5/test/fts5fault1.test b42d3296be8a75f557cf2cbce0d8b483fc9db45b F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3 -F ext/fts5/test/fts5fault4.test 420f2e23775b458eeb9a325bcdfe84650c2e9d39 +F ext/fts5/test/fts5fault4.test a40e676e3e3b40901e2142f6fa5dff9e7313f5f7 +F ext/fts5/test/fts5fault5.test 98e7e77bc1d8bb47c955e7d6dc870ab5736536e3 F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa F ext/fts5/test/fts5integrity.test 98801bd0fb7c53a40bc770280134865d61724f3a @@ -246,7 +247,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c a5cf5b4b56390cfb7b8636e8f7ddef90258dd575 F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b F src/date.c e4d50b3283696836ec1036b695ead9a19e37a5ac -F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a w src/test_stat.c +F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a F src/delete.c 37964e6c1d73ff49cbea9ff690c9605fb15f600e F src/expr.c 3fb2ab3ab69d15b4b75ae53fceb4e317f64cb306 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb @@ -1324,7 +1325,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P aa34bf666c384cf32a8d8166ab6d9afbca26a256 59e3e9e764440b7feaafadff74f422535d21bca2 -R 7fb50d1391be18153fa78235920902bf +P b5f0e8c5b4bc018d672617ffd342d12b228548b9 +R bdc9509f45810e2b38f3396d130ff7ee U dan -Z 5ed330406879741f67800dc0f7082030 +Z 2d21c26b247bac296f85906c81a77b9c diff --git a/manifest.uuid b/manifest.uuid index a27ca1b112..6c7bb30f6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5f0e8c5b4bc018d672617ffd342d12b228548b9 \ No newline at end of file +adee788586197445672013d434e7ba47ce510b59 \ No newline at end of file