int pgnoLast = 0;
if( pDlidx ){
- /* If the doclist-iterator is already at EOF, then the current doclist
- ** contains no entries except those on the current page. */
- if( fts5DlidxIterEof(p, pDlidx)==0 ){
- int iSegid = pIter->pSeg->iSegid;
- pgnoLast = fts5DlidxIterPgno(pDlidx);
- pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast));
- }else{
- pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
- }
+ int iSegid = pIter->pSeg->iSegid;
+ pgnoLast = fts5DlidxIterPgno(pDlidx);
+ pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast));
}else{
int iOff; /* Byte offset within pLeaf */
Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
pWriter->iSegid = iSegid;
pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter));
- pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter));
- if( pWriter->aDlidx==0 ) return;
+ if( fts5WriteDlidxGrow(p, pWriter, 1) ) return;
pWriter->nWriter = 1;
pWriter->nDlidx = 1;
pWriter->aWriter[0].pgno = 1;
/* The entire doclist will not fit on this leaf. The following
** loop iterates through the poslists that make up the current
** doclist. */
- while( iOff<nDoclist ){
+ while( p->rc==SQLITE_OK && iOff<nDoclist ){
int nPos;
int nCopy;
int bDummy;
** that each varint must be stored contiguously. */
const u8 *pPoslist = &pDoclist[iOff];
int iPos = 0;
- while( 1 ){
+ while( p->rc==SQLITE_OK ){
int nSpace = pgsz - pBuf->n;
int n = 0;
if( (nCopy - iPos)<=nSpace ){
*/
static void fts5IndexFlush(Fts5Index *p){
/* Unless it is empty, flush the hash table to disk */
- if( p->rc==SQLITE_OK && p->nPendingData ){
+ if( p->nPendingData ){
assert( p->pHash );
p->nPendingData = 0;
fts5FlushOneHash(p);
p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
}
+ /* Flush the hash table to disk if required */
if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
fts5IndexFlush(p);
}
){
i64 iRowid = fts5DlidxIterRowid(pDlidx);
int pgno = fts5DlidxIterPgno(pDlidx);
-
assert( fts5DlidxIterPgno(pDlidx)>iLeaf );
cksum2 += iRowid + ((i64)pgno<<32);
}
if( pLeaf==0 ) break;
/* Check that the leaf contains at least one term, and that it is equal
- ** to or larger than the split-key in iter.term. */
+ ** to or larger than the split-key in iter.term. Also check that if there
+ ** is also a rowid pointer within the leaf page header, it points to a
+ ** location before the term. */
iOff = fts5GetU16(&pLeaf->p[2]);
if( iOff==0 ){
p->rc = FTS5_CORRUPT;
}else{
+ int iRowidOff;
int nTerm; /* Size of term on leaf in bytes */
int res; /* Comparison of term and split-key */
- iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
- res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
- if( res==0 ) res = nTerm - iter.term.n;
- if( res<0 ){
+
+ iRowidOff = fts5GetU16(&pLeaf->p[0]);
+ if( iRowidOff>=iOff ){
p->rc = FTS5_CORRUPT;
+ }else{
+ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm);
+ res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n));
+ if( res==0 ) res = nTerm - iter.term.n;
+ if( res<0 ) p->rc = FTS5_CORRUPT;
}
}
fts5DataRelease(pLeaf);
if( p->rc ) break;
+
/* Now check that the iter.nEmpty leaves following the current leaf
** (a) exist and (b) contain no terms. */
for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){
pCsr->iCol = 0;
assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
- while( 1 ){
+ while( rc==SQLITE_OK ){
const u8 *pPos; int nPos; /* Position list */
i64 iPos = 0; /* 64-bit position read from poslist */
int iOff = 0; /* Current offset within position list */
rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos);
- if( rc!=SQLITE_OK ) break;
-
- if( pTab->eType==FTS5_VOCAB_ROW ){
- while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
- pCsr->aVal[1]++;
- }
- pCsr->aVal[0]++;
- }else{
- int iCol = -1;
- while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
- int ii = FTS5_POS2COLUMN(iPos);
- pCsr->aCnt[ii]++;
- if( iCol!=ii ){
- pCsr->aDoc[ii]++;
- iCol = ii;
+ if( rc==SQLITE_OK ){
+ if( pTab->eType==FTS5_VOCAB_ROW ){
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+ pCsr->aVal[1]++;
+ }
+ pCsr->aVal[0]++;
+ }else{
+ int iCol = -1;
+ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+ int ii = FTS5_POS2COLUMN(iPos);
+ pCsr->aCnt[ii]++;
+ if( iCol!=ii ){
+ pCsr->aDoc[ii]++;
+ iCol = ii;
+ }
}
}
+ rc = sqlite3Fts5IterNextScan(pCsr->pIter);
+ }
+ if( rc==SQLITE_OK ){
+ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
+ if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
+ if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
-
- rc = sqlite3Fts5IterNextScan(pCsr->pIter);
- if( rc!=SQLITE_OK ) break;
- zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
- if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break;
- if( sqlite3Fts5IterEof(pCsr->pIter) ) break;
}
}
}
}
set mask [expr 31 << 31]
+if 1 {
# Test 1:
#
} {}
}
+}
+
#-------------------------------------------------------------------------
# Test that corruption in leaf page headers is detected by queries that use
# doclist-indexes.
foreach {tn hdr} {
1 "\x00\x00\x00\x00"
2 "\xFF\xFF\xFF\xFF"
+ 3 "\x44\x45"
} {
set tn2 0
set nCorrupt 0
+ set nCorrupt2 0
foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
if {$rowid & $mask} continue
incr tn2
- do_test 3.$tn.$tn2 {
+ do_test 3.$tn.$tn2.1 {
execsql BEGIN
set fd [db incrblob main x3_data block $rowid]
fconfigure $fd -encoding binary -translation binary
+ set existing [read $fd [string length $hdr]]
+ seek $fd 0
puts -nonewline $fd $hdr
close $fd
set {} 1
} {1}
+ if {($tn2 % 10)==0 && $existing != $hdr} {
+ do_test 3.$tn.$tn2.2 {
+ catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
+ } {1 {database disk image is malformed}}
+ }
+
execsql ROLLBACK
}
uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}
-do_execsql_test 1.0 {
- CREATE VIRTUAL TABLE t1 USING fts5(x);
- INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
-}
-
# This test populates the FTS5 table containing $nEntry entries. Rows are
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
}
execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
}
- breakpoint
execsql COMMIT
+ breakpoint
do_test $tn.1 {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
}
-do_dlidx_test1 1.1 10 100 10000 0 1000
-do_dlidx_test1 1.2 10 10 10000 0 128
-do_dlidx_test1 1.3 10 10 66 0 36028797018963970
-do_dlidx_test1 1.4 10 10 50 0 150000000000000000
+foreach {tn pgsz} {
+ 1 32
+ 2 200
+} {
+ do_execsql_test $tn.0 {
+ DROP TABLE IF EXISTS t1;
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
+ }
+
+ do_dlidx_test1 1.$tn.1 10 100 10000 0 1000
+ do_dlidx_test1 1.$tn.2 10 10 10000 0 128
+ do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970
+ do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000
+ do_dlidx_test1 1.$tn.5 10 10 200 0 [expr 1<<55]
+ do_dlidx_test1 1.$tn.6 10 10 30 0 [expr 1<<58]
+}
+
+proc do_dlidx_test2 {tn nEntry iFirst nStep} {
+ set str [string repeat "a " 500]
+ execsql {
+ BEGIN;
+ DROP TABLE IF EXISTS t1;
+ CREATE VIRTUAL TABLE t1 USING fts5(x);
+ INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
+ INSERT INTO t1 VALUES('b a');
+
+ WITH iii(ii, i) AS (
+ SELECT 1, $iFirst UNION ALL
+ SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry
+ )
+ INSERT INTO t1(rowid,x) SELECT i, $str FROM iii;
+ COMMIT;
+ }
+ do_execsql_test $tn.1 {
+ SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a'
+ } {1}
+ breakpoint
+ do_execsql_test $tn.2 {
+ SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC
+ } {1}
+}
+do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
finish_test
COMMIT;
}
-}
-
do_faultsim_test 10.1 -faults oom-t* -body {
db one { SELECT fts5_expr('a AND b NEAR(a b)') }
} -test {
faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}}
}
+}
+
+
#do_faultsim_test 10.2 -faults oom-t* -body {
# db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
#} -test {
#}
-
finish_test
--- /dev/null
+# 2014 June 17
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault5
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+#-------------------------------------------------------------------------
+# OOM while creating an FTS5 table.
+#
+do_faultsim_test 1.1 -faults oom-t* -prep {
+ db eval { DROP TABLE IF EXISTS abc }
+} -body {
+ db eval { CREATE VIRTUAL TABLE abc USING fts5(x,y) }
+} -test {
+ faultsim_test_result {0 {}}
+}
+
+
+#-------------------------------------------------------------------------
+# OOM while writing a multi-tier doclist-index. And while running
+# integrity-check on the same.
+#
+reset_db
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE tt USING fts5(x);
+ INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+}
+faultsim_save_and_close
+
+do_faultsim_test 2.1 -faults oom-t* -prep {
+ faultsim_restore_and_reopen
+ db eval { SELECT * FROM tt }
+} -body {
+ set str [string repeat "abc " 50]
+ db eval {
+ WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100)
+ INSERT INTO tt(rowid, x) SELECT i, $str FROM ii;
+ }
+} -test {
+ faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 2.2 -faults oom-t* -body {
+ db eval { INSERT INTO tt(tt) VALUES('integrity-check') }
+} -test {
+ faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+# OOM while scanning an fts5vocab table.
+#
+reset_db
+do_test 3.0 {
+ execsql {
+ CREATE VIRTUAL TABLE tt USING fts5(x);
+ CREATE VIRTUAL TABLE tv USING fts5vocab(tt, 'row');
+ INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+ BEGIN;
+ }
+ for {set i 0} {$i < 20} {incr i} {
+ set str [string repeat "$i " 20]
+ execsql { INSERT INTO tt VALUES($str) }
+ }
+ execsql COMMIT
+} {}
+
+do_faultsim_test 2.1 -faults oom-t* -body {
+ db eval {
+ SELECT term FROM tv;
+ }
+} -test {
+ faultsim_test_result {0 {0 1 10 11 12 13 14 15 16 17 18 19 2 3 4 5 6 7 8 9}}
+}
+
+
+
+finish_test
+
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2015-05-13T18:12:58.028
+C Add\stest\scases.\sAnd\ssome\sfixes.
+D 2015-05-15T12:18:39.221
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in edfc69769e613a6359c42c06ea1d42c3bece1736
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F ext/fts5/fts5_config.c 05811f0bd80c396afcf3ceea68da16149a9a3258
F ext/fts5/fts5_expr.c 3fe1170453d6a322d2de8a3fd0aed3edff7b8b09
F ext/fts5/fts5_hash.c 54dd25348a46ea62ea96322c572e08cd1fb37304
-F ext/fts5/fts5_index.c b9a3382af3027f5c9717d90613fda5f29f7d57fa
+F ext/fts5/fts5_index.c 71d5ce47464f176e8708c7ec02d18613eb5eebda
F ext/fts5/fts5_storage.c cb8b585bfb7870a36101f1a8fa0b0777f4d1b68d
F ext/fts5/fts5_tcl.c f18eeb125d733f4e815a11679b971fa61cd7ec77
F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc
F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
-F ext/fts5/fts5_vocab.c 2e37ea9b4d4d5460bc778f2adb872c6a869601e7
+F ext/fts5/fts5_vocab.c e532f38a62d27fa662138a6cf33890b314225506
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f
F ext/fts5/test/fts5corrupt.test 35bfdbbb3cdcea46ae7385f6432e9b5c574e70a1
-F ext/fts5/test/fts5corrupt2.test bdad9241f73076917512f5298501d50d9e9d7dc7
-F ext/fts5/test/fts5dlidx.test 74c3c8c33dfde594c0d8a22b9358d82fe56c8c7a
+F ext/fts5/test/fts5corrupt2.test 88942d27ed581314f2867ef37352c72372c543df
+F ext/fts5/test/fts5dlidx.test 070531bd45685e545e3e6021deb543f730a4011b
F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5
F ext/fts5/test/fts5ea.test ed163ed820fd503354bd7dcf9d3b0e3801ade962
F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
F ext/fts5/test/fts5fault1.test b42d3296be8a75f557cf2cbce0d8b483fc9db45b
F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test 420f2e23775b458eeb9a325bcdfe84650c2e9d39
+F ext/fts5/test/fts5fault4.test a40e676e3e3b40901e2142f6fa5dff9e7313f5f7
+F ext/fts5/test/fts5fault5.test 98e7e77bc1d8bb47c955e7d6dc870ab5736536e3
F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa
F ext/fts5/test/fts5integrity.test 98801bd0fb7c53a40bc770280134865d61724f3a
F src/complete.c a5cf5b4b56390cfb7b8636e8f7ddef90258dd575
F src/ctime.c 5a0b735dc95604766f5dac73973658eef782ee8b
F src/date.c e4d50b3283696836ec1036b695ead9a19e37a5ac
-F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a w src/test_stat.c
+F src/dbstat.c f402e77e25089c6003d0c60b3233b9b3947d599a
F src/delete.c 37964e6c1d73ff49cbea9ff690c9605fb15f600e
F src/expr.c 3fb2ab3ab69d15b4b75ae53fceb4e317f64cb306
F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P aa34bf666c384cf32a8d8166ab6d9afbca26a256 59e3e9e764440b7feaafadff74f422535d21bca2
-R 7fb50d1391be18153fa78235920902bf
+P b5f0e8c5b4bc018d672617ffd342d12b228548b9
+R bdc9509f45810e2b38f3396d130ff7ee
U dan
-Z 5ed330406879741f67800dc0f7082030
+Z 2d21c26b247bac296f85906c81a77b9c
-b5f0e8c5b4bc018d672617ffd342d12b228548b9
\ No newline at end of file
+adee788586197445672013d434e7ba47ce510b59
\ No newline at end of file
projects / thirdparty / sqlite.git / commitdiff
