From 81b35dc66c51c9aeb963d90a2216692e0859982d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Mar 2012 15:54:19 +0000 Subject: [PATCH] Fix some integer overflow problems that can occur when using large langauge id values. FossilOrigin-Name: 3475092cff862080a020d386076d739f0d22c9b2 --- ext/fts3/fts3_write.c | 37 ++++++++------- manifest | 14 +++--- manifest.uuid | 2 +- test/fts4langid.test | 102 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 24 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f9cb3667c6..fca5d9f70e 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -510,10 +510,13 @@ static sqlite3_int64 getAbsoluteLevel( int iIndex, /* Index in p->aIndex[] */ int iLevel /* Level of segments */ ){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ assert( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndexnIndex ); - return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel; + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; } /* @@ -556,7 +559,7 @@ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iLangid, /* Language being queried */ int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select */ + int iLevel, /* Level to select (relative level) */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; @@ -571,7 +574,7 @@ int sqlite3Fts3AllSegdirs( rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } @@ -579,7 +582,7 @@ int sqlite3Fts3AllSegdirs( /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } *ppStmt = pStmt; @@ -1854,7 +1857,7 @@ static int fts3WriteSegment( */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ - int iLevel, /* Value for "level" field */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ @@ -1865,7 +1868,7 @@ static int fts3WriteSegdir( sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, iLevel); + sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); @@ -2248,7 +2251,7 @@ static int fts3SegWriterAdd( static int fts3SegWriterFlush( Fts3Table *p, /* Virtual table handle */ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - int iLevel, /* Value for 'level' column of %_segdir */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ int iIdx /* Value for 'idx' column of %_segdir */ ){ int rc; /* Return code */ @@ -2330,7 +2333,7 @@ static int fts3SegmentMaxLevel( Fts3Table *p, int iLangid, int iIndex, - int *pnMax + sqlite3_int64 *pnMax ){ sqlite3_stmt *pStmt; int rc; @@ -2344,12 +2347,12 @@ static int fts3SegmentMaxLevel( */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pStmt, 2, + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int(pStmt, 0); + *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -2414,15 +2417,17 @@ static int fts3DeleteSegdir( if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int(pDelete, 2, + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } @@ -2899,7 +2904,7 @@ static int fts3SegmentMerge( ){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level/index to create new segment at */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ @@ -3864,7 +3869,7 @@ static int fts3IncrmergeWriter( rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pOutputIdx, 1, iAbsLevel+1); + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); sqlite3_step(pOutputIdx); iIdx = sqlite3_column_int(pOutputIdx, 0) - 1; rc = sqlite3_reset(pOutputIdx); diff --git a/manifest b/manifest index be0bd8c83f..8bb46fe386 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sthe\sFTS\sgetAbsoluteLevel()\sfunction.\sNo\sactual\scode\schanges. -D 2012-03-16T14:54:07.123 +C Fix\ssome\sinteger\soverflow\sproblems\sthat\scan\soccur\swhen\susing\slarge\slangauge\sid\svalues. +D 2012-03-16T15:54:19.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3f79a373e57c3b92dabf76f40b065e719d31ac34 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 -F ext/fts3/fts3_write.c fab92bcc6de43fb6dfd3937ac67e2cb45c834a2b +F ext/fts3/fts3_write.c 548d034e16ecad0ec5dfd542c48c591564c343d3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f -F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a +F test/fts4langid.test 2f6ae2e5ab037b11b411c85346a09a5fee503f6d F test/fts4merge.test 1c3389a3be18498934baf76afe67663d8b4b5e42 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca @@ -994,7 +994,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P cd34bc1af4ba608ea3b52bab55bcfe0086711900 -R c9f36e032863c0b8f751fc4c1ff8b5a5 +P 7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd +R 3c4572a4f07a3af6855856a4d0059330 U dan -Z b14699878441d211e25f75f7c2630c95 +Z 5aeec06647e7b858688ed8df27a75aed diff --git a/manifest.uuid b/manifest.uuid index ca4d0738ba..b07a988ed8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e0f861beda4d74d0c3c9fb4abb3ddb5fee346bd \ No newline at end of file +3475092cff862080a020d386076d739f0d22c9b2 \ No newline at end of file diff --git a/test/fts4langid.test b/test/fts4langid.test index 08f1a21c17..012924c70d 100644 --- a/test/fts4langid.test +++ b/test/fts4langid.test @@ -382,4 +382,106 @@ do_catchsql_test 4.1.5 { INSERT INTO t4(content, lid) VALUES('hello world', 101) } {1 {SQL logic error or missing database}} +#------------------------------------------------------------------------- +# Test cases 5.* +# +# The following test cases are designed to detect a 32-bit overflow bug +# that existed at one point. +# +proc build_multilingual_db_3 {db} { + $db eval { + CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid); + } + set languages [list 0 1 2 [expr 1<<30]] + + foreach lid $languages { + execsql { + INSERT INTO t5(docid, content, lid) VALUES( + $lid, 'My language is ' || $lid, $lid + ) + } + } +} + +do_test 5.1.0 { + reset_db + build_multilingual_db_3 db +} {} + +do_execsql_test 5.1.1 { + SELECT level FROM t5_segdir; +} [list 0 1024 2048 [expr 1<<40]] + +do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0 +foreach langid [list 0 1 2 [expr 1<<30]] { + do_execsql_test 5.2.$langid { + SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid + } $langid +} + +set lid [expr 1<<30] +do_execsql_test 5.3.1 { + CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid); + INSERT INTO t6 VALUES('I belong to language 0!'); +} +do_test 5.3.2 { + for {set i 0} {$i < 20} {incr i} { + execsql { + INSERT INTO t6(content, lid) VALUES( + 'I (row '||$i||') belong to langauge N!', $lid + ); + } + } + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } +} {1} + +do_test 5.3.3 { + execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid} +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + +do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {} +do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1} +do_execsql_test 5.3.6 { + SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid +} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21} + + +set lid [expr 1<<30] +foreach lid [list 4 [expr 1<<30]] { + do_execsql_test 5.4.$lid.1 { + DELETE FROM t6; + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {0 0} + do_execsql_test 5.4.$lid.2 { + INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero zero one', $lid); + INSERT INTO t6(content, lid) VALUES('zero one zero', $lid); + INSERT INTO t6(content, lid) VALUES('zero one one', $lid); + INSERT INTO t6(content, lid) VALUES('one zero zero', $lid); + INSERT INTO t6(content, lid) VALUES('one zero one', $lid); + INSERT INTO t6(content, lid) VALUES('one one zero', $lid); + INSERT INTO t6(content, lid) VALUES('one one one', $lid); + + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.3 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {8 0} + + do_execsql_test 5.4.$lid.4 { + INSERT INTO t6(t6) VALUES('merge=100,3'); + INSERT INTO t6(t6) VALUES('merge=100,3'); + SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid; + } {1 2 5} + + do_execsql_test 5.4.$lid.5 { + SELECT count(*) FROM t6_segdir; + SELECT count(*) FROM t6_segments; + } {4 4} +} + + finish_test -- 2.47.2