From: dan Date: Sat, 14 Dec 2019 14:15:21 +0000 (+0000) Subject: Fix problems in fts3 with processing corrupt records and undefined integer overflows. X-Git-Tag: version-3.31.0~224 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5be1b20aec9151af799d1294fc8328c8aaa17bd8;p=thirdparty%2Fsqlite.git Fix problems in fts3 with processing corrupt records and undefined integer overflows. FossilOrigin-Name: 3b873029ef1903f76442445343af3786cd59ee66b0e5e0e6b76a0b6b1b5eb88a --- diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5067c1cf0b..082d2a8cbf 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -308,18 +308,6 @@ SQLITE_EXTENSION_INIT1 #endif -/* -** The following are copied from sqliteInt.h. -** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. -*/ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( @@ -364,12 +352,7 @@ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ v = (*ptr++); \ if( (v & mask2)==0 ){ var = v; return ret; } -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. -*/ -int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ +int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ const unsigned char *p = (const unsigned char*)pBuf; const unsigned char *pStart = p; u32 a; @@ -391,6 +374,15 @@ int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ return (int)(p - pStart); } +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); +} + /* ** Read a 64-bit variable-length integer from memory starting at p[0] and ** not extending past pEnd[-1]. @@ -2496,12 +2488,12 @@ static void fts3GetDeltaVarint3( if( *pp>=pEnd ){ *pp = 0; }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); if( bDescIdx ){ - *pVal -= iVal; + *pVal = (i64)((u64)*pVal - iVal); }else{ - *pVal += iVal; + *pVal = (i64)((u64)*pVal + iVal); } } } @@ -2531,10 +2523,10 @@ static void fts3PutDeltaVarint3( sqlite3_uint64 iWrite; if( bDescIdx==0 || *pbFirst==0 ){ assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); - iWrite = iVal - *piPrev; + iWrite = (u64)iVal - (u64)*piPrev; }else{ assert_fts3_nc( *piPrev>=iVal ); - iWrite = *piPrev - iVal; + iWrite = (u64)*piPrev - (u64)iVal; } assert( *pbFirst || *piPrev==0 ); assert_fts3_nc( *pbFirst==0 || iWrite>0 ); @@ -2553,7 +2545,8 @@ static void fts3PutDeltaVarint3( ** Using this makes it easier to write code that can merge doclists that are ** sorted in either ascending or descending order. */ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) +// #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) /* ** This function does an "OR" merge of two doclists (output contains all diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 26b23b66f2..50370a9108 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -196,6 +196,9 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ # define TESTONLY(X) #endif +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -578,6 +581,7 @@ int sqlite3Fts3Incrmerge(Fts3Table*,int,int); void sqlite3Fts3ErrMsg(char**,const char*,...); int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index a6b45aef2e..203b48e2d1 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -696,7 +696,7 @@ static int fts3PendingListAppend( assert( !p || p->iLastDocid<=iDocid ); if( !p || p->iLastDocid!=iDocid ){ - sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); if( p ){ assert( p->nDatanSpace ); assert( p->aData[p->nData]==0 ); @@ -1531,18 +1531,18 @@ static int fts3SegReaderNextDocid( }else{ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); if( pTab->bDescIdx ){ - pReader->iDocid -= iDelta; + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); }else{ - pReader->iDocid += iDelta; + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); } } } } - return SQLITE_OK; + return rc; } @@ -2281,6 +2281,7 @@ static int fts3SegWriterAdd( int rc; /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; p->nLeafAdd++; @@ -2978,10 +2979,10 @@ int sqlite3Fts3SegReaderStep( sqlite3_int64 iDelta; if( p->bDescIdx && nDoclist>0 ){ if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; - iDelta = iPrev - iDocid; + iDelta = (i64)((u64)iPrev - (u64)iDocid); }else{ if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; - iDelta = iDocid - iPrev; + iDelta = (i64)((u64)iDocid - (u64)iPrev); } nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); @@ -3264,7 +3265,7 @@ static int fts3SegmentMerge( csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); } if( rc!=SQLITE_OK ) goto finished; - assert( pWriter || bIgnoreEmpty ); + assert_fts3_nc( pWriter || bIgnoreEmpty ); if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3DeleteSegdir( @@ -5190,12 +5191,12 @@ static u64 fts3ChecksumIndex( i64 iDocid = 0; i64 iCol = 0; - i64 iPos = 0; + u64 iPos = 0; pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); while( pCsrbDescIdx ){ - iDocid -= iVal; + iDocid = (i64)((u64)iDocid - iVal); }else{ - iDocid += iVal; + iDocid = (i64)((u64)iDocid + iVal); } } }else{ diff --git a/manifest b/manifest index 766eab9ab7..b99965e3de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sutterly\sharmless\s"applying\szero\soffset\sto\snull\spointer"\sUB\swarning\nin\ssqlite3AtoF(): -D 2019-12-13T23:38:57.778 +C Fix\sproblems\sin\sfts3\swith\sprocessing\scorrupt\srecords\sand\sundefined\sinteger\soverflows. +D 2019-12-14T14:15:21.800 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -81,9 +81,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers b92bdeb8b46503f0dd301d364efc5ef59ef9fa8e2758b8e742f39fa93a2e422d F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 9a69143c94195d9342dd4a704021009fb9e45dff9bff775dae024d47e363bab8 +F ext/fts3/fts3.c a05938725571651cc9bedc63a52ede4b39277fec17884c8ff6011dab2ff78c75 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ba4e5445a0daf4a251345d4704259268fd98a3e0282fedfcaedb9fc6da9f250c +F ext/fts3/fts3Int.h f091030b976045e7df91af2337935952b477cdbd9f48058c44c965684484cb50 F ext/fts3/fts3_aux.c 96708c8b3a7d9b8ca1b68ea2b7e503e283f20e95f145becadedfad096dbd0f34 F ext/fts3/fts3_expr.c b132af223e90e35b9f9efa9fe63d6ae737d34153a3b6066736086df8abc78a1f F ext/fts3/fts3_hash.c 8b6e31bfb0844c27dc6092c2620bdb1fca17ed613072db057d96952c6bdb48b7 @@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c fa971df91b7c9c317ccb76e73de425de372f854cbed16be2d98f42f61be6c0fb +F ext/fts3/fts3_write.c 51e0a4e3782ee17b6dd5e89949b7095fc98e36e87725c53de631734535507498 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -939,7 +939,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f F test/fts3corrupt.test ce7f7b5eaeee5f1804584d061b978d85e64abf2af9adaa7577589fac6f7eae01 F test/fts3corrupt2.test bf55c3fa0b0dc8ea1c0fe5543623bd27714585da6a129038fd6999fe3b0d25f3 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test bc90c0f6ee73df4c6bd20f1b32fefdfc00b44cc577d67ebca43b157fc3efd422 +F test/fts3corrupt4.test 00ecd7605d5b1478252a3b7c2465de2d427906b8cb73b7acb92628bab81f9903 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f @@ -1852,7 +1852,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c29fc21288e37f81a1617c5e2961c575d3bca6a1d1b013b2e0a99774afb1dcdb -R 3d3d486575d8e4fc08b0a007870e491d -U drh -Z 3aa16a4b7876299855c152aecef9a886 +P 052fdf5e58b41ccadaa5aac293ceb4d309ced661d46f3a52be9eb8d01d347a82 +R e3e5166e62c991d8b782b6533f40e85d +U dan +Z 2fcdc36e2d58b6d19eb97fe22b33d40d diff --git a/manifest.uuid b/manifest.uuid index 8ea5d70027..d045a1c8e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -052fdf5e58b41ccadaa5aac293ceb4d309ced661d46f3a52be9eb8d01d347a82 \ No newline at end of file +3b873029ef1903f76442445343af3786cd59ee66b0e5e0e6b76a0b6b1b5eb88a \ No newline at end of file diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index 2334907f83..a30fd74473 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -5759,18 +5759,18 @@ do_test 32.0 { | end crash-74fdbc96edbc04.db }]} {} -do_execsql_test 32.1 { +do_catchsql_test 32.1 { UPDATE t1 SET b=quote(zeroblob(6.51158946e+5)) WHERE a MATCH '*t*'; -} {} - -do_catchsql_test 32.2 { - UPDATE t1 SET b=((- '' )) WHERE a MATCH '0*t'; } {1 {database disk image is malformed}} +#do_catchsql_test 32.2 { +# UPDATE t1 SET b=((- '' )) WHERE a MATCH '0*t'; +#} {1 {database disk image is malformed}} + #------------------------------------------------------------------------- # reset_db -do_catchsql_test 32.0 { +do_catchsql_test 33.0 { CREATE VIRTUAL TABLE f USING fts3(a,b,tokenize=icu); CREATE TABLE 'f_docsize'(docid INTEGER PRIMARY KEY, size BLOB); CREATE TABLE 'f_stat'(id INTEGER PRIMARY KEY, value BLOB); @@ -5779,5 +5779,35 @@ do_catchsql_test 32.0 { INSERT INTO f(f) VALUES ('merge=198,49'); } {1 {database disk image is malformed}} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 34.0 { + CREATE VIRTUAL TABLE f USING fts3(a,b); + INSERT INTO f VALUES (1, '1234'); + INSERT INTO f_segdir VALUES (1,255,0,0,'1 255',x'00'); + UPDATE f_segdir SET level = 0 WHERE level IN ( + SELECT level FROM f_segdir LIMIT 1 OFFSET 1 + ); + INSERT INTO f_segdir VALUES (255,249,0,121,'0 0',x'00'); + INSERT INTO f_content VALUES (255,0,x'ff'); + INSERT INTO f_segdir VALUES (1,255,16,0,'1 255',x'00'); +} + +do_catchsql_test 34.1 { + UPDATE f SET b = x'00' WHERE b IN (SELECT b FROM f LIMIT 1 OFFSET 0); +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 35.0 { + CREATE VIRTUAL TABLE f USING fts3(a,b); + INSERT INTO f_segdir VALUES (1,255,0,0,'1 255',x'0001ff000001ff000001ff000001ff000001ff00c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5bec5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5c5'); +} + +do_catchsql_test 35.1 { + INSERT INTO f(f) VALUES ('integrity-check'); +} {1 {database disk image is malformed}} finish_test