From b163b57212b8c2c68dbbef8e3de7f095b544e302 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Dec 2018 07:37:22 +0000 Subject: [PATCH] Fix problems in fts5 found by ASAN. FossilOrigin-Name: c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576 --- ext/fts3/fts3_unicode2.c | 6 +++--- ext/fts3/unicode/mkunicode.tcl | 13 +++++++------ ext/fts5/fts5Int.h | 2 +- ext/fts5/fts5_expr.c | 2 +- ext/fts5/fts5_index.c | 7 +++---- ext/fts5/fts5_test_tok.c | 2 +- ext/fts5/fts5_tokenize.c | 4 ++-- ext/fts5/fts5_unicode2.c | 10 ++++++---- ext/fts5/test/fts5unicode3.test | 2 +- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- 11 files changed, 40 insertions(+), 38 deletions(-) diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index c0395d9cf6..4f92e5265b 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -200,7 +200,7 @@ static int remove_diacritic(int c, int bComplex){ 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, - 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -231,8 +231,8 @@ int sqlite3FtsUnicodeIsdiacritic(int c){ unsigned int mask1 = 0x000361F8; if( c<768 || c>817 ) return 0; return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); } diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 8465262033..872ec130e7 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -63,14 +63,15 @@ proc print_rd {map} { } puts "" puts " \};" + puts "#define HIBIT ((char)0x80)" puts " char aChar\[\] = \{" puts -nonewline " '\\0', " set i 1 foreach c $aChar f $aFlag { if { $f } { - set str "'$c'|0x80, " + set str "'$c'|HIBIT, " } else { - set str "'$c'|0x00, " + set str "'$c', " } if {$c == ""} { set str "'\\0', " } @@ -134,8 +135,8 @@ proc print_isdiacritic {zFunc map} { puts " if( c<$iFirst || c>$iLast ) return 0;" puts " return (c < $iFirst+32) ?" - puts " (mask0 & (1 << (c-$iFirst))) :" - puts " (mask1 & (1 << (c-$iFirst-32)));" + puts " (mask0 & ((unsigned int)1 << (c-$iFirst))) :" + puts " (mask1 & ((unsigned int)1 << (c-$iFirst-32)));" puts "\}" } @@ -699,7 +700,7 @@ proc print_categories {lMap} { static u16 aFts5UnicodeMap[] = {$aMapArray}; static u16 aFts5UnicodeData[] = {$aDataArray}; - int sqlite3Fts5UnicodeCategory(int iCode) { + int sqlite3Fts5UnicodeCategory(u32 iCode) { int iRes = -1; int iHi; int iLo; @@ -782,7 +783,7 @@ proc print_test_categories {lMap} { aArray[0] = 1; } - c = sqlite3Fts5UnicodeCategory(i); + c = sqlite3Fts5UnicodeCategory((u32)i); if( aArray[c]==0 ){ *piCode = i; return 1; diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index a460a7af34..b4f4b07285 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -788,7 +788,7 @@ int sqlite3Fts5UnicodeIsdiacritic(int c); int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); int sqlite3Fts5UnicodeCatParse(const char*, u8*); -int sqlite3Fts5UnicodeCategory(int iCode); +int sqlite3Fts5UnicodeCategory(u32 iCode); void sqlite3Fts5UnicodeAscii(u8*, u8*); /* ** End of interface to code in fts5_unicode2.c. diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a09fe7def8..b56434e35a 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -2553,7 +2553,7 @@ static void fts5ExprIsAlnum( sqlite3Fts5UnicodeCatParse("N*", aArr); sqlite3Fts5UnicodeCatParse("Co", aArr); iCode = sqlite3_value_int(apVal[0]); - sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]); + sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]); } static void fts5ExprFold( diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index c5fe01bd45..6bd18c52e4 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3577,14 +3577,14 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; if( iId<=FTS5_MAX_SEGMENT ){ - aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); + aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32); } } } for(i=0; aUsed[i]==0xFFFFFFFF; i++); mask = aUsed[i]; - for(iSegid=0; mask & (1 << iSegid); iSegid++); + for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++); iSegid += 1 + i*32; #ifdef SQLITE_DEBUG @@ -6282,8 +6282,7 @@ static void fts5DecodeFunction( nSpace = n + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; - memcpy(a, aBlob, n); - + if( n>0 ) memcpy(a, aBlob, n); fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); diff --git a/ext/fts5/fts5_test_tok.c b/ext/fts5/fts5_test_tok.c index 6f71e6534d..3ac6e351a5 100644 --- a/ext/fts5/fts5_test_tok.c +++ b/ext/fts5/fts5_test_tok.c @@ -378,7 +378,7 @@ static int fts5tokFilterMethod( if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ - memcpy(pCsr->zInput, zByte, nByte); + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); pCsr->zInput[nByte] = 0; rc = pTab->tok.xTokenize( pTab->pTok, (void*)pCsr, 0, zByte, nByte, fts5tokCb diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c index 35526a3898..d166f7ff3e 100644 --- a/ext/fts5/fts5_tokenize.c +++ b/ext/fts5/fts5_tokenize.c @@ -268,7 +268,7 @@ static int fts5UnicodeAddExceptions( if( iCode<128 ){ p->aTokenChar[iCode] = (unsigned char)bTokenChars; }else{ - bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]; + bToken = p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]; assert( (bToken==0 || bToken==1) ); assert( (bTokenChars==0 || bTokenChars==1) ); if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ @@ -429,7 +429,7 @@ static int fts5UnicodeCreate( */ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ return ( - p->aCategory[sqlite3Fts5UnicodeCategory(iCode)] + p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)] ^ fts5UnicodeIsException(p, iCode) ); } diff --git a/ext/fts5/fts5_unicode2.c b/ext/fts5/fts5_unicode2.c index 8bb1cd94fb..0b0e9f7a79 100644 --- a/ext/fts5/fts5_unicode2.c +++ b/ext/fts5/fts5_unicode2.c @@ -69,7 +69,7 @@ static int fts5_remove_diacritic(int c, int bComplex){ 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, - 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -100,8 +100,8 @@ int sqlite3Fts5UnicodeIsdiacritic(int c){ unsigned int mask1 = 0x000361F8; if( c<768 || c>817 ) return 0; return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); } @@ -249,6 +249,7 @@ int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ return ret; } + int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ aArray[0] = 1; switch( zCat[0] ){ @@ -730,7 +731,7 @@ static u16 aFts5UnicodeData[] = { 34, 3074, 7692, 63, 63, }; -int sqlite3Fts5UnicodeCategory(int iCode) { +int sqlite3Fts5UnicodeCategory(u32 iCode) { int iRes = -1; int iHi; int iLo; @@ -773,3 +774,4 @@ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ iTbl++; } } + diff --git a/ext/fts5/test/fts5unicode3.test b/ext/fts5/test/fts5unicode3.test index ed89624467..30eb3c4166 100644 --- a/ext/fts5/test/fts5unicode3.test +++ b/ext/fts5/test/fts5unicode3.test @@ -21,7 +21,7 @@ ifcapable !fts5 { } proc fts3_unicode_path {file} { - file join [file dirname [info script]] .. .. fts3 unicode $file + file join .. [file dirname [info script]] .. .. fts3 unicode $file } source [fts3_unicode_path parseunicode.tcl] diff --git a/manifest b/manifest index 6bf867686a..5c9cdcc430 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sproblem\swith\scorrupt\sdatabase\shandling\sin\sfts5. -D 2018-12-27T20:12:02.887 +C Fix\sproblems\sin\sfts5\sfound\sby\sASAN. +D 2018-12-28T07:37:22.836 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F Makefile.in d8b254f8bb81bab43c340d70d17dc3babab40fcc8a348c8255881f780a45fee6 @@ -97,7 +97,7 @@ F ext/fts3/fts3_tokenizer.c a22bf311a71f3efa9d7012d8cc48fc9b0f3dace7 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c b1902e9ad47a6569fbb8ecb5ce52f20fe59b590d5c5e3bbdd56b10b03bdf632b -F ext/fts3/fts3_unicode2.c e49f9e015f239bf5faf2f4fa483bbf1b08a9978f0ad1f31159d952f8b8a10d08 +F ext/fts3/fts3_unicode2.c faf0c750a1d0e9c9902f947212fc88f84e9450cddb3d1ffdbacbd0084b91d647 F ext/fts3/fts3_write.c e36d2f7e8f180e8030e92a5c2d09ccf87021afedcc5148a9d823b496667bf2f2 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 @@ -105,24 +105,24 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87 F ext/fts3/tool/fts3view.c 202801a2056995b763864d60c2dee744d46f1677 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl 2ea30d8122ccf1e33142c9cc8913d8cad9eb6668db359a228f10aeb37e2ab863 +F ext/fts3/unicode/mkunicode.tcl 2315b3f8c83dcb09091009db01b321fb401d98d8c5489c147c1a7c5a54d21e20 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 4f5d19b7973dae23de368728f06d3eb1fe9f5cca2990366b40e9379996f35e61 -F ext/fts5/fts5Int.h 39f12034b598df4e0f59bbe6cf03af03a905a534b04f182d155641a04e1eb797 +F ext/fts5/fts5Int.h 8590e08a485c98a7e3075ed72abe3452fe944a9e58e63dfa51b732cced344cab F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c 5aef080ba3c8947e22f38ce1ff9fe548e4a740e72b77241f35ed941ae128d2c7 +F ext/fts5/fts5_expr.c 0eff5bad90838be83c5cc16837ca215a961abe05cfbf741a285ba62a79ad586f F ext/fts5/fts5_hash.c 32be400cf761868c9db33efe81a06eb19a17c5402ad477ee9efb51301546dd55 -F ext/fts5/fts5_index.c f3601579efeca9a859bf4cc82cbe99106fa8e3277a96b96c44ce35c7ceffb6d0 +F ext/fts5/fts5_index.c c67a3bf1c185ff05937a9e3a01b8a6981f5a6add8da93911f461d59e57eea8d5 F ext/fts5/fts5_main.c 287a1a56580df304d7fa2fc1890f85b9cb6ac6b9e7c8af7dfa2151528db4b059 F ext/fts5/fts5_storage.c 5862f1b785a983acb8420281340f3f424896ab48f396f6fd8540787be7459139 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95 F ext/fts5/fts5_test_mi.c 65864ba1e5c34a61d409c4c587e0bbe0466eb4f8f478d85dc42a92caad1338e6 -F ext/fts5/fts5_test_tok.c 80de1a4b1a3caa216c3be8862440f0117a8357dd9b7cfc5a2a2ce11fe6eb64ae -F ext/fts5/fts5_tokenize.c ca2b6a033794945ac505241a86b0aa978709c23aa2e6121984d3e3ede96003c8 -F ext/fts5/fts5_unicode2.c 03bc8ebf44d825d0b7ac519f7b16139cf8e879ae7a27e62fb6326f0d68e8bdd3 +F ext/fts5/fts5_test_tok.c 6a5574354ce61a98737e150fd4f7a002000db0cb4bcd655dc8694cb3e035381d +F ext/fts5/fts5_tokenize.c 240f849c9136103f77a67818deb172d8a7564cbefaf00ec7b2fd04ab17d6d66c +F ext/fts5/fts5_unicode2.c 3f1bad6f2924ad13e25408f00301edd12d4216388a11daa518cb28f79b337f5e F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c fbe38044889b2d2d99babeeef239c620fb0332bb928a84506ac748d81500b354 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05 @@ -211,7 +211,7 @@ F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d21537 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602 F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9 F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e -F ext/fts5/test/fts5unicode3.test 9cbc82e2b02e2e3b7504103580c90f095e07fe8230b1951a9ed7558717b5feb7 +F ext/fts5/test/fts5unicode3.test 590c72e18195bda2446133f9d82d04a4e89d094bba58c75ae10f4afc6faa0744 F ext/fts5/test/fts5unicode4.test 6463301d669f963c83988017aa354108be0b947d325aef58d3abddf27147b687 F ext/fts5/test/fts5unindexed.test 9021af86a0fb9fc616f7a69a996db0116e7936d0db63892db6bafabbec21af4d F ext/fts5/test/fts5update.test 0737876e20e97a6a6abf45de19fc99315727bcee6a83fadcada1cc080b9aa8f0 @@ -1794,7 +1794,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d9e4f9574659c6285deba3dca68eca9989e615971cea3a901ffcda12fb4ee36f -R 308e6395f3ea79d50b990f4e574e128e +P fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9 +R 31f0c322d87dce0c17eac6b9b39fdaff U dan -Z d6ac7bbdbfe101fd5a8ad83d11fdc195 +Z f209c1d4c450aa64c337c8ad21da7144 diff --git a/manifest.uuid b/manifest.uuid index 197a3c367c..893a553862 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb0d7fba07865ff557bfcdacfda7b625d099ac8b4dcbe0c08cd2d4b84a068bb9 \ No newline at end of file +c564bf870106faef297594a51995619c80311d06bd5f8a0c7644f666f22ba576 \ No newline at end of file -- 2.47.2