From: dan Date: Fri, 19 May 2017 12:32:32 +0000 (+0000) Subject: Fix some problems in fts5 code detected by -fsanitize=undefined. X-Git-Tag: version-3.19.0~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=22025ef1984c4f3f3774f0e4b232d0166f55d458;p=thirdparty%2Fsqlite.git Fix some problems in fts5 code detected by -fsanitize=undefined. FossilOrigin-Name: 35f721045dfe3f82e016938ab1a668bfc37b6b57b8813cc963ef640ec82de58d --- diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index 0098846be9..b11689745b 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -67,9 +67,11 @@ void sqlite3Fts5BufferAppendBlob( const u8 *pData ){ assert_nc( *pRc || nData>=0 ); - if( fts5BufferGrow(pRc, pBuf, nData) ) return; - memcpy(&pBuf->p[pBuf->n], pData, nData); - pBuf->n += nData; + if( nData ){ + if( fts5BufferGrow(pRc, pBuf, nData) ) return; + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; + } } /* @@ -246,8 +248,8 @@ void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ pRet = sqlite3_malloc(nByte); - if( pRet==0 && nByte>0 ){ - *pRc = SQLITE_NOMEM; + if( pRet==0 ){ + if( nByte>0 ) *pRc = SQLITE_NOMEM; }else{ memset(pRet, 0, nByte); } diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index afa2a30739..1327597ce7 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -36,9 +36,10 @@ struct Fts5Hash { /* ** Each entry in the hash table is represented by an object of the -** following type. Each object, its key (zKey[]) and its current data -** are stored in a single memory allocation. The position list data -** immediately follows the key data in memory. +** following type. Each object, its key (a nul-terminated string) and +** its current data are stored in a single memory allocation. The +** key immediately follows the object in memory. The position list +** data immediately follows the key data in memory. ** ** The data that follows the key is in a similar, but not identical format ** to the doclist data stored in the database. It is: @@ -62,20 +63,20 @@ struct Fts5HashEntry { int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ - int nKey; /* Length of zKey[] in bytes */ + int nKey; /* Length of key in bytes */ u8 bDel; /* Set delete-flag @ iSzPoslist */ u8 bContent; /* Set content-flag (detail=none mode) */ i16 iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ - char zKey[8]; /* Nul-terminated entry key */ }; /* -** Size of Fts5HashEntry without the zKey[] array. +** Eqivalent to: +** +** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } */ -#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) - +#define fts5EntryKey(p) ( ((char *)(&(p)[1])) ) /* @@ -173,7 +174,7 @@ static int fts5HashResize(Fts5Hash *pHash){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey)); + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), strlen(fts5EntryKey(p))); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -244,9 +245,10 @@ int sqlite3Fts5HashWrite( /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ - if( p->zKey[0]==bByte + char *zKey = fts5EntryKey(p); + if( zKey[0]==bByte && p->nKey==nToken - && memcmp(&p->zKey[1], pToken, nToken)==0 + && memcmp(&zKey[1], pToken, nToken)==0 ){ break; } @@ -255,7 +257,8 @@ int sqlite3Fts5HashWrite( /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ /* Figure out how much space to allocate */ - int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; + char *zKey; + int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; /* Grow the Fts5Hash.aSlot[] array if necessary. */ @@ -268,14 +271,15 @@ int sqlite3Fts5HashWrite( /* Allocate new Fts5HashEntry and add it to the hash table. */ p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, FTS5_HASHENTRYSIZE); + memset(p, 0, sizeof(Fts5HashEntry)); p->nAlloc = nByte; - p->zKey[0] = bByte; - memcpy(&p->zKey[1], pToken, nToken); - assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); + zKey = fts5EntryKey(p); + zKey[0] = bByte; + memcpy(&zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); p->nKey = nToken; - p->zKey[nToken+1] = '\0'; - p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; + zKey[nToken+1] = '\0'; + p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -393,9 +397,11 @@ static Fts5HashEntry *fts5HashEntryMerge( p1 = 0; }else{ int i = 0; - while( p1->zKey[i]==p2->zKey[i] ) i++; + char *zKey1 = fts5EntryKey(p1); + char *zKey2 = fts5EntryKey(p2); + while( zKey1[i]==zKey2[i] ) i++; - if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){ + if( ((u8)zKey1[i])>((u8)zKey2[i]) ){ /* p2 is smaller */ *ppOut = p2; ppOut = &p2->pScanNext; @@ -438,7 +444,7 @@ static int fts5HashEntrySort( for(iSlot=0; iSlotnSlot; iSlot++){ Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ - if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){ + if( pTerm==0 || 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm) ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; for(i=0; ap[i]; i++){ @@ -471,16 +477,18 @@ int sqlite3Fts5HashQuery( int *pnDoclist /* OUT: Size of doclist in bytes */ ){ unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); + char *zKey; Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ - if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; + zKey = fts5EntryKey(p); + if( memcmp(zKey, pTerm, nTerm)==0 && zKey[nTerm]==0 ) break; } if( p ){ fts5HashAddPoslistSize(pHash, p); - *ppDoclist = (const u8*)&p->zKey[nTerm+1]; - *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + *ppDoclist = (const u8*)&zKey[nTerm+1]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); }else{ *ppDoclist = 0; *pnDoclist = 0; @@ -513,11 +521,12 @@ void sqlite3Fts5HashScanEntry( ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ - int nTerm = (int)strlen(p->zKey); + char *zKey = fts5EntryKey(p); + int nTerm = (int)strlen(zKey); fts5HashAddPoslistSize(pHash, p); - *pzTerm = p->zKey; - *ppDoclist = (const u8*)&p->zKey[nTerm+1]; - *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + *pzTerm = zKey; + *ppDoclist = (const u8*)&zKey[nTerm+1]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1); }else{ *pzTerm = 0; *ppDoclist = 0; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 6dc9f8470e..7b13718ee0 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -5093,7 +5093,7 @@ static void fts5SetupPrefixIter( if( pData ){ pData->p = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; - memcpy(pData->p, doclist.p, doclist.n); + if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } fts5BufferFree(&doclist); @@ -5332,7 +5332,7 @@ int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ - memcpy(&buf.p[1], pToken, nToken); + if( nToken ) memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to @@ -5381,7 +5381,7 @@ int sqlite3Fts5IndexQuery( } if( p->rc ){ - sqlite3Fts5IterClose(&pRet->base); + sqlite3Fts5IterClose((Fts5IndexIter*)pRet); pRet = 0; fts5CloseReader(p); } diff --git a/manifest b/manifest index e49d285261..c4c9173c64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\stool/warnings.sh\sscript\sto\sautomatically\suse\sthe\sright\soptions\son\sOpenBSD.. -D 2017-05-16T09:49:42.023 +C Fix\ssome\sproblems\sin\sfts5\scode\sdetected\sby\s-fsanitize=undefined. +D 2017-05-19T12:32:32.722 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 8eeb80162074004e906b53d7340a12a14c471a83743aab975947e95ce061efcc @@ -101,11 +101,11 @@ F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 F ext/fts5/fts5Int.h 15e7514b46a845937d7c62e5c69e935091f0dbb72eb61aa4c8bcfbd39fdea158 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 -F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd +F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c f2825f714d91bbe62ab5820aee9ad12e0c94205b2a01725eaa9072415ae9ff1c -F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 -F ext/fts5/fts5_index.c 9ce10106f42f8b84278a8ea859940224e2af5f0cc882f909364469f6f52769cb +F ext/fts5/fts5_hash.c 534d5591f479c0999543689122ad6952823bc7c85273a0ff4f7f91d9f914a54b +F ext/fts5/fts5_index.c cdceac47287c66500214ee946ca871ac48027a82a0ca82177c1c6af19f181ca0 F ext/fts5/fts5_main.c 1ba0e7806886c1bc16e20d0dde1c2b535d1aeb98cbbb937c4c3e064af5ac6f03 F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a @@ -1580,7 +1580,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb0d9281588b8cc24bf2f1f10d0c56277004226adaa2ce5037782503b283b45d -R 0af7da97ec99dfdc47acd61562c849dd +P 7940bff32aa6ea868a53680822d148b7ec7a075c01ae9e0d5ad9859bcc339054 +R 43b2f6640d15533e1a63dfe250cd3268 U dan -Z 1c514bbb65da185a2b54f96dcba53c59 +Z f9fce9521e2a47d8f39c7a8284ffa988 diff --git a/manifest.uuid b/manifest.uuid index 23df76dc03..cee88ede1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7940bff32aa6ea868a53680822d148b7ec7a075c01ae9e0d5ad9859bcc339054 \ No newline at end of file +35f721045dfe3f82e016938ab1a668bfc37b6b57b8813cc963ef640ec82de58d \ No newline at end of file