From: drh Date: Tue, 20 Oct 2020 14:40:53 +0000 (+0000) Subject: Fix the sqlite3_hard_heap_limit() so that it works with sqlite3_realloc64() X-Git-Tag: version-3.34.0~48 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=672f07c642fad9ad76b1a8cd1c1d196d0286ffc4;p=thirdparty%2Fsqlite.git Fix the sqlite3_hard_heap_limit() so that it works with sqlite3_realloc64() in addition to sqlite3_malloc64(). Improvements to OOM processing and debugging aids in the fuzzcheck utility. FossilOrigin-Name: 602d7369166d406a26834aa47d71d565a17d377d32e41f308821a50b41f91896 --- diff --git a/manifest b/manifest index 9a823e9a09..04ee23927b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sinside\sof\sdebug-only\scode\sthat\scan\sfail\sfollowing\san\sOOM. -D 2020-10-20T12:37:51.678 +C Fix\sthe\ssqlite3_hard_heap_limit()\sso\sthat\sit\sworks\swith\ssqlite3_realloc64()\nin\saddition\sto\ssqlite3_malloc64().\s\sImprovements\sto\sOOM\sprocessing\sand\ndebugging\saids\sin\sthe\sfuzzcheck\sutility. +D 2020-10-20T14:40:53.201 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,13 +502,13 @@ F src/insert.c 7e081d33aab4a9d761c39dccf3c3872c35501565d2ed9db66301918d23bc7901 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 436af4968c6954d304fce9efa12719367bd8f37b19b93b71d6ad607e85adbb47 F src/main.c 54d61d1e94f225636053c862c930d7dd7b29162b93d3f3ecb00439910ccba02a -F src/malloc.c 22d5bdd9fe88ae4fad1b91a1b9735104b82853ffef868f1f05517d60dc1875f5 +F src/malloc.c 413b0ffea2b45c37c6cf2ed5509f6d6b508b0a033caa9b4bf90bed4ee044a2ff F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c b93b8762ab999a29ae7751532dadf0a1ac78040308a5fb1d17fcc365171d67eb F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944 -F src/memdb.c 23e8f0b975e91e23f60794935252bec2040d8db09f598fd738d5903dd6c0143f +F src/memdb.c ab0632d42407e866d2b616bd19d4211ac0ad1b430f04c4e187d60005b8700b98 F src/memjournal.c 90b2ca7e2f465d57c16b69d15a9f3e3294af61088eb4938f2f7664d5ac50f813 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25 @@ -1029,7 +1029,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 656ee850f331872a784e7d6a10649efe2af123bdaacb728b5a03e4faee8b959c +F test/fuzzcheck.c 5f81f2cc65f13068620245f2e2c6059657d3b26be476df379ae2da539f17676d F test/fuzzdata1.db d36e88741b4f23bcbaaf55b006290669d03c6c891cf13c7b3a53bc1b097b693f F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1883,7 +1883,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75a0288871ccb2a69a636cbb328fe19045a0d0ef96a193ecd118b9a196784d2d -R 8238397a2980379334c8295b533cb894 +P 79da254f41a0d7d5fdc57f0cc40b3560bfcf7f79eed24a9fada1b6b4ca3adb5a +R 80e6b2572c25cd544b35c7442c689807 U drh -Z 64f5a98fdc563dc665a2183f760ec9f1 +Z d733a42fa58690400682bda5195b036c diff --git a/manifest.uuid b/manifest.uuid index 76a23f18bc..871e6c40f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79da254f41a0d7d5fdc57f0cc40b3560bfcf7f79eed24a9fada1b6b4ca3adb5a \ No newline at end of file +602d7369166d406a26834aa47d71d565a17d377d32e41f308821a50b41f91896 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 70e6ff94e5..35ad21ecc7 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -474,12 +474,17 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_int64 nUsed; sqlite3_mutex_enter(mem0.mutex); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; - if( nDiff>0 && sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); + if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ + sqlite3_mutex_leave(mem0.mutex); + return 0; + } } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT diff --git a/src/memdb.c b/src/memdb.c index d07023dbab..a48980aa34 100644 --- a/src/memdb.c +++ b/src/memdb.c @@ -576,8 +576,12 @@ int sqlite3_deserialize( goto end_deserialize; } zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema); - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } if( rc ) goto end_deserialize; db->init.iDb = (u8)iDb; db->init.reopenMemdb = 1; diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index c57cf56521..cf49956b47 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -627,7 +627,7 @@ static int decodeDatabase( unsigned char **paDecode, /* OUT: decoded database file */ int *pnDecode /* OUT: Size of decoded database */ ){ - unsigned char *a; /* Database under construction */ + unsigned char *a, *aNew; /* Database under construction */ int mx = 0; /* Current size of the database */ sqlite3_uint64 nAlloc = 4096; /* Space allocated in a[] */ unsigned int i; /* Next byte of zIn[] to read */ @@ -673,11 +673,12 @@ static int decodeDatabase( } newSize = MX_FILE_SZ; } - a = sqlite3_realloc64( a, newSize ); - if( a==0 ){ - fprintf(stderr, "Out of memory!\n"); - exit(1); + aNew = sqlite3_realloc64( a, newSize ); + if( aNew==0 ){ + sqlite3_free(a); + return -1; } + a = aNew; assert( newSize > nAlloc ); memset(a+nAlloc, 0, (size_t)(newSize - nAlloc)); nAlloc = newSize; @@ -851,7 +852,8 @@ int runCombinedDbSqlInput(const uint8_t *aData, size_t nByte){ int nAlloc = 0; int nNotUsed = 0; sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0); - fprintf(stderr,"Memory leak in mutator: %lld bytes in %d allocations\n", + fprintf(stderr,"memory leak prior to test start:" + " %lld bytes in %d allocations\n", sqlite3_memory_used(), nAlloc); exit(1); } @@ -866,7 +868,10 @@ int runCombinedDbSqlInput(const uint8_t *aData, size_t nByte){ fflush(stdout); } rc = sqlite3_open(0, &cx.db); - if( rc ) return 1; + if( rc ){ + sqlite3_free(aDb); + return 1; + } if( bVdbeDebug ){ sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0); } @@ -1410,7 +1415,8 @@ static void showHelp(void){ " --export-sql DIR Write SQL to file(s) in DIR. Also works with --sqlid\n" " --help Show this help text\n" " --info Show information about SOURCE-DB w/o running tests\n" -" --limit-depth N Limit expression depth to N\n" +" --limit-depth N Limit expression depth to N. Default: 500\n" +" --limit-heap N Limit heap memory to N. Default: 100M\n" " --limit-mem N Limit memory used by test SQLite instance to N bytes\n" " --limit-vdbe Panic if any test runs for more than 100,000 cycles\n" " --load-sql ARGS... Load SQL scripts fron files into SOURCE-DB\n" @@ -1424,6 +1430,7 @@ static void showHelp(void){ " -q|--quiet Reduced output\n" " --rebuild Rebuild and vacuum the database file\n" " --result-trace Show the results of each SQL command\n" +" --skip N Skip the first N test cases\n" " --spinner Use a spinner to show progress\n" " --sqlid N Use only SQL where sqlid=N\n" " --timeout N Abort if any single test needs more than N seconds\n" @@ -1451,6 +1458,7 @@ int main(int argc, char **argv){ int rebuildFlag = 0; /* --rebuild */ int vdbeLimitFlag = 0; /* --limit-vdbe */ int infoFlag = 0; /* --info */ + int nSkip = 0; /* --skip */ int bSpinner = 0; /* True for --spinner */ int timeoutTest = 0; /* undocumented --timeout-test flag */ int runFlags = 0; /* Flags sent to runSql() */ @@ -1520,6 +1528,10 @@ int main(int argc, char **argv){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); depthLimit = integerValue(argv[++i]); }else + if( strcmp(z,"limit-heap")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + heapLimit = integerValue(argv[++i]); + }else if( strcmp(z,"limit-mem")==0 ){ if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); nMem = integerValue(argv[++i]); @@ -1578,6 +1590,10 @@ int main(int argc, char **argv){ if( strcmp(z,"result-trace")==0 ){ runFlags |= SQL_OUTPUT; }else + if( strcmp(z,"skip")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + nSkip = atoi(argv[++i]); + }else if( strcmp(z,"spinner")==0 ){ bSpinner = 1; }else @@ -1898,7 +1914,11 @@ int main(int argc, char **argv){ prevAmt = amt; } } - runCombinedDbSqlInput(pSql->a, pSql->sz); + if( nSkip>0 ){ + nSkip--; + }else{ + runCombinedDbSqlInput(pSql->a, pSql->sz); + } nTest++; g.zTestName[0] = 0; disableOom(); @@ -1927,6 +1947,10 @@ int main(int argc, char **argv){ prevAmt = amt; } } + if( nSkip>0 ){ + nSkip--; + continue; + } createVFile("main.db", pDb->sz, pDb->a); sqlite3_randomness(0,0); if( ossFuzzThisDb ){