Fix the sqlite3_hard_heap_limit() so that it works with sqlite3_realloc64()

in addition to sqlite3_malloc64().  Improvements to OOM processing and
debugging aids in the fuzzcheck utility.

FossilOrigin-Name: 602d7369166d406a26834aa47d71d565a17d377d32e41f308821a50b41f91896

diff --git a/manifest b/manifest
index 9a823e9a09756881da1ee49927b3e5903050ca21..04ee23927bcb493ee4cc84ec80dfdd24f9af2e7e 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\san\sassert()\sinside\sof\sdebug-only\scode\sthat\scan\sfail\sfollowing\san\sOOM.
-D 2020-10-20T12:37:51.678
+C Fix\sthe\ssqlite3_hard_heap_limit()\sso\sthat\sit\sworks\swith\ssqlite3_realloc64()\nin\saddition\sto\ssqlite3_malloc64().\s\sImprovements\sto\sOOM\sprocessing\sand\ndebugging\saids\sin\sthe\sfuzzcheck\sutility.
+D 2020-10-20T14:40:53.201
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -502,13 +502,13 @@ F src/insert.c 7e081d33aab4a9d761c39dccf3c3872c35501565d2ed9db66301918d23bc7901
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 436af4968c6954d304fce9efa12719367bd8f37b19b93b71d6ad607e85adbb47
 F src/main.c 54d61d1e94f225636053c862c930d7dd7b29162b93d3f3ecb00439910ccba02a
-F src/malloc.c 22d5bdd9fe88ae4fad1b91a1b9735104b82853ffef868f1f05517d60dc1875f5
+F src/malloc.c 413b0ffea2b45c37c6cf2ed5509f6d6b508b0a033caa9b4bf90bed4ee044a2ff
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c b93b8762ab999a29ae7751532dadf0a1ac78040308a5fb1d17fcc365171d67eb
 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6
 F src/mem5.c 9bf955937b07f8c32541c8a9991f33ce3173d944
-F src/memdb.c 23e8f0b975e91e23f60794935252bec2040d8db09f598fd738d5903dd6c0143f
+F src/memdb.c ab0632d42407e866d2b616bd19d4211ac0ad1b430f04c4e187d60005b8700b98
 F src/memjournal.c 90b2ca7e2f465d57c16b69d15a9f3e3294af61088eb4938f2f7664d5ac50f813
 F src/msvc.h 3a15918220367a8876be3fa4f2abe423a861491e84b864fb2b7426bf022a28f8
 F src/mutex.c 5e3409715552348732e97b9194abe92fdfcd934cfb681df4ba0ab87ac6c18d25
@@ -1029,7 +1029,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c
 F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634
 F test/fuzz_common.tcl b7197de6ed1ee8250a4f82d67876f4561b42ee8cbbfc6160dcb66331bad3f830
 F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2
-F test/fuzzcheck.c 656ee850f331872a784e7d6a10649efe2af123bdaacb728b5a03e4faee8b959c
+F test/fuzzcheck.c 5f81f2cc65f13068620245f2e2c6059657d3b26be476df379ae2da539f17676d
 F test/fuzzdata1.db d36e88741b4f23bcbaaf55b006290669d03c6c891cf13c7b3a53bc1b097b693f
 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f
 F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba
@@ -1883,7 +1883,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 75a0288871ccb2a69a636cbb328fe19045a0d0ef96a193ecd118b9a196784d2d
-R 8238397a2980379334c8295b533cb894
+P 79da254f41a0d7d5fdc57f0cc40b3560bfcf7f79eed24a9fada1b6b4ca3adb5a
+R 80e6b2572c25cd544b35c7442c689807
 U drh
-Z 64f5a98fdc563dc665a2183f760ec9f1
+Z d733a42fa58690400682bda5195b036c

diff --git a/manifest.uuid b/manifest.uuid
index 76a23f18bc0c7e0cf029c8a735283c7eb45f923e..871e6c40f015df47003e6f7447bcfeb5179de3f6 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-79da254f41a0d7d5fdc57f0cc40b3560bfcf7f79eed24a9fada1b6b4ca3adb5a
\ No newline at end of file
+602d7369166d406a26834aa47d71d565a17d377d32e41f308821a50b41f91896
\ No newline at end of file

diff --git a/src/malloc.c b/src/malloc.c
index 70e6ff94e5a8affdcae26eb889ef03a3393a3f1d..35ad21ecc7541c92423327313347a6f81e530800 100644 (file)
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -474,12 +474,17 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){
   if( nOld==nNew ){
     pNew = pOld;
   }else if( sqlite3GlobalConfig.bMemstat ){
+    sqlite3_int64 nUsed;
     sqlite3_mutex_enter(mem0.mutex);
     sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes);
     nDiff = nNew - nOld;
-    if( nDiff>0 && sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= 
+    if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= 
           mem0.alarmThreshold-nDiff ){
       sqlite3MallocAlarm(nDiff);
+      if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){
+        sqlite3_mutex_leave(mem0.mutex);
+        return 0;
+      }
     }
     pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew);
 #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT

diff --git a/src/memdb.c b/src/memdb.c
index d07023dbab17fa4840e6897430db4636bfcc4165..a48980aa341736fd918fa14ecf7f860a3e15ee44 100644 (file)
--- a/src/memdb.c
+++ b/src/memdb.c
@@ -576,8 +576,12 @@ int sqlite3_deserialize(
     goto end_deserialize;
   }    
   zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+    sqlite3_free(zSql);
+  }
   if( rc ) goto end_deserialize;
   db->init.iDb = (u8)iDb;
   db->init.reopenMemdb = 1;

diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c
index c57cf565215f10848b525b468014e0f78308750a..cf49956b471260269b0fbf934d35e2582e97d4cd 100644 (file)
--- a/test/fuzzcheck.c
+++ b/test/fuzzcheck.c
@@ -627,7 +627,7 @@ static int decodeDatabase(
   unsigned char **paDecode,      /* OUT: decoded database file */
   int *pnDecode                  /* OUT: Size of decoded database */
 ){
-  unsigned char *a;              /* Database under construction */
+  unsigned char *a, *aNew;       /* Database under construction */
   int mx = 0;                    /* Current size of the database */
   sqlite3_uint64 nAlloc = 4096;  /* Space allocated in a[] */
   unsigned int i;                /* Next byte of zIn[] to read */
@@ -673,11 +673,12 @@ static int decodeDatabase(
             }
             newSize = MX_FILE_SZ;
           }
-          a = sqlite3_realloc64( a, newSize );
-          if( a==0 ){
-            fprintf(stderr, "Out of memory!\n");
-            exit(1);
+          aNew = sqlite3_realloc64( a, newSize );
+          if( aNew==0 ){
+            sqlite3_free(a);
+            return -1;
           }
+          a = aNew;
           assert( newSize > nAlloc );
           memset(a+nAlloc, 0, (size_t)(newSize - nAlloc));
           nAlloc = newSize;
@@ -851,7 +852,8 @@ int runCombinedDbSqlInput(const uint8_t *aData, size_t nByte){
     int nAlloc = 0;
     int nNotUsed = 0;
     sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0);
-    fprintf(stderr,"Memory leak in mutator: %lld bytes in %d allocations\n",
+    fprintf(stderr,"memory leak prior to test start:"
+                   " %lld bytes in %d allocations\n",
             sqlite3_memory_used(), nAlloc);
     exit(1);
   }
@@ -866,7 +868,10 @@ int runCombinedDbSqlInput(const uint8_t *aData, size_t nByte){
     fflush(stdout);
   }
   rc = sqlite3_open(0, &cx.db);
-  if( rc ) return 1;
+  if( rc ){
+    sqlite3_free(aDb);
+    return 1;
+  }
   if( bVdbeDebug ){
     sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0);
   }
@@ -1410,7 +1415,8 @@ static void showHelp(void){
 "  --export-sql DIR     Write SQL to file(s) in DIR. Also works with --sqlid\n"
 "  --help               Show this help text\n"
 "  --info               Show information about SOURCE-DB w/o running tests\n"
-"  --limit-depth N      Limit expression depth to N\n"
+"  --limit-depth N      Limit expression depth to N.  Default: 500\n"
+"  --limit-heap N       Limit heap memory to N.  Default: 100M\n"
 "  --limit-mem N        Limit memory used by test SQLite instance to N bytes\n"
 "  --limit-vdbe         Panic if any test runs for more than 100,000 cycles\n"
 "  --load-sql ARGS...   Load SQL scripts fron files into SOURCE-DB\n"
@@ -1424,6 +1430,7 @@ static void showHelp(void){
 "  -q|--quiet           Reduced output\n"
 "  --rebuild            Rebuild and vacuum the database file\n"
 "  --result-trace       Show the results of each SQL command\n"
+"  --skip N             Skip the first N test cases\n"
 "  --spinner            Use a spinner to show progress\n"
 "  --sqlid N            Use only SQL where sqlid=N\n"
 "  --timeout N          Abort if any single test needs more than N seconds\n"
@@ -1451,6 +1458,7 @@ int main(int argc, char **argv){
   int rebuildFlag = 0;         /* --rebuild */
   int vdbeLimitFlag = 0;       /* --limit-vdbe */
   int infoFlag = 0;            /* --info */
+  int nSkip = 0;               /* --skip */
   int bSpinner = 0;            /* True for --spinner */
   int timeoutTest = 0;         /* undocumented --timeout-test flag */
   int runFlags = 0;            /* Flags sent to runSql() */
@@ -1520,6 +1528,10 @@ int main(int argc, char **argv){
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
         depthLimit = integerValue(argv[++i]);
       }else
+      if( strcmp(z,"limit-heap")==0 ){
+        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
+        heapLimit = integerValue(argv[++i]);
+      }else
       if( strcmp(z,"limit-mem")==0 ){
         if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
         nMem = integerValue(argv[++i]);
@@ -1578,6 +1590,10 @@ int main(int argc, char **argv){
       if( strcmp(z,"result-trace")==0 ){
         runFlags |= SQL_OUTPUT;
       }else
+      if( strcmp(z,"skip")==0 ){
+        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
+        nSkip = atoi(argv[++i]);
+      }else
       if( strcmp(z,"spinner")==0 ){
         bSpinner = 1;
       }else
@@ -1898,7 +1914,11 @@ int main(int argc, char **argv){
             prevAmt = amt;
           }
         }
-        runCombinedDbSqlInput(pSql->a, pSql->sz);
+        if( nSkip>0 ){
+          nSkip--;
+        }else{
+          runCombinedDbSqlInput(pSql->a, pSql->sz);
+        }
         nTest++;
         g.zTestName[0] = 0;
         disableOom();
@@ -1927,6 +1947,10 @@ int main(int argc, char **argv){
             prevAmt = amt;
           }
         }
+        if( nSkip>0 ){
+          nSkip--;
+          continue;
+        }
         createVFile("main.db", pDb->sz, pDb->a);
         sqlite3_randomness(0,0);
         if( ossFuzzThisDb ){