From: drh <>
Date: Tue, 18 Feb 2025 20:27:04 +0000 (+0000)
Subject: Add the SQLITE_MEM5_FAILOVER compile-time option, which if enabled allows
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=0e9332fb6376fcd262184ac0128bfc25a57a8859;p=thirdparty%2Fsqlite.git

Add the SQLITE_MEM5_FAILOVER compile-time option, which if enabled allows
MEMSYS5 to go out to the system heap if it is unable to fulfill an allocation
request.

FossilOrigin-Name: bdd8617b3f4a4dcec03095e1f8810bd020477378fde7bbc2556f7ed8c845dd4e
---

diff --git a/manifest b/manifest
index f4faa2a67f..99a7c000ce 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sSQLITE_FTS5_ENABLE_TEST_MI\sbuilds\savoid\sreading\sthe\sdatabase\sschema\sfrom\nwithin\ssqlite3_open().
-D 2025-02-18T15:11:30.645
+C Add\sthe\sSQLITE_MEM5_FAILOVER\scompile-time\soption,\swhich\sif\senabled\sallows\nMEMSYS5\sto\sgo\sout\sto\sthe\ssystem\sheap\sif\sit\sis\sunable\sto\sfulfill\san\sallocation\nrequest.
+D 2025-02-18T20:27:04.279
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -748,7 +748,7 @@ F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
 F src/mem2.c c8bfc9446fd0798bddd495eb5d9dbafa7d4b7287d8c22d50a83ac9daa26d8a75
 F src/mem3.c 30301196cace2a085cbedee1326a49f4b26deff0af68774ca82c1f7c06fda4f6
-F src/mem5.c b7da5c10a726aacacc9ad7cdcb0667deec643e117591cc69cf9b4b9e7f3e96ff
+F src/mem5.c 8cd656ba3647e2dfa3f5e62496eb5308db8eefcd42b388670c7c4ed773a1d124
 F src/memdb.c a3feb427cdd4036ea2db0ba56d152f14c8212ca760ccb05fb7aa49ff6b897df3
 F src/memjournal.c c283c6c95d940eb9dc70f1863eef3ee40382dbd35e5a1108026e7817c206e8a0
 F src/msvc.h 80b35f95d93bf996ccb3e498535255f2ef1118c78764719a7cd15ab4106ccac9
@@ -779,13 +779,13 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
 F src/resolve.c 626c24b258b111f75c22107aa5614ad89810df3026f5ca071116d3fe75925c75
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 57893cc8b099f231f7ed5b84faff14841f2aabb4776e32e17fae00aeae0a8993
-F src/shell.c.in b377a59822f207106424f08aead37e78b609222e98f86f04cc8a03563ccf3237
-F src/sqlite.h.in 8d4486fb28a90de818ac1e8c6206ea458e7de6bd8e0dfa3d554494f155be8c01
+F src/shell.c.in cbb4b0e27eba5c7cbb0c50ff9ce6e7508a1f555f473153b8caf23711ceff55f5
+F src/sqlite.h.in 08b88ac4488121affd43c54c07660f5742e47251c8bb956e4f95f9b4eb9450e7
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
 F src/sqliteInt.h 020aff180111b7dfe5bbdf8e59e8595c195b956488e9ca955f876cb7482e6de5
 F src/sqliteLimit.h 1bbdbf72bd0411d003267ffebc59a262f061df5653027a75627d03f48ca30523
-F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
+F src/status.c a676968f09d44a1aab2741c61efda9c2732a8681047bec5ead953af5e3865848
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
 F src/tclsqlite.c 5c1e367e26711044730c93d4b81312170918a8d1fe811f45be740ab48f7de8c1
 F src/tclsqlite.h 65e2c761446e1c9fa0342b7d2612a703483643c8b6a316d12a65b745a4727395
@@ -2207,8 +2207,11 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 57caa3136d1bfca06e4f2285734a4977b8d3fa1f75bf87453b975867e9de38fc
-R 35202329cbe88699b6bddc518439e5e7
-U dan
-Z 2607e4e091da584eea58069e1dc59b17
+P 15dc524fd4113026cc542140c39c1c8f9e052d36946f0c599f282d9ac27efdab
+R 1c6fc5fc5921c8f54e054da530ee60da
+T *branch * mem5-failover
+T *sym-mem5-failover *
+T -sym-trunk *
+U drh
+Z a3502dae1ff66c5ee52104389dd44963
 # Remove this line to create a well-formed Fossil manifest.
diff --git a/manifest.uuid b/manifest.uuid
index 1217ba4cdf..f5a6cfd441 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-15dc524fd4113026cc542140c39c1c8f9e052d36946f0c599f282d9ac27efdab
+bdd8617b3f4a4dcec03095e1f8810bd020477378fde7bbc2556f7ed8c845dd4e
diff --git a/src/mem5.c b/src/mem5.c
index 02f4c2744c..1b1e1bef4b 100644
--- a/src/mem5.c
+++ b/src/mem5.c
@@ -94,8 +94,13 @@ static SQLITE_WSD struct Mem5Global {
   ** Memory available for allocation
   */
   int szAtom;      /* Smallest possible allocation in bytes */
-  int nBlock;      /* Number of szAtom sized blocks in zPool */
-  u8 *zPool;       /* Memory available to be allocated */
+  int nBlock;      /* Number of szAtom sized blocks in aPool */
+  u8 *aPool;       /* Memory available to be allocated */
+  u8 *aEnd;        /* Last byte of aPool[] */
+
+#ifdef SQLITE_MEM5_FAILOVER
+  u64 nFailover;   /* Number of fail-over allocations taken from system heap */
+#endif
   
   /*
   ** Mutex to control access to the memory allocation subsystem.
@@ -137,10 +142,10 @@ static SQLITE_WSD struct Mem5Global {
 #define mem5 GLOBAL(struct Mem5Global, mem5)
 
 /*
-** Assuming mem5.zPool is divided up into an array of Mem5Link
+** Assuming mem5.aPool is divided up into an array of Mem5Link
 ** structures, return a pointer to the idx-th such link.
 */
-#define MEM5LINK(idx) ((Mem5Link *)(&mem5.zPool[(idx)*mem5.szAtom]))
+#define MEM5LINK(idx) ((Mem5Link *)(&mem5.aPool[(idx)*mem5.szAtom]))
 
 /*
 ** Unlink the chunk at mem5.aPool[i] from list it is currently
@@ -201,7 +206,14 @@ static void memsys5Leave(void){
 static int memsys5Size(void *p){
   int iSize, i;
   assert( p!=0 );
-  i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom);
+#ifdef SQLITE_MEM5_FAILOVER
+  if( ((u8*)p)<mem5.aPool || ((u8*)p)>mem5.aEnd ){
+    u64 *p64 = (u64*)p;
+    p64--;
+    return (int)p64[0];
+  }
+#endif
+  i = (int)(((u8*)p - mem5.aPool)/mem5.szAtom);
   assert( i>=0 && i<mem5.nBlock );
   iSize = mem5.szAtom * (1 << (mem5.aCtrl[i]&CTRL_LOGSIZE));
   return iSize;
@@ -277,11 +289,11 @@ static void *memsys5MallocUnsafe(int nByte){
 #ifdef SQLITE_DEBUG
   /* Make sure the allocated memory does not assume that it is set to zero
   ** or retains a value from a previous allocation */
-  memset(&mem5.zPool[i*mem5.szAtom], 0xAA, iFullSz);
+  memset(&mem5.aPool[i*mem5.szAtom], 0xAA, iFullSz);
 #endif
 
   /* Return a pointer to the allocated memory. */
-  return (void*)&mem5.zPool[i*mem5.szAtom];
+  return (void*)&mem5.aPool[i*mem5.szAtom];
 }
 
 /*
@@ -292,13 +304,13 @@ static void memsys5FreeUnsafe(void *pOld){
   int iBlock;
 
   /* Set iBlock to the index of the block pointed to by pOld in 
-  ** the array of mem5.szAtom byte blocks pointed to by mem5.zPool.
+  ** the array of mem5.szAtom byte blocks pointed to by mem5.aPool.
   */
-  iBlock = (int)(((u8 *)pOld-mem5.zPool)/mem5.szAtom);
+  iBlock = (int)(((u8 *)pOld-mem5.aPool)/mem5.szAtom);
 
   /* Check that the pointer pOld points to a valid, non-free block. */
   assert( iBlock>=0 && iBlock<mem5.nBlock );
-  assert( ((u8 *)pOld-mem5.zPool)%mem5.szAtom==0 );
+  assert( ((u8 *)pOld-mem5.aPool)%mem5.szAtom==0 );
   assert( (mem5.aCtrl[iBlock] & CTRL_FREE)==0 );
 
   iLogsize = mem5.aCtrl[iBlock] & CTRL_LOGSIZE;
@@ -344,7 +356,7 @@ static void memsys5FreeUnsafe(void *pOld){
 #ifdef SQLITE_DEBUG
   /* Overwrite freed memory with the 0x55 bit pattern to verify that it is
   ** not used after being freed */
-  memset(&mem5.zPool[iBlock*mem5.szAtom], 0x55, size);
+  memset(&mem5.aPool[iBlock*mem5.szAtom], 0x55, size);
 #endif
 
   memsys5Link(iBlock, iLogsize);
@@ -358,6 +370,20 @@ static void *memsys5Malloc(int nBytes){
   if( nBytes>0 ){
     memsys5Enter();
     p = memsys5MallocUnsafe(nBytes);
+#ifdef SQLITE_MEM5_FAILOVER
+    if( p==0
+     && (p = malloc(nBytes + sizeof(u64)))!=0
+    ){
+      u64 *p64 = (u64*)p;
+      p64[0] = nBytes;
+      p = (i64*)&p64[1];
+      mem5.nFailover++;
+      sqlite3StatusHighwater(SQLITE_STATUS_FAILOVER_SIZE, nBytes);
+      sqlite3StatusUp(SQLITE_STATUS_FAILOVER_USED, nBytes);
+      sqlite3StatusUp(SQLITE_STATUS_FAILOVER_COUNT, 1);
+      assert( ((u8*)p)<mem5.aPool || ((u8*)p)>mem5.aEnd );
+    }
+#endif
     memsys5Leave();
   }
   return (void*)p; 
@@ -372,7 +398,22 @@ static void *memsys5Malloc(int nBytes){
 static void memsys5Free(void *pPrior){
   assert( pPrior!=0 );
   memsys5Enter();
-  memsys5FreeUnsafe(pPrior);
+#ifdef SQLITE_MEM5_FAILOVER
+  if( mem5.nFailover
+   && pPrior!=0
+   && (((u8*)pPrior)>mem5.aEnd || ((u8*)pPrior)<mem5.aPool)
+  ){
+    u64 *p64 = (u64*)pPrior;
+    p64--;
+    sqlite3StatusDown(SQLITE_STATUS_FAILOVER_USED, (int)p64[0]);
+    sqlite3StatusDown(SQLITE_STATUS_FAILOVER_COUNT, 1);
+    free(p64);
+    mem5.nFailover--;
+  }else
+#endif
+  {
+    memsys5FreeUnsafe(pPrior);
+  }
   memsys5Leave();  
 }
 
@@ -459,7 +500,7 @@ static int memsys5Log(int iValue){
 static int memsys5Init(void *NotUsed){
   int ii;            /* Loop counter */
   int nByte;         /* Number of bytes of memory available to this allocator */
-  u8 *zByte;         /* Memory usable by this allocator */
+  u8 *aByte;         /* Memory usable by this allocator */
   int nMinLog;       /* Log base 2 of minimum allocation size in bytes */
   int iOffset;       /* An offset into mem5.aCtrl[] */
 
@@ -474,8 +515,8 @@ static int memsys5Init(void *NotUsed){
   assert( (sizeof(Mem5Link)&(sizeof(Mem5Link)-1))==0 );
 
   nByte = sqlite3GlobalConfig.nHeap;
-  zByte = (u8*)sqlite3GlobalConfig.pHeap;
-  assert( zByte!=0 );  /* sqlite3_config() does not allow otherwise */
+  aByte = (u8*)sqlite3GlobalConfig.pHeap;
+  assert( aByte!=0 );  /* sqlite3_config() does not allow otherwise */
 
   /* boundaries on sqlite3GlobalConfig.mnReq are enforced in sqlite3_config() */
   nMinLog = memsys5Log(sqlite3GlobalConfig.mnReq);
@@ -485,8 +526,9 @@ static int memsys5Init(void *NotUsed){
   }
 
   mem5.nBlock = (nByte / (mem5.szAtom+sizeof(u8)));
-  mem5.zPool = zByte;
-  mem5.aCtrl = (u8 *)&mem5.zPool[mem5.nBlock*mem5.szAtom];
+  mem5.aPool = aByte;
+  mem5.aEnd = aByte + nByte - 1;
+  mem5.aCtrl = &mem5.aPool[mem5.nBlock*mem5.szAtom];
 
   for(ii=0; ii<=LOGMAX; ii++){
     mem5.aiFreelist[ii] = -1;
diff --git a/src/shell.c.in b/src/shell.c.in
index fcc9316b00..3902a17bef 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -3313,6 +3313,14 @@ static int display_stats(
      "%lld (max %lld) bytes", SQLITE_STATUS_MEMORY_USED, bReset);
   displayStatLine(out, "Number of Outstanding Allocations:",
      "%lld (max %lld)", SQLITE_STATUS_MALLOC_COUNT, bReset);
+#ifdef SQLITE_MEM5_FAILOVER
+  displayStatLine(out, "Failover Memory Used:",
+     "%lld (max %lld) bytes", SQLITE_STATUS_FAILOVER_USED, bReset);
+  displayStatLine(out, "Failover Allocations:",
+     "%lld (max %lld)", SQLITE_STATUS_FAILOVER_COUNT, bReset);
+  displayStatLine(out, "Largest Failover Allocation:",
+     "%lld", SQLITE_STATUS_FAILOVER_SIZE, bReset);
+#endif
   if( pArg->shellFlgs & SHFLG_Pagecache ){
     displayStatLine(out, "Number of Pcache Pages Used:",
        "%lld (max %lld) pages", SQLITE_STATUS_PAGECACHE_USED, bReset);
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 43cc4962e9..4868663413 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -8798,6 +8798,31 @@ int sqlite3_status64(
 ** <dd>The *pHighwater parameter records the deepest parser stack.
 ** The *pCurrent value is undefined.  The *pHighwater value is only
 ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].</dd>)^
+**
+** [[SQLITE_STATUS_FAILOVER_COUNT]] <dt>SQLITE_STATUS_FAILOVER_COUNT</dt>
+** <dd>
+** If [memsys5] is enabled and SQLite is compiled with extra option
+** -DSQLITE_MEM5_FAILOVER, then memsys5 will fail over to the system
+** memory allocator if it is unable to complete an allocation.  This
+** status value reports the number of times that happened.
+**
+** [[SQLITE_STATUS_FAILOVER_USED]] <dt>SQLITE_STATUS_FAILOVER_USED</dt>
+** <dd>
+** If [memsys5] is enabled and SQLite is compiled with extra option
+** -DSQLITE_MEM5_FAILOVER, then memsys5 will fail over to the system
+** memory allocator if it is unable to complete an allocation.  This
+** status value reports the number of bytes currently allocated from
+** the heap due to memsys5 overflow.
+** </dd>
+**
+** [[SQLITE_STATUS_FAILOVER_SIZE]] <dt>SQLITE_STATUS_FAILOVER_SIZE</dt>
+** <dd>
+** If [memsys5] is enabled and SQLite is compiled with extra option
+** -DSQLITE_MEM5_FAILOVER, then memsys5 will fail over to the system
+** memory allocator if it is unable to complete an allocation.  The
+** highwater mark for this status value is the maximum size of any
+** memory allocation that was allocated from the system heap in this way.
+** </dd>
 ** </dl>
 **
 ** New status parameters may be added from time to time.
@@ -8806,11 +8831,14 @@ int sqlite3_status64(
 #define SQLITE_STATUS_PAGECACHE_USED       1
 #define SQLITE_STATUS_PAGECACHE_OVERFLOW   2
 #define SQLITE_STATUS_SCRATCH_USED         3  /* NOT USED */
+#define SQLITE_STATUS_FAILOVER_COUNT       3
 #define SQLITE_STATUS_SCRATCH_OVERFLOW     4  /* NOT USED */
+#define SQLITE_STATUS_FAILOVER_USED        4
 #define SQLITE_STATUS_MALLOC_SIZE          5
 #define SQLITE_STATUS_PARSER_STACK         6
 #define SQLITE_STATUS_PAGECACHE_SIZE       7
 #define SQLITE_STATUS_SCRATCH_SIZE         8  /* NOT USED */
+#define SQLITE_STATUS_FAILOVER_SIZE        8
 #define SQLITE_STATUS_MALLOC_COUNT         9
 
 /*
diff --git a/src/status.c b/src/status.c
index a462c94293..5d9feb323c 100644
--- a/src/status.c
+++ b/src/status.c
@@ -121,6 +121,7 @@ void sqlite3StatusHighwater(int op, int X){
   assert( sqlite3_mutex_held(statMutex[op] ? sqlite3Pcache1Mutex()
                                            : sqlite3MallocMutex()) );
   assert( op==SQLITE_STATUS_MALLOC_SIZE
+          || op==SQLITE_STATUS_FAILOVER_SIZE
           || op==SQLITE_STATUS_PAGECACHE_SIZE
           || op==SQLITE_STATUS_PARSER_STACK );
   if( newValue>wsdStat.mxValue[op] ){