From: drh <drh@noemail.net>
Date: Thu, 7 Nov 2019 19:59:14 +0000 (+0000)
Subject: Experimental branch to provide SQLITE_LIMIT_HEAP_K to limit the amount of
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=34483dec7da4395cbec641faabe38e4ad5806479;p=thirdparty%2Fsqlite.git

Experimental branch to provide SQLITE_LIMIT_HEAP_K to limit the amount of
allocated memory used by each database connection, individually.

FossilOrigin-Name: 4ab9dbb9394ceaa014a55394a98d74008036ef0cdbc28e626024ada3d2079e39
---

diff --git a/manifest b/manifest
index 465b28edbb..a60a9c8389 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sthe\sxferCompatibleIndex()\sfunction\sso\sthat\sit\srecognizes\sthat\sa\sPRIMARY\sKEY\nindex\sfor\sa\sWITHOUT\sROWID\stable\sis\sdifferent\sfrom\sa\sUNIQUE\sconstraint\sindex\non\sthe\sprimary\skey.\s\sTicket\s[302027baf1374498]
-D 2019-11-07T14:51:24.956
+C Experimental\sbranch\sto\sprovide\sSQLITE_LIMIT_HEAP_K\sto\slimit\sthe\samount\sof\nallocated\smemory\sused\sby\seach\sdatabase\sconnection,\sindividually.
+D 2019-11-07T19:59:14.570
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -490,8 +490,8 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 8701d80b3cecd47f0375483c1ef35dfcdc777f3c67e2d3581dd7841b980910a3
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 4ddc65ae13c0d93db0ceedc8b14a28c8c260513448b0eb8c5a2ac375e3b6a85d
-F src/main.c 20c6325c581123b8771ecb938f60d7e31c8152be01b4bf8e5eefbb74671ea86f
-F src/malloc.c b7640bcf8992cf2e98447da0d27e372abdc4621a6760418bad6d1470f041ada9
+F src/main.c 003804487422633b99b9db051f686852470c796b42384883179ccb8593018e5f
+F src/malloc.c 70aae7cc8accb049dcc03112ed8badd78d0fff93ed14574f62997cc899edc153
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -527,16 +527,16 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c cf2391c93d425455388389e7a47674b9da107d2ed69ebf49979044d70dbeb045
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
 F src/select.c 3395765ea3749341deb4c25e8339c3d626a8ac641a52c216e9632e48e620ba68
-F src/shell.c.in c06961f202f3cc50b819744b7331d3caea3d79d958402f4900b0668248557468
-F src/sqlite.h.in 5ba20664cede7f4e6861541fad1f17bac50f7bf576b40a8784c54f9126a9edd4
+F src/shell.c.in 5e402d7f77113bea90f9cf001d7d640d3cc4d4d08eafb16e171ced938ee05626
+F src/sqlite.h.in f0218cb514b4c446b3330b68073b1df80ee8515b228272a2c4f187c3dc09fb4c
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h cef696ce3293242c67b2339763608427bf72ee66f1f3a05389ac2a7b46001c31
-F src/sqliteInt.h a9afee1875df1784bf2fb5d02aece62a6a93dfd082b43bb4d5d2b5295a0042ae
-F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
+F src/sqliteInt.h 6b73ae93011f792d9186a73939aa01d2e257b6791a5c0c18d763d053907d0a23
+F src/sqliteLimit.h b1f05b390de2ae884cf17f1fe8381237b40d020091d50763fccba07cecb8d193
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 50c93be3e1c03b4e6cf6756e5197afcfe7f5cd0497d83a7ac317cde09e19b290
-F src/test1.c 95c7af53c4761c57b5dbfeb5da154de31a7dae0246da8976b0bdaee1062f0826
+F src/test1.c d007f593298b98089155604d0c620f84a8cd803c19267a01d97ce48f6b88345a
 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5
 F src/test3.c 61798bb0d38b915067a8c8e03f5a534b431181f802659a6616f9b4ff7d872644
 F src/test4.c 405834f6a93ec395cc4c9bb8ecebf7c3d8079e7ca16ae65e82d01afd229694bb
@@ -1849,7 +1849,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 104a2beb57037f9353ffa77096aae0eb64e4682e667f31623bfd3d064dd8d881
-R c8b53b705fa0400a6763ef7952da070a
+P 34f64f11ca481996b1ff1686bdc5e278946635b6856cd1fe2355fce8e298e0f8
+R d66fe9db5595aad5bbfd0a97b477cd6e
+T *branch * limit_heap_k
+T *sym-limit_heap_k *
+T -sym-trunk *
 U drh
-Z 1a2695a1ff301ea728b819cbcc928cab
+Z dfa7ee02c85a708dc5c324d8cd8d8443
diff --git a/manifest.uuid b/manifest.uuid
index 59ba873211..b87e08f6d5 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-34f64f11ca481996b1ff1686bdc5e278946635b6856cd1fe2355fce8e298e0f8
\ No newline at end of file
+4ab9dbb9394ceaa014a55394a98d74008036ef0cdbc28e626024ada3d2079e39
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index 1172188821..b2a54b2de3 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2595,6 +2595,7 @@ static const int aHardLimit[] = {
   SQLITE_MAX_VARIABLE_NUMBER,      /* IMP: R-38091-32352 */
   SQLITE_MAX_TRIGGER_DEPTH,
   SQLITE_MAX_WORKER_THREADS,
+  SQLITE_MAX_HEAP_K,
 };
 
 /*
@@ -2673,7 +2674,8 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){
   assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER);
   assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH );
   assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS );
-  assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) );
+  assert( aHardLimit[SQLITE_LIMIT_HEAP_K]==SQLITE_MAX_HEAP_K );
+  assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-2) );
 
 
   if( limitId<0 || limitId>=SQLITE_N_LIMIT ){
@@ -3068,6 +3070,7 @@ static int openDatabase(
   db->aDb = db->aDbStatic;
   db->lookaside.bDisable = 1;
   db->lookaside.sz = 0;
+  db->nMemUsed = 0;
 
   assert( sizeof(db->aLimit)==sizeof(aHardLimit) );
   memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit));
diff --git a/src/malloc.c b/src/malloc.c
index 8a2b234853..17e1fcf8b2 100644
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -333,6 +333,15 @@ static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){
   *db->pnBytesFreed += sqlite3DbMallocSize(db,p);
 }
 
+static SQLITE_NOINLINE void sqlite3DbFreeTail(sqlite3 *db, void *p){
+  db->nMemUsed -= sqlite3MallocSize(p);
+  assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+  assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
+  assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) );
+  sqlite3MemdebugSetType(p, MEMTYPE_HEAP);
+  sqlite3_free(p);
+}
+
 /*
 ** Free memory that might be associated with a particular database
 ** connection.  Calling sqlite3DbFree(D,X) for X==0 is a harmless no-op.
@@ -356,6 +365,8 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){
       db->lookaside.pFree = pBuf;
       return;
     }
+    sqlite3DbFreeTail(db, p);
+    return;
   }
   assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
   assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
@@ -469,9 +480,18 @@ static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){
   void *p;
   assert( db!=0 );
   p = sqlite3Malloc(n);
-  if( !p ) sqlite3OomFault(db);
-  sqlite3MemdebugSetType(p, 
+  if( p==0 ){
+    sqlite3OomFault(db);
+  }else{
+    sqlite3MemdebugSetType(p, 
          (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP);
+    db->nMemUsed += sqlite3MallocSize(p);
+    if( db->nMemUsed > 1024*(i64)db->aLimit[SQLITE_LIMIT_HEAP_K] ){
+      sqlite3DbFreeNN(db, p);
+      p = 0;
+      sqlite3OomFault(db);
+    }
+  }
   return p;
 }
 
diff --git a/src/shell.c.in b/src/shell.c.in
index 105cab6e5e..99b90bbcfd 100644
--- a/src/shell.c.in
+++ b/src/shell.c.in
@@ -7895,6 +7895,7 @@ static int do_meta_command(char *zLine, ShellState *p){
       { "variable_number",       SQLITE_LIMIT_VARIABLE_NUMBER           },
       { "trigger_depth",         SQLITE_LIMIT_TRIGGER_DEPTH             },
       { "worker_threads",        SQLITE_LIMIT_WORKER_THREADS            },
+      { "heap_k",                SQLITE_LIMIT_HEAP_K                    },
     };
     int i, n2;
     open_db(p, 0);
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 5c3827e2d4..1c91bde550 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -3705,6 +3705,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** [[SQLITE_LIMIT_WORKER_THREADS]] ^(<dt>SQLITE_LIMIT_WORKER_THREADS</dt>
 ** <dd>The maximum number of auxiliary worker threads that a single
 ** [prepared statement] may start.</dd>)^
+**
+** [[SQLITE_LIMIT_HEAP_K]] ^(<dt>SQLITE_LIMIT_HEAP_K</dt>
+** <dd>The maximum number of kibibytes (one kibibyte is 1024 bytes) that
+** an individual [database connection] is allowed to allocate.)^
 ** </dl>
 */
 #define SQLITE_LIMIT_LENGTH                    0
@@ -3719,6 +3723,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 #define SQLITE_LIMIT_VARIABLE_NUMBER           9
 #define SQLITE_LIMIT_TRIGGER_DEPTH            10
 #define SQLITE_LIMIT_WORKER_THREADS           11
+#define SQLITE_LIMIT_HEAP_K                   12
 
 /*
 ** CAPI3REF: Prepare Flags
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index b12391a3ed..5e58a1db2c 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1250,7 +1250,7 @@ struct Schema {
 ** The number of different kinds of things that can be limited
 ** using the sqlite3_limit() interface.
 */
-#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1)
+#define SQLITE_N_LIMIT (SQLITE_LIMIT_HEAP_K+1)
 
 /*
 ** Lookaside malloc is a set of fixed-size buffers that can be used
@@ -1417,6 +1417,7 @@ struct sqlite3 {
   int nVdbeExec;                /* Number of nested calls to VdbeExec() */
   int nVDestroy;                /* Number of active OP_VDestroy operations */
   int nExtension;               /* Number of loaded extensions */
+  i64 nMemUsed;                 /* Memory used by this database connection */
   void **aExtension;            /* Array of shared library handles */
   int (*xTrace)(u32,void*,void*,void*);     /* Trace function */
   void *pTraceArg;                          /* Argument to the trace function */
diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h
index 28e7a41cc3..9d6c54fdc9 100644
--- a/src/sqliteLimit.h
+++ b/src/sqliteLimit.h
@@ -209,3 +209,14 @@
 #ifndef SQLITE_MAX_TRIGGER_DEPTH
 # define SQLITE_MAX_TRIGGER_DEPTH 1000
 #endif
+
+/*
+** Maximum number of kibibytes (1 kibibyte == 1024 bytes) that a single
+** database connection is allowed to allocate.  Memory allocations that
+** are percipitated by a database conection but are not directly attached
+** to a database connection (such as memory used for the page cache) are
+** not counted toward this limit.
+*/
+#ifndef SQLITE_MAX_HEAP_K
+# define SQLITE_MAX_HEAP_K 1048576
+#endif
diff --git a/src/test1.c b/src/test1.c
index b25873bbd4..3afc570875 100644
--- a/src/test1.c
+++ b/src/test1.c
@@ -6327,6 +6327,7 @@ static int SQLITE_TCLAPI test_limit(
     { "SQLITE_LIMIT_VARIABLE_NUMBER",     SQLITE_LIMIT_VARIABLE_NUMBER      },
     { "SQLITE_LIMIT_TRIGGER_DEPTH",       SQLITE_LIMIT_TRIGGER_DEPTH        },
     { "SQLITE_LIMIT_WORKER_THREADS",      SQLITE_LIMIT_WORKER_THREADS       },
+    { "SQLITE_LIMIT_HEAP_K",              SQLITE_LIMIT_HEAP_K               },
     
     /* Out of range test cases */
     { "SQLITE_LIMIT_TOOSMALL",            -1,                               },