Cleanup and performance enhancements for mini-lookaside.

FossilOrigin-Name: 74805668430051032ae9b256c84e252755ee03075fc08293c948675ed40ec280

diff --git a/manifest b/manifest
index 448d34a0ee626d488374274defb9990d20cc2017..6425c9f1a093178621e8afebbfa108ab516f7970 100644 (file)
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Merge\sfixes\sfrom\strunk.
-D 2019-12-13T12:14:23.880
+C Cleanup\sand\sperformance\senhancements\sfor\smini-lookaside.
+D 2019-12-13T15:48:21.644
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -491,8 +491,8 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
 F src/insert.c 276463aa7a26ee73958b9f4bcbef4eaec516f22bf6b8d7d2428f39c8dea90840
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c d74f5e7bd51f3c9d283442473eb65aef359664efd6513591c03f01881c4ae2da
-F src/main.c 94bf8b6a283914cc8fa8743b3db5350206a4be5cccfd9c84d53409ca5d8bd1c1
-F src/malloc.c 02235d039c72102d4bddf203a493c2613164487c70ec84bab679fc30daa883f0
+F src/main.c 34d73303ac76015d3c8961f16d49ab44a56cbc998f178b27ca07847db4cf9273
+F src/malloc.c d74d986f91ebd6a4a9b2995467f4c13a1fdb57ac87df5bb0a7fa3afe1db72d1d
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
 F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3
@@ -532,9 +532,9 @@ F src/shell.c.in 4a3a9e1c11847b1904f2b01d087af1c052f660902755abab457cab1756817de
 F src/sqlite.h.in 2a23e8161775253d9cf383c2c6aa559005dc787d350dcb0be67a6c4cc3bd1d19
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 72af51aa4e912e14cd495fb6e7fac65f0940db80ed950d90911aff292cc47ce2
-F src/sqliteInt.h a036ca4ee94ac0dea82c90b2c902dc84bce187adf75e68284e890bc0c4cd3d95
+F src/sqliteInt.h 4948c2046c921461d2021e52ac8cec5b65cd56ab0c58aa7150313abace90153c
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
-F src/status.c 0fe55b92810509eac821ded2e177e1da944a78cdcdfec7813ad1d6b36d6d667e
+F src/status.c baf38f521ac079be616dab1ff13f60c6ce0703c7f0429a65238c695197a56935
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
 F src/tclsqlite.c 8cd2600e8de23dff6cdf84d39f46ca57139b061b28f6f80b166bace17d52ab1c
 F src/test1.c c654981c1d86ebc90dd23fcc0969e6c85e28112f0acc2e2224a97a2a33e7c42f
@@ -1852,7 +1852,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bda92d92bf69a73bc6dd498f92578ad6e81111a3f97bc8724d1720e7283a83b6 c1014e80b26131200a115beb86929a8f0ded2dd65b075e47373346c0f170576a
-R b2be5daabe2de4420861c8338271e7de
+P 9c471195f6d3e4b00e2d0f909b306a4036352082dca5f016a8eece226e82163d
+R 40d26fbfedcb39151f8726a7fa468e8a
 U drh
-Z 5e37344191b5a16624379099587382f5
+Z 7742eaad4b11604e1540b9f51bda9209

diff --git a/manifest.uuid b/manifest.uuid
index 69d04cf3efcf35aa8e03b5575d15e919ec7e3a2e..4e1f2b833d0d562620e2cb933203d8c9d6437414 100644 (file)
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-9c471195f6d3e4b00e2d0f909b306a4036352082dca5f016a8eece226e82163d
\ No newline at end of file
+74805668430051032ae9b256c84e252755ee03075fc08293c948675ed40ec280
\ No newline at end of file

diff --git a/src/main.c b/src/main.c
index 5b70273af2b852e7085211c1ee046cfca5677394..9f541ff0d09f9efff51c1904b51a24f527948d9e 100644 (file)
--- a/src/main.c
+++ b/src/main.c
@@ -684,6 +684,8 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
 #ifndef SQLITE_OMIT_LOOKASIDE
   void *pStart;
   sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt;
+  int nBig;   /* Number of full-size slots */
+  int nSm;    /* Number smaller mini-slots */
   
   if( sqlite3LookasideUsed(db,0)>0 ){
     return SQLITE_BUSY;
@@ -708,56 +710,69 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){
     sqlite3BeginBenignMalloc();
     pStart = sqlite3Malloc( szAlloc );  /* IMP: R-61949-35727 */
     sqlite3EndBenignMalloc();
-    if( pStart ) cnt = sqlite3MallocSize(pStart)/sz;
+    if( pStart ) szAlloc = sqlite3MallocSize(pStart);
   }else{
     pStart = pBuf;
   }
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+  if( sz>=MINI_SZ*3 ){
+    nBig = szAlloc/(3*MINI_SZ+sz);
+    nSm = (szAlloc - sz*nBig)/MINI_SZ;
+  }else if( sz>=MINI_SZ*2 ){
+    nBig = szAlloc/(MINI_SZ+sz);
+    nSm = (szAlloc - sz*nBig)/MINI_SZ;
+  }else
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
+  if( sz>0 ){
+    nBig = szAlloc/sz;
+    nSm = 0;
+  }else{
+    nBig = nSm = 0;
+  }
   db->lookaside.pStart = pStart;
   db->lookaside.pInit = 0;
   db->lookaside.pFree = 0;
-#ifndef SQLITE_OMIT_MINI_LOOKASIDE
-  db->lookaside.pMiniInit = 0;
-  db->lookaside.pMiniFree = 0;
-  /* The arithmetic below causes the number of large lookaside slots to be 1/3
-  ** the number of mini slots, based on the observation that 75% of allocations
-  ** are <= MINI_SZ bytes.
-  */
-  cnt = szAlloc/(3*MINI_SZ+sz);
-#endif
   db->lookaside.sz = (u16)sz;
   db->lookaside.szTrue = (u16)sz;
   if( pStart ){
     int i;
     LookasideSlot *p;
     assert( sz > (int)sizeof(LookasideSlot*) );
-    db->lookaside.nSlot = cnt;
     p = (LookasideSlot*)pStart;
-    for(i=cnt-1; i>=0; i--){
+    for(i=0; i<nBig; i++){
       p->pNext = db->lookaside.pInit;
       db->lookaside.pInit = p;
       p = (LookasideSlot*)&((u8*)p)[sz];
     }
 #ifndef SQLITE_OMIT_MINI_LOOKASIDE
+    db->lookaside.pMiniInit = 0;
+    db->lookaside.pMiniFree = 0;
     db->lookaside.pMiddle = p;
-    /* Fill the remainder of the buffer with mini slots */
-    while(p<=(LookasideSlot*)&((u8*)pStart)[szAlloc-MINI_SZ]){
+    for(i=0; i<nSm; i++){
       p->pNext = db->lookaside.pMiniInit;
       db->lookaside.pMiniInit = p;
-      db->lookaside.nSlot++;
       p = (LookasideSlot*)&((u8*)p)[MINI_SZ];
     }
-#endif
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
+    assert( ((uptr)p)<=szAlloc + (uptr)pStart );
     db->lookaside.pEnd = p;
     db->lookaside.bDisable = 0;
     db->lookaside.bMalloced = pBuf==0 ?1:0;
+    db->lookaside.nSlot = nBig+nSm;
   }else{
     db->lookaside.pStart = db;
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+    db->lookaside.pMiniInit = 0;
+    db->lookaside.pMiniFree = 0;
+    db->lookaside.pMiddle = db;
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
     db->lookaside.pEnd = db;
     db->lookaside.bDisable = 1;
     db->lookaside.sz = 0;
     db->lookaside.bMalloced = 0;
     db->lookaside.nSlot = 0;
   }
+  assert( sqlite3LookasideUsed(db,0)==0 );
 #endif /* SQLITE_OMIT_LOOKASIDE */
   return SQLITE_OK;
 }

diff --git a/src/malloc.c b/src/malloc.c
index 26a258b347b3b05a59176b26247c6f4f2c2fc0b9..1074156eec2b4f698907d9b4f5e5c9fffa22858a 100644 (file)
--- a/src/malloc.c
+++ b/src/malloc.c
@@ -341,8 +341,8 @@ static int lookasideMallocSize(sqlite3 *db, void *p){
 }
 int sqlite3DbMallocSize(sqlite3 *db, void *p){
   assert( p!=0 );
-  if( db==0 || !isLookaside(db,p) ){
 #ifdef SQLITE_DEBUG
+  if( db==0 || !isLookaside(db,p) ){
     if( db==0 ){
       assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
       assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) );
@@ -350,12 +350,23 @@ int sqlite3DbMallocSize(sqlite3 *db, void *p){
       assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
       assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
     }
+  }
 #endif
-    return sqlite3GlobalConfig.m.xSize(p);
-  }else{
-    assert( sqlite3_mutex_held(db->mutex) );
-    return lookasideMallocSize(db, p);
+  if( db ){
+    if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+      if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
+        assert( sqlite3_mutex_held(db->mutex) );
+        return MINI_SZ;
+      }
+#endif
+      if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
+        assert( sqlite3_mutex_held(db->mutex) );
+        return db->lookaside.szTrue;
+      }
+    }
   }
+  return sqlite3GlobalConfig.m.xSize(p);
 }
 sqlite3_uint64 sqlite3_msize(void *p){
   assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) );
@@ -402,26 +413,27 @@ void sqlite3DbFreeNN(sqlite3 *db, void *p){
       measureAllocationSize(db, p);
       return;
     }
-    if( isLookaside(db, p) ){
-      LookasideSlot *pBuf = (LookasideSlot*)p;
+    if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){
 #ifndef SQLITE_OMIT_MINI_LOOKASIDE
-      if( p>=db->lookaside.pMiddle ){
-# ifdef SQLITE_DEBUG
-        /* Trash all content in the buffer being freed */
-        memset(p, 0xaa, MINI_SZ);
-# endif
+      if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){
+        LookasideSlot *pBuf = (LookasideSlot*)p;
+#ifdef SQLITE_DEBUG
+        memset(p, 0xaa, MINI_SZ);  /* Trash freed content */
+#endif
         pBuf->pNext = db->lookaside.pMiniFree;
         db->lookaside.pMiniFree = pBuf;
         return;
       }
-#endif
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
+      if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){
+        LookasideSlot *pBuf = (LookasideSlot*)p;
 #ifdef SQLITE_DEBUG
-      /* Trash all content in the buffer being freed */
-      memset(p, 0xaa, db->lookaside.szTrue);
+        memset(p, 0xaa, db->lookaside.szTrue);  /* Trash freed content */
 #endif
-      pBuf->pNext = db->lookaside.pFree;
-      db->lookaside.pFree = pBuf;
-      return;
+        pBuf->pNext = db->lookaside.pFree;
+        db->lookaside.pFree = pBuf;
+        return;
+      }
     }
   }
   assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) );
@@ -585,7 +597,7 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
     }
     return dbMallocRawFinish(db, n);
   }
-# ifndef SQLITE_OMIT_MINI_LOOKASIDE
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
   if( n<=MINI_SZ ){
     if( (pBuf = db->lookaside.pMiniFree)!=0 ){
       db->lookaside.pMiniFree = pBuf->pNext;
@@ -597,7 +609,7 @@ void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){
       return (void*)pBuf;
     }
   }
-# endif
+#endif
   if( (pBuf = db->lookaside.pFree)!=0 ){
     db->lookaside.pFree = pBuf->pNext;
     db->lookaside.anStat[0]++;
@@ -631,7 +643,16 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){
   assert( db!=0 );
   if( p==0 ) return sqlite3DbMallocRawNN(db, n);
   assert( sqlite3_mutex_held(db->mutex) );
-  if( isLookaside(db,p) && n<lookasideMallocSize(db, p) ) return p;
+  if( ((uptr)p)<(uptr)db->lookaside.pEnd ){
+#ifndef SQLITE_OMIT_MINI_LOOKASIDE
+    if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){
+      if( n<=MINI_SZ ) return p;
+    }else
+#endif
+    if( ((uptr)p)>=(uptr)db->lookaside.pStart ){
+      if( n<=db->lookaside.szTrue ) return p;
+    }
+  }
   return dbReallocFinish(db, p, n);
 }
 static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){

diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index da9a7ff811f8ff85fc7906d3c5e5c9c39fc54d8d..b18f05c85e5f6d04b6f041bb3983f8a8ebdba598 100644 (file)
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -1277,6 +1277,25 @@ struct Schema {
 ** disables lookaside without adding a new test for the bDisable flag
 ** in a performance-critical path.  sz should be set by to szTrue whenever
 ** bDisable changes back to zero.
+**
+** Lookaside buffers are initially held on the pInit list.  As they are
+** used and freed, they are added back to the pFree list.  New allocations
+** come off of pFree first, then pInit as a fallback.  This dual-list
+** allows use to compute a high-water mark - the maximum number of allocations
+** outstanding at any point in the past - by subtracting the number of
+** allocations on the pInit list from the total number of allocations.
+**
+** Enhancement on 2019-12-12:  Mini-lookaside
+** The default lookaside configuration is 100 slots of 1200 bytes each.
+** The larger slot sizes are important for performance, but they waste
+** a lot of space, as most lookaside allocations are less than 128 bytes.
+** The mini-lookaside enhancement breaks up the lookaside allocation into
+** two pools:  One of 128-byte slots and the other of the default size
+** (1200-byte) slots.   Allocations are filled from the mini-pool first,
+** failing over to the full-size pool if that does not work.  Thus more
+** lookaside slots are available while also using less memory.
+** This enhancement can be omitted by compiling with
+** SQLITE_OMIT_MINI_LOOKASIDE.
 */
 struct Lookaside {
   u32 bDisable;           /* Only operate the lookaside when zero */
@@ -1290,8 +1309,9 @@ struct Lookaside {
 #ifndef SQLITE_OMIT_MINI_LOOKASIDE
   LookasideSlot *pMiniInit; /* List of mini buffers not prediously used */
   LookasideSlot *pMiniFree; /* List of available mini buffers */
-  void *pMiddle;          /* An address between the fullsize and mini buffers */
-#endif
+  void *pMiddle;          /* First byte past end of full-size buffers and
+                          ** the first byte of mini-buffers */
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
   void *pStart;           /* First byte of available memory space */
   void *pEnd;             /* First byte past end of available space */
 };
@@ -1304,7 +1324,11 @@ struct LookasideSlot {
    db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue
 
 /* Size of the MINI lookside allocation */
-#define MINI_SZ           128
+#ifdef SQLITE_OMIT_MINI_LOOKASIDE
+#  define MINI_SZ           0
+#else
+#  define MINI_SZ         128
+#endif
 
 /*
 ** A hash table for built-in function definitions.  (Application-defined

diff --git a/src/status.c b/src/status.c
index d7ffe99ede1e9c7ab1f795128283a99f47ffc1af..977457f8e3f408bf2ddfd4868b76fe080e5a91cc 100644 (file)
--- a/src/status.c
+++ b/src/status.c
@@ -191,7 +191,7 @@ int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){
 #ifndef SQLITE_OMIT_MINI_LOOKASIDE
   nInit += countLookasideSlots(db->lookaside.pMiniInit);
   nFree += countLookasideSlots(db->lookaside.pMiniFree);
-#endif
+#endif /* SQLITE_OMIT_MINI_LOOKASIDE */
   if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit;
   return db->lookaside.nSlot - (nInit+nFree);
 }