ITS#8977 make IDL size configurable

diff --git a/doc/man/man5/slapd-mdb.5 b/doc/man/man5/slapd-mdb.5
index ccfc7e91a0dbaf2a329b432885122806cb2ec92f..e609e4e9a8eb0fb25555cfb14fb0e2e669753ab4 100644 (file)
--- a/doc/man/man5/slapd-mdb.5
+++ b/doc/man/man5/slapd-mdb.5
@@ -20,6 +20,18 @@ supports subtree renames. It is both more space-efficient and more
 execution-efficient than the \fBbdb\fP backend, while being overall
 much simpler to manage.
 .SH CONFIGURATION
+These
+.B slapd.conf
+options apply to the \fBmdb\fP backend.
+That is, they must follow a "backend mdb" line and
+come before any subsequent "backend" or "database" lines.
+.TP
+.BI idlexp \ <exp>
+Specify a power of 2 for the maximum size of an index slot.
+The default is 16, yielding a maximum slot size of 2^16 or 65536.
+Once set, this option applies to every \fBmdb\fP database instance.
+.LP
+
 These
 .B slapd.conf
 options apply to the \fBmdb\fP backend database.

diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c
index 42e8f0d915f6cc6d04a7cc33ee72f16ffa5fd120..19d1942477d3783c2f21d02a3d85162a5fa4acff 100644 (file)
--- a/servers/slapd/back-mdb/config.c
+++ b/servers/slapd/back-mdb/config.c
@@ -22,13 +22,16 @@
 #include <ac/errno.h>
 
 #include "back-mdb.h"
+#include "idl.h"
 
 #include "config.h"
 
 #include "lutil.h"
 #include "ldap_rq.h"
 
+
 static ConfigDriver mdb_cf_gen;
+static ConfigDriver mdb_bk_cfg;
 
 enum {
        MDB_CHKPT = 1,
@@ -41,9 +44,15 @@ enum {
        MDB_MODE,
        MDB_SSTACK,
        MDB_MULTIVAL,
+       MDB_IDLEXP,
 };
 
 static ConfigTable mdbcfg[] = {
+       { "idlexp", "log", 2, 2, 0, ARG_UINT|ARG_MAGIC|MDB_IDLEXP,
+               mdb_bk_cfg, "( OLcfgBkAt:12.1 NAME 'olcBkMdbIdlExp' "
+                       "DESC 'Power of 2 used to set IDL size' "
+                       "EQUALITY integerMatch "
+                       "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL },
        { "directory", "dir", 2, 2, 0, ARG_STRING|ARG_MAGIC|MDB_DIRECTORY,
                mdb_cf_gen, "( OLcfgDbAt:0.1 NAME 'olcDbDirectory' "
                        "DESC 'Directory for database content' "
@@ -112,17 +121,24 @@ static ConfigTable mdbcfg[] = {
 };
 
 static ConfigOCs mdbocs[] = {
+       {
+               "( OLcfgBkOc:12.1 "
+               "NAME 'olcMdbBkConfig' "
+               "DESC 'MDB backend configuration' "
+               "SUP olcBackendConfig "
+               "MAY olcBkMdbIdlExp )",
+                       Cft_Backend, mdbcfg },
        {
                "( OLcfgDbOc:12.1 "
                "NAME 'olcMdbConfig' "
-               "DESC 'MDB backend configuration' "
+               "DESC 'MDB database configuration' "
                "SUP olcDatabaseConfig "
                "MUST olcDbDirectory "
                "MAY ( olcDbCheckpoint $ olcDbEnvFlags $ "
                "olcDbNoSync $ olcDbIndex $ olcDbMaxReaders $ olcDbMaxSize $ "
                "olcDbMode $ olcDbSearchStack $ olcDbMaxEntrySize $ olcDbRtxnSize $ "
                "olcDbMultival ) )",
-                       Cft_Database, mdbcfg },
+                       Cft_Database, mdbcfg+1 },
        { NULL, 0, NULL }
 };
 
@@ -135,6 +151,29 @@ static slap_verbmasks mdb_envflags[] = {
        { BER_BVNULL, 0 }
 };
 
+static int
+mdb_bk_cfg( ConfigArgs *c )
+{
+       int rc = 0;
+       if ( c->op == SLAP_CONFIG_EMIT ) {
+               if ( MDB_idl_logn != MDB_IDL_LOGN )
+                       c->value_int = MDB_idl_logn;
+               else
+                       rc = 1;
+       } else if ( c->op == LDAP_MOD_DELETE ) {
+               MDB_idl_logn = 0;
+               mdb_idl_reset();
+       } else {
+               if ( c->value_int >= MDB_IDL_LOGN && c->value_int < sizeof(int) * CHAR_BIT ) {
+                       MDB_idl_logn = c->value_int;
+                       mdb_idl_reset();
+               } else {
+                       rc = 1;
+               }
+       }
+       return rc;
+}
+
 /* perform periodic syncs */
 static void *
 mdb_checkpoint( void *ctx, void *arg )

diff --git a/servers/slapd/back-mdb/dn2id.c b/servers/slapd/back-mdb/dn2id.c
index 1ac804f227bafe9390a6bf3ab2e81e9b0c0d8fea..7020bc6ab2f0b21392a9dc703e849db45519a91d 100644 (file)
--- a/servers/slapd/back-mdb/dn2id.c
+++ b/servers/slapd/back-mdb/dn2id.c
@@ -664,7 +664,7 @@ mdb_idscope(
                        ptr += data.mv_size - sizeof(ID);
                        memcpy( &id, ptr, sizeof(ID) );
                        if ( id == base ) {
-                               if ( res[0] >= MDB_IDL_DB_SIZE-1 ) {
+                               if ( res[0] >= MDB_idl_db_max ) {
                                        /* too many aliases in scope. Fallback to range */
                                        MDB_IDL_RANGE( res, MDB_IDL_FIRST( ids ), MDB_IDL_LAST( ids ));
                                        goto leave;

diff --git a/servers/slapd/back-mdb/filterindex.c b/servers/slapd/back-mdb/filterindex.c
index e3895d2767ce110e2ed02cc79264c14c7bfaf265..0183ab6955eb024d6251f434ef7eab01c7f2fb2f 100644 (file)
--- a/servers/slapd/back-mdb/filterindex.c
+++ b/servers/slapd/back-mdb/filterindex.c
@@ -270,7 +270,7 @@ comp_list_candidates(
                        continue;
                }
                MDB_IDL_ZERO( save );
-               rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_IDL_UM_SIZE );
+               rc = comp_candidates( op, rtxn, mra, f, save, tmp, save+MDB_idl_um_size );
 
                if ( rc != 0 ) {
                        if ( ftype == LDAP_COMP_FILTER_AND ) {
@@ -580,7 +580,7 @@ list_candidates(
                }
                MDB_IDL_ZERO( save );
                rc = mdb_filter_candidates( op, rtxn, f, save, tmp,
-                       save+MDB_IDL_UM_SIZE );
+                       save+MDB_idl_um_size );
 
                if ( rc != 0 ) {
                        if ( ftype == LDAP_FILTER_AND ) {

diff --git a/servers/slapd/back-mdb/idl.c b/servers/slapd/back-mdb/idl.c
index c605b30dd068047aa976083aa110cd65465e3ebc..637eaa5b9fdff30f987b7d8aef83a61e59af0ddc 100644 (file)
--- a/servers/slapd/back-mdb/idl.c
+++ b/servers/slapd/back-mdb/idl.c
@@ -22,6 +22,12 @@
 #include "back-mdb.h"
 #include "idl.h"
 
+unsigned int MDB_idl_logn = MDB_IDL_LOGN;
+unsigned int MDB_idl_db_size = 1 << MDB_IDL_LOGN;
+unsigned int MDB_idl_um_size = 1 << (MDB_IDL_LOGN+1);
+unsigned int MDB_idl_db_max = (1 << MDB_IDL_LOGN) - 1;
+unsigned int MDB_idl_um_max = (1 << (MDB_IDL_LOGN+1) - 1);
+
 #define IDL_MAX(x,y)   ( (x) > (y) ? (x) : (y) )
 #define IDL_MIN(x,y)   ( (x) < (y) ? (x) : (y) )
 #define IDL_CMP(x,y)   ( (x) < (y) ? -1 : (x) > (y) )
@@ -67,6 +73,17 @@ static void idl_dump( ID *ids )
 #endif /* IDL_DEBUG > 1 */
 #endif /* IDL_DEBUG > 0 */
 
+void mdb_idl_reset()
+{
+       if ( !MDB_idl_logn )
+               MDB_idl_logn = MDB_IDL_LOGN;
+
+       MDB_idl_db_size = 1 << MDB_idl_logn;
+       MDB_idl_um_size = 1 << (MDB_idl_logn+1);
+       MDB_idl_db_max = MDB_idl_db_size - 1;
+       MDB_idl_um_max = MDB_idl_um_size - 1;
+}
+
 unsigned mdb_idl_search( ID *ids, ID id )
 {
 #define IDL_BINARY_SEARCH 1
@@ -160,7 +177,7 @@ int mdb_idl_insert( ID *ids, ID id )
                return -1;
        }
 
-       if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
+       if ( ++ids[0] >= MDB_idl_db_max ) {
                if( id < ids[1] ) {
                        ids[1] = id;
                        ids[2] = ids[ids[0]-1];
@@ -447,7 +464,7 @@ mdb_idl_insert_keys(
                                err = "c_count";
                                goto fail;
                        }
-                       if ( count >= MDB_IDL_DB_MAX ) {
+                       if ( count >= MDB_idl_db_max ) {
                        /* No room, convert to a range */
                                lo = *i;
                                rc = mdb_cursor_get( cursor, &key, &data, MDB_LAST_DUP );
@@ -784,7 +801,7 @@ over:               ida = IDL_MIN( MDB_IDL_FIRST(a), MDB_IDL_FIRST(b) );
        /* The distinct elements of a are cat'd to b */
        while( ida != NOID || idb != NOID ) {
                if ( ida < idb ) {
-                       if( ++cursorc > MDB_IDL_UM_MAX ) {
+                       if( ++cursorc > MDB_idl_um_max ) {
                                goto over;
                        }
                        b[cursorc] = ida;
@@ -949,7 +966,7 @@ int mdb_idl_append_one( ID *ids, ID id )
                }
        }
        ids[0]++;
-       if ( ids[0] >= MDB_IDL_UM_MAX ) {
+       if ( ids[0] >= MDB_idl_um_max ) {
                ids[0] = NOID;
                ids[2] = id;
        } else {
@@ -977,7 +994,7 @@ int mdb_idl_append( ID *a, ID *b )
        ida = MDB_IDL_LAST( a );
        idb = MDB_IDL_LAST( b );
        if ( MDB_IDL_IS_RANGE( a ) || MDB_IDL_IS_RANGE(b) ||
-               a[0] + b[0] >= MDB_IDL_UM_MAX ) {
+               a[0] + b[0] >= MDB_idl_um_max ) {
                a[2] = IDL_MAX( ida, idb );
                a[1] = IDL_MIN( a[1], b[1] );
                a[0] = NOID;
@@ -1239,7 +1256,7 @@ int mdb_id2l_insert( ID2L ids, ID2 *id )
                return -1;
        }
 
-       if ( ids[0].mid >= MDB_IDL_UM_MAX ) {
+       if ( ids[0].mid >= MDB_idl_um_max ) {
                /* too big */
                return -2;
 

diff --git a/servers/slapd/back-mdb/idl.h b/servers/slapd/back-mdb/idl.h
index d686d0c634c330fc78bfc4655fd894d75d406499..1c21b7d77c575d9139f6827d80d0577698c98fab 100644 (file)
--- a/servers/slapd/back-mdb/idl.h
+++ b/servers/slapd/back-mdb/idl.h
@@ -21,13 +21,11 @@
  *   limiting factors: sizeof(ID), thread stack size
  */
 #define        MDB_IDL_LOGN    16      /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
-#define MDB_IDL_DB_SIZE                (1<<MDB_IDL_LOGN)
-#define MDB_IDL_UM_SIZE                (1<<(MDB_IDL_LOGN+1))
-#define MDB_IDL_UM_SIZEOF      (MDB_IDL_UM_SIZE * sizeof(ID))
-
-#define MDB_IDL_DB_MAX         (MDB_IDL_DB_SIZE-1)
-
-#define MDB_IDL_UM_MAX         (MDB_IDL_UM_SIZE-1)
+extern unsigned int MDB_idl_logn;
+extern unsigned int MDB_idl_db_size;
+extern unsigned int MDB_idl_um_size;
+extern unsigned int MDB_idl_db_max;
+extern unsigned int MDB_idl_um_max;
 
 #define MDB_IDL_IS_RANGE(ids)  ((ids)[0] == NOID)
 #define MDB_IDL_RANGE_SIZE             (3)
@@ -97,6 +95,10 @@ typedef struct IdScopes {
 } IdScopes;
 
 LDAP_BEGIN_DECL
+       /** Reset IDL params after changing logn */
+void mdb_idl_reset();
+
+
        /** Search for an ID in an ID2L.
         * @param[in] ids       The ID2L to search.
         * @param[in] id        The ID to search for.

diff --git a/servers/slapd/back-mdb/init.c b/servers/slapd/back-mdb/init.c
index b794569953edc7fcf85d6d5a9aadc471bbcf7c9b..3560f1b91131630dfd8ec08142476b29424bad26 100644 (file)
--- a/servers/slapd/back-mdb/init.c
+++ b/servers/slapd/back-mdb/init.c
@@ -68,7 +68,7 @@ mdb_db_init( BackendDB *be, ConfigReply *cr )
        mdb->mi_multi_lo = UINT_MAX;
 
        be->be_private = mdb;
-       be->be_cf_ocs = be->bd_info->bi_cf_ocs;
+       be->be_cf_ocs = be->bd_info->bi_cf_ocs+1;
 
 #ifndef MDB_MULTIPLE_SUFFIXES
        SLAP_DBFLAGS( be ) |= SLAP_DBFLAG_ONE_SUFFIX;

diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c
index fcda47f40d8ac6ef51a2a803ade1e01572203415..1c1211c62935107497ea98a91771db8ef9bfd71f 100644 (file)
--- a/servers/slapd/back-mdb/search.c
+++ b/servers/slapd/back-mdb/search.c
@@ -142,11 +142,11 @@ static int search_aliases(
        Filter  af;
 
        aliases = stack;        /* IDL of all aliases in the database */
-       curscop = aliases + MDB_IDL_DB_SIZE;    /* Aliases in the current scope */
-       visited = curscop + MDB_IDL_DB_SIZE;    /* IDs we've seen in this search */
-       newsubs = visited + MDB_IDL_DB_SIZE;    /* New subtrees we've added */
-       oldsubs = newsubs + MDB_IDL_DB_SIZE;    /* Subtrees added previously */
-       tmp = oldsubs + MDB_IDL_DB_SIZE;        /* Scratch space for deref_base() */
+       curscop = aliases + MDB_idl_db_size;    /* Aliases in the current scope */
+       visited = curscop + MDB_idl_db_size;    /* IDs we've seen in this search */
+       newsubs = visited + MDB_idl_db_size;    /* New subtrees we've added */
+       oldsubs = newsubs + MDB_idl_db_size;    /* Subtrees added previously */
+       tmp = oldsubs + MDB_idl_db_size;        /* Scratch space for deref_base() */
 
        af.f_choice = LDAP_FILTER_EQUALITY;
        af.f_ava = &aa_alias;
@@ -297,7 +297,7 @@ static ID2 *scope_chunk_get( Operation *op )
        ldap_pvt_thread_pool_getkey( op->o_threadctx, (void *)scope_chunk_get,
                        (void *)&ret, NULL );
        if ( !ret ) {
-               ret = ch_malloc( MDB_IDL_UM_SIZE * sizeof( ID2 ));
+               ret = ch_malloc( MDB_idl_um_size * sizeof( ID2 ));
        } else {
                void *r2 = ret[0].mval.mv_data;
                ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)scope_chunk_get,
@@ -416,8 +416,8 @@ mdb_search( Operation *op, SlapReply *rs )
        struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private;
        ID              id, cursor, nsubs, ncand, cscope;
        ID              lastid = NOID;
-       ID              candidates[MDB_IDL_UM_SIZE];
-       ID              iscopes[MDB_IDL_DB_SIZE];
+       ID              candidates[MDB_idl_um_size];
+       ID              iscopes[MDB_idl_db_size];
        ID2             *scopes;
        void    *stack;
        Entry           *e = NULL, *base = NULL;
@@ -1289,7 +1289,7 @@ static void *search_stack( Operation *op )
        }
 
        if ( !ret ) {
-               ret = ch_malloc( mdb->mi_search_stack_depth * MDB_IDL_UM_SIZE
+               ret = ch_malloc( mdb->mi_search_stack_depth * MDB_idl_um_size
                        * sizeof( ID ) );
                if ( op->o_threadctx ) {
                        ldap_pvt_thread_pool_setkey( op->o_threadctx, (void *)search_stack,
@@ -1366,7 +1366,7 @@ static int search_candidates(
 
        /* Allocate IDL stack, plus 1 more for former tmp */
        if ( depth+1 > mdb->mi_search_stack_depth ) {
-               stack = ch_malloc( (depth + 1) * MDB_IDL_UM_SIZE * sizeof( ID ) );
+               stack = ch_malloc( (depth + 1) * MDB_idl_um_size * sizeof( ID ) );
        }
 
        if( op->ors_deref & LDAP_DEREF_SEARCHING ) {
@@ -1377,7 +1377,7 @@ static int search_candidates(
 
        if ( rc == LDAP_SUCCESS ) {
                rc = mdb_filter_candidates( op, isc->mt, f, ids,
-                       stack, stack+MDB_IDL_UM_SIZE );
+                       stack, stack+MDB_idl_um_size );
        }
 
        if ( depth+1 > mdb->mi_search_stack_depth ) {

diff --git a/servers/slapd/back-mdb/tools.c b/servers/slapd/back-mdb/tools.c
index cce8115ff3716c1488a80e6acde3643311bc8360..fe4e2210e5648dcb4687ba2acda47920cf98aa26 100644 (file)
--- a/servers/slapd/back-mdb/tools.c
+++ b/servers/slapd/back-mdb/tools.c
@@ -1268,14 +1268,14 @@ mdb_tool_idl_flush_one( MDB_cursor *mc, AttrIxInfo *ai, mdb_tool_idl_cache *ic )
        ID id, nid;
 
        /* Freshly allocated, ignore it */
-       if ( !ic->head && ic->count <= MDB_IDL_DB_SIZE ) {
+       if ( !ic->head && ic->count <= MDB_idl_db_size ) {
                return 0;
        }
 
        key.mv_data = ic->kstr.bv_val;
        key.mv_size = ic->kstr.bv_len;
 
-       if ( ic->count > MDB_IDL_DB_SIZE ) {
+       if ( ic->count > MDB_idl_db_size ) {
                while ( ic->flags & WAS_FOUND ) {
                        rc = mdb_cursor_get( mc, &key, data, MDB_SET );
                        if ( rc ) {
@@ -1438,7 +1438,7 @@ int mdb_tool_idl_add(
                        ic->flags |= WAS_FOUND;
                        nid = *(ID *)data.mv_data;
                        if ( nid == 0 ) {
-                               ic->count = MDB_IDL_DB_SIZE+1;
+                               ic->count = MDB_idl_db_size+1;
                                ic->flags |= WAS_RANGE;
                        } else {
                                size_t count;
@@ -1451,11 +1451,11 @@ int mdb_tool_idl_add(
                }
        }
        /* are we a range already? */
-       if ( ic->count > MDB_IDL_DB_SIZE ) {
+       if ( ic->count > MDB_idl_db_size ) {
                ic->last = id;
                continue;
        /* Are we at the limit, and converting to a range? */
-       } else if ( ic->count == MDB_IDL_DB_SIZE ) {
+       } else if ( ic->count == MDB_idl_db_size ) {
                if ( ic->head ) {
                        ic->tail->next = ax->ai_flist;
                        ax->ai_flist = ic->head;