From: Ondřej Kuzník Date: Tue, 24 Jul 2018 09:22:57 +0000 (+0100) Subject: ITS#9599 Implement tiered load balancing X-Git-Tag: OPENLDAP_REL_ENG_2_6_0~96 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ab90bfd95c2d5a5242d25d99a6bfce599d9e1207;p=thirdparty%2Fopenldap.git ITS#9599 Implement tiered load balancing --- diff --git a/doc/man/man5/lloadd.conf.5 b/doc/man/man5/lloadd.conf.5 index d6c6a836e6..a25b9044d9 100644 --- a/doc/man/man5/lloadd.conf.5 +++ b/doc/man/man5/lloadd.conf.5 @@ -644,7 +644,13 @@ only valid when using GnuTLS and Mozilla NSS. .SH BACKEND CONFIGURATION Options in this section describe how the .B lloadd -connects and authenticates to the backend servers. +connects and authenticates to the backend servers. Backends are organised in groups +.RB ( tiers ). +Backends in the first tier are tried first, if none of them are reachable, the +following tier is tried in the same way. If there is a backend in the tier that +has suitable connections, but they are busy, no further tier is consulted. This +is useful in high availability scenarios where a group of servers (e.g. the +local environment) should be contacted if possible. It is assumed all backend servers serve the same data. On startup, the configured connections are set up and those not dedicated to handle bind @@ -730,6 +736,33 @@ set on the upstream connections, overriding the operating system setting. Only some systems support the customization of this parameter, it is ignored otherwise and system-wide settings are used. +.SH TIER OPTIONS + +.TP +.B tier +.B + +Groups servers which should be considered in the same try. If a viable +connection is found even if busy, the load balancer does not proceed to the +next tier. The process of selection a connection within a tier depends on the +tier's type. + +.RE +Available types are: +.TP +.B roundrobin +Servers are tried in order and if one is selected successfully, the following +search will try from the one next on the list. +.TP +.B weighted +Backend servers accept a new option +.B weight= +which indicates how often it should be selected. If unspecified, weight +defaults to 0 and such backends have a slight chance of being selected even +when a non-zero weight backend is configured in the tier. The selection process +is along the lines of +.BR RFC2782 . + .SH BACKEND OPTIONS .TP @@ -879,6 +912,7 @@ bindconf binddn=cn=test credentials=pass +tier weighted backend-server uri=ldap://ldap1.example.com numconns=3 @@ -886,6 +920,7 @@ backend-server retry=5000 max-pending-ops=5 conn-max-pending=3 + weight=5 backend-server uri=ldap://ldap2.example.com @@ -894,6 +929,7 @@ backend-server retry=5000 max-pending-ops=5 conn-max-pending=3 + weight=10 .fi .RE .LP diff --git a/servers/lloadd/Makefile.in b/servers/lloadd/Makefile.in index 217e220295..2de57f885e 100644 --- a/servers/lloadd/Makefile.in +++ b/servers/lloadd/Makefile.in @@ -21,6 +21,7 @@ NT_OBJS = nt_svc.o ../../libraries/liblutil/slapdmsg.res SRCS = backend.c bind.c config.c connection.c client.c \ daemon.c epoch.c extended.c init.c operation.c \ + tier.c tier_roundrobin.c tier_weighted.c \ upstream.c libevent_support.c \ $(@PLAT@_SRCS) diff --git a/servers/lloadd/backend.c b/servers/lloadd/backend.c index dd7a4cbc83..07def84afc 100644 --- a/servers/lloadd/backend.c +++ b/servers/lloadd/backend.c @@ -392,44 +392,17 @@ upstream_select( int *res, char **message ) { - LloadBackend *b, *first, *next; - int rc = 0; + LloadTier *tier; + int finished = 0; - checked_lock( &backend_mutex ); - first = b = current_backend; - checked_unlock( &backend_mutex ); - - *res = LDAP_UNAVAILABLE; - - if ( !first ) { - return NULL; - } - - /* TODO: Two runs, one with trylock, then one actually locked if we don't - * find anything? */ - do { - checked_lock( &b->b_mutex ); - next = LDAP_CIRCLEQ_LOOP_NEXT( &backend, b, b_next ); - - rc = backend_select( b, op, cp, res, message ); - checked_unlock( &b->b_mutex ); - - if ( rc && *cp ) { - /* - * Round-robin step: - * Rotate the queue to put this backend at the end. The race here - * is acceptable. - */ - checked_lock( &backend_mutex ); - current_backend = next; - checked_unlock( &backend_mutex ); - return rc; + LDAP_STAILQ_FOREACH( tier, &tiers, t_next ) { + if ( (finished = tier->t_type.tier_select( + tier, op, cp, res, message )) ) { + break; } + } - b = next; - } while ( b != first ); - - return rc; + return finished; } /* @@ -726,26 +699,41 @@ backend_reset( LloadBackend *b, int gentle ) assert_locked( &b->b_mutex ); } +LloadBackend * +lload_backend_new( void ) +{ + LloadBackend *b; + + b = ch_calloc( 1, sizeof(LloadBackend) ); + + LDAP_CIRCLEQ_INIT( &b->b_conns ); + LDAP_CIRCLEQ_INIT( &b->b_bindconns ); + LDAP_CIRCLEQ_INIT( &b->b_preparing ); + LDAP_CIRCLEQ_ENTRY_INIT( b, b_next ); + + b->b_numconns = 1; + b->b_numbindconns = 1; + b->b_weight = 1; + + b->b_retry_timeout = 5000; + + ldap_pvt_thread_mutex_init( &b->b_mutex ); + + return b; +} + void lload_backend_destroy( LloadBackend *b ) { - LloadBackend *next = LDAP_CIRCLEQ_LOOP_NEXT( &backend, b, b_next ); - Debug( LDAP_DEBUG_CONNS, "lload_backend_destroy: " "destroying backend uri='%s', numconns=%d, numbindconns=%d\n", b->b_uri.bv_val, b->b_numconns, b->b_numbindconns ); checked_lock( &b->b_mutex ); + b->b_tier->t_type.tier_remove_backend( b->b_tier, b ); b->b_numconns = b->b_numbindconns = 0; backend_reset( b, 0 ); - LDAP_CIRCLEQ_REMOVE( &backend, b, b_next ); - if ( b == next ) { - current_backend = NULL; - } else { - current_backend = next; - } - #ifdef BALANCER_MODULE if ( b->b_monitor ) { BackendDB *be; @@ -760,6 +748,7 @@ lload_backend_destroy( LloadBackend *b ) assert( rc == LDAP_SUCCESS ); } #endif /* BALANCER_MODULE */ + checked_unlock( &b->b_mutex ); ldap_pvt_thread_mutex_destroy( &b->b_mutex ); @@ -774,13 +763,3 @@ lload_backend_destroy( LloadBackend *b ) ch_free( b->b_name.bv_val ); ch_free( b ); } - -void -lload_backends_destroy( void ) -{ - while ( !LDAP_CIRCLEQ_EMPTY( &backend ) ) { - LloadBackend *b = LDAP_CIRCLEQ_FIRST( &backend ); - - lload_backend_destroy( b ); - } -} diff --git a/servers/lloadd/config.c b/servers/lloadd/config.c index 16f08009d4..566625273f 100644 --- a/servers/lloadd/config.c +++ b/servers/lloadd/config.c @@ -113,6 +113,7 @@ static ConfigFile *cfn; static ConfigDriver config_fname; static ConfigDriver config_generic; +static ConfigDriver config_tier; static ConfigDriver config_backend; static ConfigDriver config_bindconf; static ConfigDriver config_restrict_oid; @@ -132,10 +133,6 @@ static ConfigDriver config_share_tls_ctx; static ConfigDriver backend_cf_gen; #endif /* BALANCER_MODULE */ -lload_b_head backend = LDAP_CIRCLEQ_HEAD_INITIALIZER(backend); -ldap_pvt_thread_mutex_t backend_mutex; -LloadBackend *current_backend = NULL; - struct slap_bindconf bindconf = {}; struct berval lloadd_identity = BER_BVNULL; @@ -182,6 +179,8 @@ enum { CFG_CLIENT_PENDING, CFG_RESTRICT_EXOP, CFG_RESTRICT_CONTROL, + CFG_TIER, + CFG_WEIGHT, CFG_LAST }; @@ -205,6 +204,17 @@ static ConfigTable config_back_cf_table[] = { &config_generic, NULL, NULL, NULL }, + { "tier", "name", 2, 2, 0, + ARG_MAGIC|ARG_STRING|CFG_TIER, + &config_tier, + "( OLcfgBkAt:13.39 " + "NAME 'olcBkLloadTierType' " + "DESC 'Tier type' " + "EQUALITY caseIgnoreMatch " + "SYNTAX OMsDirectoryString " + "SINGLE-VALUE )", + NULL, NULL + }, /* conf-file only option */ { "backend-server", "backend options", 2, 0, 0, ARG_MAGIC|CFG_BACKEND, @@ -747,6 +757,17 @@ static ConfigTable config_back_cf_table[] = { "SINGLE-VALUE )", NULL, NULL }, + { "", NULL, 2, 2, 0, + ARG_MAGIC|ARG_UINT|CFG_WEIGHT, + &backend_cf_gen, + "( OLcfgBkAt:13.40 " + "NAME 'olcBkLloadWeight' " + "DESC 'Backend weight' " + "SYNTAX OMsInteger " + "SINGLE-VALUE )", + NULL, + { .v_uint = 0 }, + }, #endif /* BALANCER_MODULE */ { NULL, NULL, 0, 0, 0, ARG_IGNORED, NULL } @@ -754,9 +775,13 @@ static ConfigTable config_back_cf_table[] = { #ifdef BALANCER_MODULE static ConfigCfAdd lload_cfadd; + static ConfigLDAPadd lload_backend_ldadd; +static ConfigLDAPadd lload_tier_ldadd; + #ifdef SLAP_CONFIG_DELETE static ConfigLDAPdel lload_backend_lddel; +static ConfigLDAPdel lload_tier_lddel; #endif /* SLAP_CONFIG_DELETE */ static ConfigOCs lloadocs[] = { @@ -807,12 +832,27 @@ static ConfigOCs lloadocs[] = { "$ olcBkLloadMaxPendingOps " "$ olcBkLloadMaxPendingConns ) " "MAY ( olcBkLloadStartTLS " + "$ olcBkLloadWeight ) " ") )", Cft_Misc, config_back_cf_table, lload_backend_ldadd, NULL, #ifdef SLAP_CONFIG_DELETE lload_backend_lddel, +#endif /* SLAP_CONFIG_DELETE */ + }, + { "( OLcfgBkOc:13.3 " + "NAME 'olcBkLloadTierConfig' " + "DESC 'Lload tier configuration' " + "SUP olcConfig STRUCTURAL " + "MUST ( cn " + "$ olcBkLloadTierType " + ") )", + Cft_Misc, config_back_cf_table, + lload_tier_ldadd, + NULL, +#ifdef SLAP_CONFIG_DELETE + lload_tier_lddel, #endif /* SLAP_CONFIG_DELETE */ }, { NULL, 0, NULL } @@ -1073,6 +1113,26 @@ lload_backend_finish( ConfigArgs *ca ) b->b_retry_event = event; } + if ( BER_BVISEMPTY( &b->b_name ) ) { + struct berval bv; + LloadBackend *b2; + int i = 1; + + LDAP_CIRCLEQ_FOREACH ( b2, &b->b_tier->t_backends, b_next ) { + i++; + } + + bv.bv_val = ca->cr_msg; + bv.bv_len = + snprintf( ca->cr_msg, sizeof(ca->cr_msg), "server %d", i ); + + ber_dupbv( &b->b_name, &bv ); + } + + if ( b->b_tier->t_type.tier_add_backend( b->b_tier, b ) ) { + goto fail; + } + return LDAP_SUCCESS; fail: @@ -1085,28 +1145,6 @@ fail: return -1; } -static LloadBackend * -backend_alloc( void ) -{ - LloadBackend *b; - - b = ch_calloc( 1, sizeof(LloadBackend) ); - - LDAP_CIRCLEQ_INIT( &b->b_conns ); - LDAP_CIRCLEQ_INIT( &b->b_bindconns ); - LDAP_CIRCLEQ_INIT( &b->b_preparing ); - - b->b_numconns = 1; - b->b_numbindconns = 1; - - b->b_retry_timeout = 5000; - - ldap_pvt_thread_mutex_init( &b->b_mutex ); - - LDAP_CIRCLEQ_INSERT_TAIL( &backend, b, b_next ); - return b; -} - static int backend_config_url( LloadBackend *b, struct berval *uri ) { @@ -1183,16 +1221,29 @@ static int config_backend( ConfigArgs *c ) { LloadBackend *b; + LloadTier *tier; int i, rc = 0; - b = backend_alloc(); + tier = LDAP_STAILQ_LAST( &tiers, LloadTier, t_next ); + if ( !tier ) { + Debug( LDAP_DEBUG_ANY, "config_backend: " + "no tier configured yet\n" ); + return -1; + } + + /* FIXME: maybe tier_add_backend could allocate it? */ + b = lload_backend_new(); + b->b_tier = tier; for ( i = 1; i < c->argc; i++ ) { if ( lload_backend_parse( c->argv[i], b ) ) { - Debug( LDAP_DEBUG_ANY, "config_backend: " - "error parsing backend configuration item '%s'\n", - c->argv[i] ); - return -1; + if ( !tier->t_type.tier_backend_config || + tier->t_type.tier_backend_config( tier, b, c->argv[i] ) ) { + Debug( LDAP_DEBUG_ANY, "config_backend: " + "error parsing backend configuration item '%s'\n", + c->argv[i] ); + return -1; + } } } @@ -1463,6 +1514,80 @@ done: return rc; } +static int +config_tier( ConfigArgs *c ) +{ + int rc = LDAP_SUCCESS; + struct lload_tier_type *tier_impl; + LloadTier *tier = c->ca_private; + struct berval bv; + int i = 1; + + if ( c->op == SLAP_CONFIG_EMIT ) { + switch ( c->type ) { + case CFG_TIER: + c->value_string = ch_strdup( tier->t_type.tier_name ); + break; + default: + goto fail; + break; + } + return rc; + + } else if ( c->op == LDAP_MOD_DELETE ) { + if ( lload_change.type != LLOAD_CHANGE_DEL ) { + /* + * TODO: Shouldn't really happen while this attribute is in the + * RDN, but we don't enforce it yet. + * + * How would we go about changing the backend type if we ever supported that? + */ + goto fail; + } + return rc; + } + + if ( CONFIG_ONLINE_ADD( c ) ) { + assert( tier ); + lload_change.target = tier; + return rc; + } + + tier_impl = lload_tier_find( c->value_string ); + if ( !tier_impl ) { + goto fail; + } + tier = tier_impl->tier_init(); + if ( !tier ) { + goto fail; + } + + lload_change.target = tier; + + if ( LDAP_STAILQ_EMPTY( &tiers ) ) { + LDAP_STAILQ_INSERT_HEAD( &tiers, tier, t_next ); + } else { + LloadTier *tier2; + LDAP_STAILQ_FOREACH ( tier2, &tiers, t_next ) { + i++; + } + LDAP_STAILQ_INSERT_TAIL( &tiers, tier, t_next ); + } + + bv.bv_val = c->cr_msg; + bv.bv_len = snprintf( c->cr_msg, sizeof(c->cr_msg), "tier %d", i ); + ber_dupbv( &tier->t_name, &bv ); + + return rc; + +fail: + if ( lload_change.type == LLOAD_CHANGE_ADD ) { + /* Abort the ADD */ + lload_change.type = LLOAD_CHANGE_DEL; + } + return 1; +} + static int config_fname( ConfigArgs *c ) { @@ -2957,6 +3082,9 @@ static slap_cf_aux_table backendkey[] = { { BER_BVC("max-pending-ops="), offsetof(LloadBackend, b_max_pending), 'i', 0, NULL }, { BER_BVC("conn-max-pending="), offsetof(LloadBackend, b_max_conn_pending), 'i', 0, NULL }, { BER_BVC("starttls="), offsetof(LloadBackend, b_tls_conf), 'i', 0, tlskey }, + + { BER_BVC("weight="), offsetof(LloadBackend, b_weight), 'i', 0, NULL }, + { BER_BVNULL, 0, 0, 0, NULL } }; @@ -3803,6 +3931,9 @@ backend_cf_gen( ConfigArgs *c ) case CFG_STARTTLS: enum_to_verb( tlskey, b->b_tls_conf, &c->value_bv ); break; + case CFG_WEIGHT: + c->value_uint = b->b_weight; + break; default: rc = 1; break; @@ -3884,6 +4015,9 @@ backend_cf_gen( ConfigArgs *c ) #endif /* ! HAVE_TLS */ b->b_tls_conf = tlskey[i].mask; } break; + case CFG_WEIGHT: + b->b_weight = c->value_uint; + break; default: rc = 1; break; @@ -3922,9 +4056,82 @@ lload_back_init_cf( BackendInfo *bi ) return config_register_schema( config_back_cf_table, lloadocs ); } +static int +lload_tier_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) +{ + LloadTier *tier; + Attribute *a; + AttributeDescription *ad = NULL; + struct lload_tier_type *tier_impl; + struct berval bv, type, rdn; + const char *text; + char *name; + + Debug( LDAP_DEBUG_TRACE, "lload_tier_ldadd: " + "a new tier is being added\n" ); + + if ( p->ce_type != Cft_Backend || !p->ce_bi || + p->ce_bi->bi_cf_ocs != lloadocs ) + return LDAP_CONSTRAINT_VIOLATION; + + dnRdn( &e->e_name, &rdn ); + type.bv_len = strchr( rdn.bv_val, '=' ) - rdn.bv_val; + type.bv_val = rdn.bv_val; + + /* Find attr */ + slap_bv2ad( &type, &ad, &text ); + if ( ad != slap_schema.si_ad_cn ) return LDAP_NAMING_VIOLATION; + + a = attr_find( e->e_attrs, ad ); + if ( !a || a->a_numvals != 1 ) return LDAP_NAMING_VIOLATION; + bv = a->a_vals[0]; + + if ( bv.bv_val[0] == '{' && ( name = strchr( bv.bv_val, '}' ) ) ) { + name++; + bv.bv_len -= name - bv.bv_val; + bv.bv_val = name; + } + + ad = NULL; + slap_str2ad( "olcBkLloadTierType", &ad, &text ); + assert( ad != NULL ); + + a = attr_find( e->e_attrs, ad ); + if ( !a || a->a_numvals != 1 ) return LDAP_OBJECT_CLASS_VIOLATION; + + tier_impl = lload_tier_find( a->a_vals[0].bv_val ); + if ( !tier_impl ) { + Debug( LDAP_DEBUG_ANY, "lload_tier_ldadd: " + "tier type %s not recongnised\n", + bv.bv_val ); + return LDAP_OTHER; + } + + tier = tier_impl->tier_init(); + if ( !tier ) { + return LDAP_OTHER; + } + + ber_dupbv( &tier->t_name, &bv ); + + ca->bi = p->ce_bi; + ca->ca_private = tier; + + /* ca cleanups are only run in the case of online config but we use it to + * save the new config when done with the entry */ + ca->lineno = 0; + + lload_change.type = LLOAD_CHANGE_ADD; + lload_change.object = LLOAD_TIER; + lload_change.target = tier; + + return LDAP_SUCCESS; +} + static int lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) { + LloadTier *tier = p->ce_private; LloadBackend *b; Attribute *a; AttributeDescription *ad = NULL; @@ -3935,7 +4142,7 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) Debug( LDAP_DEBUG_TRACE, "lload_backend_ldadd: " "a new backend-server is being added\n" ); - if ( p->ce_type != Cft_Backend || !p->ce_bi || + if ( p->ce_type != Cft_Misc || !p->ce_bi || p->ce_bi->bi_cf_ocs != lloadocs ) return LDAP_CONSTRAINT_VIOLATION; @@ -3957,8 +4164,9 @@ lload_backend_ldadd( CfEntryInfo *p, Entry *e, ConfigArgs *ca ) bv.bv_val = name; } - b = backend_alloc(); + b = lload_backend_new(); ber_dupbv( &b->b_name, &bv ); + b->b_tier = tier; ca->bi = p->ce_bi; ca->ca_private = b; @@ -3987,29 +4195,74 @@ lload_backend_lddel( CfEntryInfo *ce, Operation *op ) return LDAP_SUCCESS; } + +static int +lload_tier_lddel( CfEntryInfo *ce, Operation *op ) +{ + LloadTier *tier = ce->ce_private; + + lload_change.type = LLOAD_CHANGE_DEL; + lload_change.object = LLOAD_TIER; + lload_change.target = tier; + + return LDAP_SUCCESS; +} #endif /* SLAP_CONFIG_DELETE */ static int lload_cfadd( Operation *op, SlapReply *rs, Entry *p, ConfigArgs *c ) { struct berval bv; - LloadBackend *b; + LloadTier *tier; int i = 0; bv.bv_val = c->cr_msg; - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - char buf[STRLENOF( "server 4294967295" ) + 1] = { 0 }; + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + LloadBackend *b; + ConfigOCs *coc; + Entry *e; + int j = 0; bv.bv_len = snprintf( c->cr_msg, sizeof(c->cr_msg), - "cn=" SLAP_X_ORDERED_FMT "server %d", i, i + 1 ); - - snprintf( buf, sizeof(buf), "server %d", i + 1 ); - ber_str2bv( buf, 0, 1, &b->b_name ); + "cn=" SLAP_X_ORDERED_FMT "%s", i, tier->t_name.bv_val ); - c->ca_private = b; + c->ca_private = tier; c->valx = i; - config_build_entry( op, rs, p->e_private, c, &bv, &lloadocs[1], NULL ); + for ( coc = lloadocs; coc->co_type; coc++ ) { + if ( !ber_bvcmp( coc->co_name, &tier->t_type.tier_oc ) ) { + break; + } + } + assert( coc->co_type ); + + e = config_build_entry( op, rs, p->e_private, c, &bv, coc, NULL ); + if ( !e ) { + return 1; + } + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + bv.bv_len = snprintf( c->cr_msg, sizeof(c->cr_msg), + "cn=" SLAP_X_ORDERED_FMT "%s", j, b->b_name.bv_val ); + + for ( coc = lloadocs; coc->co_type; coc++ ) { + if ( !ber_bvcmp( + coc->co_name, &tier->t_type.tier_backend_oc ) ) { + break; + } + } + assert( coc->co_type ); + + c->ca_private = b; + c->valx = j; + + if ( !config_build_entry( + op, rs, e->e_private, c, &bv, coc, NULL ) ) { + return 1; + } + + j++; + } i++; } diff --git a/servers/lloadd/daemon.c b/servers/lloadd/daemon.c index 2c9a2f6e98..fe84454ae6 100644 --- a/servers/lloadd/daemon.c +++ b/servers/lloadd/daemon.c @@ -1231,7 +1231,7 @@ int lloadd_daemon( struct event_base *daemon_base ) { int i, rc; - LloadBackend *b; + LloadTier *tier; struct event_base *base; struct event *event; @@ -1276,20 +1276,9 @@ lloadd_daemon( struct event_base *daemon_base ) return rc; } - if ( !LDAP_CIRCLEQ_EMPTY( &backend ) ) { - current_backend = LDAP_CIRCLEQ_FIRST( &backend ); - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - event = evtimer_new( daemon_base, backend_connect, b ); - if ( !event ) { - Debug( LDAP_DEBUG_ANY, "lloadd: " - "failed to allocate retry event\n" ); - return -1; - } - - checked_lock( &b->b_mutex ); - b->b_retry_event = event; - backend_retry( b ); - checked_unlock( &b->b_mutex ); + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + if ( tier->t_type.tier_startup( tier ) ) { + return -1; } } @@ -1331,16 +1320,7 @@ lloadd_daemon( struct event_base *daemon_base ) destroy_listeners(); /* Mark upstream connections closing and prevent from opening new ones */ - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - epoch_t epoch = epoch_join(); - - checked_lock( &b->b_mutex ); - b->b_numconns = b->b_numbindconns = 0; - backend_reset( b, 1 ); - checked_unlock( &b->b_mutex ); - - epoch_leave( epoch ); - } + lload_tiers_shutdown(); /* Do the same for clients */ clients_destroy( 1 ); @@ -1368,7 +1348,7 @@ lloadd_daemon( struct event_base *daemon_base ) ldap_pvt_thread_pool_close( &connection_pool, 1 ); #endif - lload_backends_destroy(); + lload_tiers_destroy(); clients_destroy( 0 ); lload_bindconf_free( &bindconf ); evdns_base_free( dnsbase, 0 ); @@ -1468,6 +1448,7 @@ void lload_handle_backend_invalidation( LloadChange *change ) { LloadBackend *b = change->target; + LloadTier *tier = b->b_tier; assert( change->object == LLOAD_BACKEND ); @@ -1477,13 +1458,14 @@ lload_handle_backend_invalidation( LloadChange *change ) if ( mi ) { monitor_extra_t *mbe = mi->bi_extra; if ( mbe->is_configured() ) { - lload_monitor_backend_init( mi, b ); + lload_monitor_backend_init( mi, tier->t_monitor, b ); } } - if ( !current_backend ) { - current_backend = b; + if ( tier->t_type.tier_change ) { + tier->t_type.tier_change( tier, change ); } + checked_lock( &b->b_mutex ); backend_retry( b ); checked_unlock( &b->b_mutex ); @@ -1500,6 +1482,9 @@ lload_handle_backend_invalidation( LloadChange *change ) (CONNCB)detach_linked_backend_cb, b ); checked_unlock( &clients_mutex ); + if ( tier->t_type.tier_change ) { + tier->t_type.tier_change( tier, change ); + } lload_backend_destroy( b ); return; } @@ -1637,6 +1622,44 @@ lload_handle_backend_invalidation( LloadChange *change ) } } +void +lload_handle_tier_invalidation( LloadChange *change ) +{ + LloadTier *tier; + + assert( change->object == LLOAD_TIER ); + tier = change->target; + + if ( change->type == LLOAD_CHANGE_ADD ) { + BackendInfo *mi = backend_info( "monitor" ); + + if ( mi ) { + monitor_extra_t *mbe = mi->bi_extra; + if ( mbe->is_configured() ) { + lload_monitor_tier_init( mi, tier ); + } + } + + tier->t_type.tier_startup( tier ); + if ( LDAP_STAILQ_EMPTY( &tiers ) ) { + LDAP_STAILQ_INSERT_HEAD( &tiers, tier, t_next ); + } else { + LDAP_STAILQ_INSERT_TAIL( &tiers, tier, t_next ); + } + return; + } else if ( change->type == LLOAD_CHANGE_DEL ) { + LDAP_STAILQ_REMOVE( &tiers, tier, LloadTier, t_next ); + tier->t_type.tier_reset( tier, 1 ); + tier->t_type.tier_destroy( tier ); + return; + } + assert( change->type == LLOAD_CHANGE_MODIFY ); + + if ( tier->t_type.tier_change ) { + tier->t_type.tier_change( tier, change ); + } +} + void lload_handle_global_invalidation( LloadChange *change ) { @@ -1722,7 +1745,6 @@ lload_handle_global_invalidation( LloadChange *change ) #endif /* HAVE_TLS */ if ( change->flags.daemon & LLOAD_DAEMON_MOD_BINDCONF ) { - LloadBackend *b; LloadConnection *c; /* @@ -1734,12 +1756,7 @@ lload_handle_global_invalidation( LloadChange *change ) ldap_pvt_thread_pool_walk( &connection_pool, upstream_bind, backend_conn_cb, NULL ); - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - checked_lock( &b->b_mutex ); - backend_reset( b, 0 ); - backend_retry( b ); - checked_unlock( &b->b_mutex ); - } + lload_tiers_reset( 0 ); /* Reconsider the PRIVILEGED flag on all clients */ LDAP_CIRCLEQ_FOREACH ( c, &clients, c_next ) { @@ -1767,6 +1784,9 @@ lload_handle_invalidation( LloadChange *change ) case LLOAD_BACKEND: lload_handle_backend_invalidation( change ); break; + case LLOAD_TIER: + lload_handle_tier_invalidation( change ); + break; case LLOAD_DAEMON: lload_handle_global_invalidation( change ); break; diff --git a/servers/lloadd/init.c b/servers/lloadd/init.c index 57c6938820..a43b1d5162 100644 --- a/servers/lloadd/init.c +++ b/servers/lloadd/init.c @@ -98,7 +98,6 @@ lload_global_init( void ) ldap_pvt_thread_cond_init( &lload_wait_cond ); ldap_pvt_thread_cond_init( &lload_pause_cond ); - ldap_pvt_thread_mutex_init( &backend_mutex ); ldap_pvt_thread_mutex_init( &clients_mutex ); ldap_pvt_thread_mutex_init( &lload_pin_mutex ); diff --git a/servers/lloadd/lload.h b/servers/lloadd/lload.h index 6db6179cbd..5af765e8b2 100644 --- a/servers/lloadd/lload.h +++ b/servers/lloadd/lload.h @@ -98,6 +98,7 @@ LDAP_BEGIN_DECL #define assert_locked( mutex ) ( (void)0 ) #endif +typedef struct LloadTier LloadTier; typedef struct LloadBackend LloadBackend; typedef struct LloadPendingConnection LloadPendingConnection; typedef struct LloadConnection LloadConnection; @@ -105,13 +106,12 @@ typedef struct LloadOperation LloadOperation; typedef struct LloadChange LloadChange; /* end of forward declarations */ +typedef LDAP_STAILQ_HEAD(TierSt, LloadTier) lload_t_head; typedef LDAP_CIRCLEQ_HEAD(BeSt, LloadBackend) lload_b_head; typedef LDAP_CIRCLEQ_HEAD(ConnSt, LloadConnection) lload_c_head; -LDAP_SLAPD_V (lload_b_head) backend; +LDAP_SLAPD_V (lload_t_head) tiers; LDAP_SLAPD_V (lload_c_head) clients; -LDAP_SLAPD_V (ldap_pvt_thread_mutex_t) backend_mutex; -LDAP_SLAPD_V (LloadBackend *) current_backend; LDAP_SLAPD_V (struct slap_bindconf) bindconf; LDAP_SLAPD_V (struct berval) lloadd_identity; @@ -141,6 +141,7 @@ enum lc_object { /* LLOAD_BINDCONF, */ + LLOAD_TIER, LLOAD_BACKEND, }; @@ -153,6 +154,10 @@ enum lcf_daemon { LLOAD_DAEMON_MOD_BINDCONF = 1 << 5, }; +enum lcf_tier { + LLOAD_TIER_MOD_TYPE = 1 << 0, +}; + enum lcf_backend { LLOAD_BACKEND_MOD_OTHER = 1 << 0, LLOAD_BACKEND_MOD_CONNS = 1 << 1, @@ -164,6 +169,7 @@ struct LloadChange { union { int generic; enum lcf_daemon daemon; + enum lcf_tier tier; enum lcf_backend backend; } flags; void *target; @@ -224,6 +230,58 @@ typedef struct lload_global_stats_t { lload_counters_t counters[LLOAD_STATS_OPS_LAST]; } lload_global_stats_t; +typedef LloadTier *(LloadTierInit)( void ); +typedef int (LloadTierConfigCb)( LloadTier *tier, char *arg ); +typedef int (LloadTierBackendConfigCb)( LloadTier *tier, LloadBackend *b, char *arg ); +typedef int (LloadTierCb)( LloadTier *tier ); +typedef int (LloadTierResetCb)( LloadTier *tier, int shutdown ); +typedef int (LloadTierBackendCb)( LloadTier *tier, LloadBackend *b ); +typedef void (LloadTierChange)( LloadTier *tier, LloadChange *change ); +typedef int (LloadTierSelect)( LloadTier *tier, + LloadOperation *op, + LloadConnection **cp, + int *res, + char **message ); + +struct lload_tier_type { + char *tier_name; + + struct berval tier_oc, tier_backend_oc; + + LloadTierInit *tier_init; + LloadTierConfigCb *tier_config; + LloadTierBackendConfigCb *tier_backend_config; + LloadTierCb *tier_startup; + LloadTierResetCb *tier_reset; + LloadTierCb *tier_destroy; + + LloadTierBackendCb *tier_add_backend; + LloadTierBackendCb *tier_remove_backend; + LloadTierChange *tier_change; + + LloadTierSelect *tier_select; +}; + +struct LloadTier { + struct lload_tier_type t_type; + ldap_pvt_thread_mutex_t t_mutex; + + lload_b_head t_backends; + int t_nbackends; + + enum { + LLOAD_TIER_EXCLUSIVE = 1 << 0, /* Reject if busy */ + } t_flags; + + struct berval t_name; +#ifdef BALANCER_MODULE + monitor_subsys_t *t_monitor; +#endif /* BALANCER_MODULE */ + + void *t_private; + LDAP_STAILQ_ENTRY(LloadTier) t_next; +}; + /* Can hold mutex when locking a linked connection */ struct LloadBackend { ldap_pvt_thread_mutex_t b_mutex; @@ -248,6 +306,11 @@ struct LloadBackend { lload_counters_t b_counters[LLOAD_STATS_OPS_LAST]; + LloadTier *b_tier; + + uintptr_t b_fitness; + int b_weight; + #ifdef BALANCER_MODULE monitor_subsys_t *b_monitor; #endif /* BALANCER_MODULE */ diff --git a/servers/lloadd/monitor.c b/servers/lloadd/monitor.c index a9ee85bdee..f4acc9d461 100644 --- a/servers/lloadd/monitor.c +++ b/servers/lloadd/monitor.c @@ -57,11 +57,10 @@ #define LLOAD_MONITOR_OPERATIONS_DN \ LLOAD_MONITOR_OPERATIONS_RDN "," LLOAD_MONITOR_BALANCER_DN -#define LLOAD_MONITOR_BACKENDS_NAME "Backend Servers" -#define LLOAD_MONITOR_BACKENDS_RDN \ - SLAPD_MONITOR_AT "=" LLOAD_MONITOR_BACKENDS_NAME -#define LLOAD_MONITOR_BACKENDS_DN \ - LLOAD_MONITOR_BACKENDS_RDN "," LLOAD_MONITOR_BALANCER_DN +#define LLOAD_MONITOR_TIERS_NAME "Backend Tiers" +#define LLOAD_MONITOR_TIERS_RDN SLAPD_MONITOR_AT "=" LLOAD_MONITOR_TIERS_NAME +#define LLOAD_MONITOR_TIERS_DN \ + LLOAD_MONITOR_TIERS_RDN "," LLOAD_MONITOR_BALANCER_DN struct lload_monitor_ops_t { struct berval rdn; @@ -291,6 +290,25 @@ lload_monitor_backend_destroy( BackendDB *be, monitor_subsys_t *ms ) return rc; } +static int +lload_monitor_tier_destroy( BackendDB *be, monitor_subsys_t *ms ) +{ + LloadTier *tier = ms->mss_private; + monitor_extra_t *mbe; + + mbe = (monitor_extra_t *)be->bd_info->bi_extra; + if ( tier->t_monitor ) { + ms->mss_destroy = lload_monitor_subsystem_destroy; + + assert( tier->t_monitor == ms ); + tier->t_monitor = NULL; + + return mbe->unregister_entry( &ms->mss_ndn ); + } + + return LDAP_SUCCESS; +} + static void lload_monitor_balancer_dispose( void **priv ) { @@ -782,22 +800,21 @@ lload_monitor_backend_open( BackendDB *be, monitor_subsys_t *ms ) monitor_extra_t *mbe; monitor_callback_t *cb; LloadBackend *b = ms->mss_private; + LloadTier *tier = b->b_tier; int rc; assert( be != NULL ); mbe = (monitor_extra_t *)be->bd_info->bi_extra; - dnNormalize( 0, NULL, NULL, &ms->mss_dn, &ms->mss_ndn, NULL ); - e = mbe->entry_stub( &ms->mss_dn, &ms->mss_ndn, &ms->mss_rdn, - oc_olmBalancerServer, NULL, NULL ); + e = mbe->entry_stub( &tier->t_monitor->mss_dn, &tier->t_monitor->mss_ndn, + &ms->mss_rdn, oc_olmBalancerServer, NULL, NULL ); if ( e == NULL ) { Debug( LDAP_DEBUG_ANY, "lload_monitor_backend_open: " "unable to create entry \"%s,%s\"\n", - ms->mss_rdn.bv_val, ms->mss_ndn.bv_val ); + ms->mss_rdn.bv_val, tier->t_monitor->mss_dn.bv_val ); return -1; } - ch_free( ms->mss_ndn.bv_val ); ber_dupbv( &ms->mss_dn, &e->e_name ); ber_dupbv( &ms->mss_ndn, &e->e_nname ); @@ -833,13 +850,14 @@ done: } int -lload_monitor_backend_init( BackendInfo *bi, LloadBackend *b ) +lload_monitor_backend_init( + BackendInfo *bi, + monitor_subsys_t *ms, + LloadBackend *b ) { - monitor_extra_t *mbe; + monitor_extra_t *mbe = bi->bi_extra; monitor_subsys_t *bk_mss; - mbe = (monitor_extra_t *)bi->bi_extra; - /* FIXME: With back-monitor as it works now, there is no way to know when * this can be safely freed so we leak it on shutdown */ bk_mss = ch_calloc( 1, sizeof(monitor_subsys_t) ); @@ -848,7 +866,6 @@ lload_monitor_backend_init( BackendInfo *bi, LloadBackend *b ) bk_mss->mss_rdn.bv_len = snprintf( bk_mss->mss_rdn.bv_val, bk_mss->mss_rdn.bv_len, "cn=%s", b->b_name.bv_val ); - ber_str2bv( LLOAD_MONITOR_BACKENDS_DN, 0, 0, &bk_mss->mss_dn ); bk_mss->mss_name = b->b_name.bv_val; bk_mss->mss_flags = MONITOR_F_VOLATILE_CH; bk_mss->mss_open = lload_monitor_backend_open; @@ -866,13 +883,93 @@ lload_monitor_backend_init( BackendInfo *bi, LloadBackend *b ) return LDAP_SUCCESS; } +static int +lload_monitor_tier_open( BackendDB *be, monitor_subsys_t *ms ) +{ + Entry *e; + monitor_extra_t *mbe; + LloadTier *tier = ms->mss_private; + int rc; + + assert( be != NULL ); + mbe = (monitor_extra_t *)be->bd_info->bi_extra; + + dnNormalize( 0, NULL, NULL, &ms->mss_dn, &ms->mss_ndn, NULL ); + e = mbe->entry_stub( &ms->mss_dn, &ms->mss_ndn, &ms->mss_rdn, + oc_monitorContainer, NULL, NULL ); + if ( e == NULL ) { + Debug( LDAP_DEBUG_ANY, "lload_monitor_tier_open: " + "unable to create entry \"%s,%s\"\n", + ms->mss_rdn.bv_val, ms->mss_ndn.bv_val ); + return -1; + } + + ch_free( ms->mss_ndn.bv_val ); + ber_dupbv( &ms->mss_dn, &e->e_name ); + ber_dupbv( &ms->mss_ndn, &e->e_nname ); + + rc = mbe->register_entry( e, NULL, ms, MONITOR_F_PERSISTENT_CH ); + + if ( rc != LDAP_SUCCESS ) { + Debug( LDAP_DEBUG_ANY, "lload_monitor_tier_open: " + "unable to register entry \"%s\" for monitoring\n", + e->e_name.bv_val ); + goto done; + } + + tier->t_monitor = ms; + ms->mss_destroy = lload_monitor_tier_destroy; + +done: + entry_free( e ); + return rc; +} + int -lload_monitor_backends_init( BackendDB *be, monitor_subsys_t *ms ) +lload_monitor_tier_init( BackendInfo *bi, LloadTier *tier ) { monitor_extra_t *mbe; + monitor_subsys_t *mss; + LloadBackend *b; + + mbe = (monitor_extra_t *)bi->bi_extra; + + mss = ch_calloc( 1, sizeof(monitor_subsys_t) ); + mss->mss_rdn.bv_len = sizeof("cn=") + tier->t_name.bv_len; + mss->mss_rdn.bv_val = ch_malloc( mss->mss_rdn.bv_len ); + mss->mss_rdn.bv_len = snprintf( mss->mss_rdn.bv_val, mss->mss_rdn.bv_len, + "cn=%s", tier->t_name.bv_val ); + + ber_str2bv( LLOAD_MONITOR_TIERS_DN, 0, 0, &mss->mss_dn ); + mss->mss_name = tier->t_name.bv_val; + mss->mss_open = lload_monitor_tier_open; + mss->mss_destroy = lload_monitor_subsystem_destroy; + mss->mss_update = NULL; + mss->mss_private = tier; + + if ( mbe->register_subsys_late( mss ) ) { + Debug( LDAP_DEBUG_ANY, "lload_monitor_tier_init: " + "failed to register backend %s\n", + mss->mss_name ); + return -1; + } + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + if ( lload_monitor_backend_init( bi, mss, b ) ) { + return -1; + } + } + + return LDAP_SUCCESS; +} + +int +lload_monitor_tiers_init( BackendDB *be, monitor_subsys_t *ms ) +{ + monitor_extra_t *mbe; + LloadTier *tier; Entry *e; int rc; - LloadBackend *b; assert( be != NULL ); mbe = (monitor_extra_t *)be->bd_info->bi_extra; @@ -882,7 +979,7 @@ lload_monitor_backends_init( BackendDB *be, monitor_subsys_t *ms ) e = mbe->entry_stub( &ms->mss_dn, &ms->mss_ndn, &ms->mss_rdn, oc_monitorContainer, NULL, NULL ); if ( e == NULL ) { - Debug( LDAP_DEBUG_ANY, "lload_monitor_incoming_conn_init: " + Debug( LDAP_DEBUG_ANY, "lload_monitor_tiers_init: " "unable to create entry \"%s,%s\"\n", ms->mss_rdn.bv_val, ms->mss_ndn.bv_val ); return -1; @@ -894,14 +991,14 @@ lload_monitor_backends_init( BackendDB *be, monitor_subsys_t *ms ) rc = mbe->register_entry( e, NULL, ms, MONITOR_F_PERSISTENT_CH ); if ( rc != LDAP_SUCCESS ) { - Debug( LDAP_DEBUG_ANY, "lload_monitor_backends_init: " + Debug( LDAP_DEBUG_ANY, "lload_monitor_tiers_init: " "unable to register entry \"%s\" for monitoring\n", e->e_name.bv_val ); goto done; } - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - if ( (rc = lload_monitor_backend_init( be->bd_info, b )) ) { + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + if ( (rc = lload_monitor_tier_init( be->bd_info, tier )) ) { break; } } @@ -928,7 +1025,7 @@ lload_monitor_update_global_stats( void *ctx, void *arg ) { struct re_s *rtask = arg; lload_global_stats_t tmp_stats = {}; - LloadBackend *b; + LloadTier *tier; int i; Debug( LDAP_DEBUG_TRACE, "lload_monitor_update_global_stats: " @@ -940,18 +1037,22 @@ lload_monitor_update_global_stats( void *ctx, void *arg ) &tmp_stats ); checked_unlock( &clients_mutex ); - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - checked_lock( &b->b_mutex ); - tmp_stats.global_outgoing += b->b_active + b->b_bindavail; - - /* merge completed and failed stats */ - for ( i = 0; i < LLOAD_STATS_OPS_LAST; i++ ) { - tmp_stats.counters[i].lc_ops_completed += - b->b_counters[i].lc_ops_completed; - tmp_stats.counters[i].lc_ops_failed += - b->b_counters[i].lc_ops_failed; + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + LloadBackend *b; + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + checked_lock( &b->b_mutex ); + tmp_stats.global_outgoing += b->b_active + b->b_bindavail; + + /* merge completed and failed stats */ + for ( i = 0; i < LLOAD_STATS_OPS_LAST; i++ ) { + tmp_stats.counters[i].lc_ops_completed += + b->b_counters[i].lc_ops_completed; + tmp_stats.counters[i].lc_ops_failed += + b->b_counters[i].lc_ops_failed; + } + checked_unlock( &b->b_mutex ); } - checked_unlock( &b->b_mutex ); } /* update lload_stats */ @@ -975,7 +1076,7 @@ static char *lload_subsys_rdn[] = { LLOAD_MONITOR_BALANCER_RDN, LLOAD_MONITOR_INCOMING_RDN, LLOAD_MONITOR_OPERATIONS_RDN, - LLOAD_MONITOR_BACKENDS_RDN, + LLOAD_MONITOR_TIERS_RDN, NULL }; @@ -1023,14 +1124,14 @@ static struct monitor_subsys_t balancer_subsys[] = { NULL /* modify */ }, { - LLOAD_MONITOR_BACKENDS_NAME, + LLOAD_MONITOR_TIERS_NAME, BER_BVNULL, BER_BVC(LLOAD_MONITOR_BALANCER_DN), BER_BVNULL, { BER_BVC("Load Balancer Backends information"), BER_BVNULL }, MONITOR_F_PERSISTENT_CH, - lload_monitor_backends_init, + lload_monitor_tiers_init, lload_monitor_subsystem_destroy, /* destroy */ NULL, /* update */ NULL, /* create */ diff --git a/servers/lloadd/operation.c b/servers/lloadd/operation.c index 95657b9204..2ba475b61e 100644 --- a/servers/lloadd/operation.c +++ b/servers/lloadd/operation.c @@ -641,7 +641,7 @@ void operations_timeout( evutil_socket_t s, short what, void *arg ) { struct event *self = arg; - LloadBackend *b; + LloadTier *tier; time_t threshold; Debug( LDAP_DEBUG_TRACE, "operations_timeout: " @@ -650,31 +650,35 @@ operations_timeout( evutil_socket_t s, short what, void *arg ) threshold = slap_get_time() - lload_timeout_api->tv_sec; - LDAP_CIRCLEQ_FOREACH ( b, &backend, b_next ) { - epoch_t epoch; + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + LloadBackend *b; - checked_lock( &b->b_mutex ); - if ( b->b_n_ops_executing == 0 ) { - checked_unlock( &b->b_mutex ); - continue; - } + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + epoch_t epoch; - epoch = epoch_join(); + checked_lock( &b->b_mutex ); + if ( b->b_n_ops_executing == 0 ) { + checked_unlock( &b->b_mutex ); + continue; + } - Debug( LDAP_DEBUG_TRACE, "operations_timeout: " - "timing out binds for backend uri=%s\n", - b->b_uri.bv_val ); - connections_walk_last( &b->b_mutex, &b->b_bindconns, b->b_last_bindconn, - connection_timeout, &threshold ); + epoch = epoch_join(); - Debug( LDAP_DEBUG_TRACE, "operations_timeout: " - "timing out other operations for backend uri=%s\n", - b->b_uri.bv_val ); - connections_walk_last( &b->b_mutex, &b->b_conns, b->b_last_conn, - connection_timeout, &threshold ); + Debug( LDAP_DEBUG_TRACE, "operations_timeout: " + "timing out binds for backend uri=%s\n", + b->b_uri.bv_val ); + connections_walk_last( &b->b_mutex, &b->b_bindconns, + b->b_last_bindconn, connection_timeout, &threshold ); - epoch_leave( epoch ); - checked_unlock( &b->b_mutex ); + Debug( LDAP_DEBUG_TRACE, "operations_timeout: " + "timing out other operations for backend uri=%s\n", + b->b_uri.bv_val ); + connections_walk_last( &b->b_mutex, &b->b_conns, b->b_last_conn, + connection_timeout, &threshold ); + + epoch_leave( epoch ); + checked_unlock( &b->b_mutex ); + } } done: Debug( LDAP_DEBUG_TRACE, "operations_timeout: " diff --git a/servers/lloadd/proto-lload.h b/servers/lloadd/proto-lload.h index 5f7f93d568..e8903ba383 100644 --- a/servers/lloadd/proto-lload.h +++ b/servers/lloadd/proto-lload.h @@ -44,8 +44,8 @@ LDAP_SLAPD_F (int) upstream_select( LloadOperation *op, LloadConnection **c, int LDAP_SLAPD_F (int) backend_select( LloadBackend *b, LloadOperation *op, LloadConnection **c, int *res, char **message ); LDAP_SLAPD_F (int) try_upstream( LloadBackend *b, lload_c_head *head, LloadOperation *op, LloadConnection *c, int *res, char **message ); LDAP_SLAPD_F (void) backend_reset( LloadBackend *b, int gentle ); +LDAP_SLAPD_F (LloadBackend *) lload_backend_new( void ); LDAP_SLAPD_F (void) lload_backend_destroy( LloadBackend *b ); -LDAP_SLAPD_F (void) lload_backends_destroy( void ); /* * bind.c @@ -164,7 +164,8 @@ LDAP_SLAPD_F (void) lload_libevent_destroy( void ); * monitor.c */ LDAP_SLAPD_F (int) lload_monitor_open( void ); -LDAP_SLAPD_F (int) lload_monitor_backend_init( BackendInfo *bi, LloadBackend *b ); +LDAP_SLAPD_F (int) lload_monitor_backend_init( BackendInfo *bi, monitor_subsys_t *ms, LloadBackend *b ); +LDAP_SLAPD_F (int) lload_monitor_tier_init( BackendInfo *bi, LloadTier *tier ); #endif /* BALANCER_MODULE */ /* @@ -193,6 +194,18 @@ LDAP_SLAPD_F (void) operations_timeout( evutil_socket_t s, short what, void *arg LDAP_SLAPD_F (void) operation_update_conn_counters( LloadOperation *op, LloadConnection *upstream ); LDAP_SLAPD_F (void) operation_update_backend_counters( LloadOperation *op, LloadBackend *b ); LDAP_SLAPD_F (void) operation_update_global_rejected( LloadOperation *op ); + +/* + * tier.c + */ +LDAP_SLAPD_F (int) tier_startup( LloadTier *tier ); +LDAP_SLAPD_F (int) tier_reset( LloadTier *tier, int shutdown ); +LDAP_SLAPD_F (int) tier_destroy( LloadTier *tier ); +LDAP_SLAPD_F (void) lload_tiers_shutdown( void ); +LDAP_SLAPD_F (void) lload_tiers_reset( int shutdown ); +LDAP_SLAPD_F (void) lload_tiers_destroy( void ); +LDAP_SLAPD_F (struct lload_tier_type *) lload_tier_find( char *type ); + /* * upstream.c */ diff --git a/servers/lloadd/tier.c b/servers/lloadd/tier.c new file mode 100644 index 0000000000..e4970add8c --- /dev/null +++ b/servers/lloadd/tier.c @@ -0,0 +1,144 @@ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2019 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "portable.h" + +#include "lload.h" + +lload_t_head tiers; + +int +tier_startup( LloadTier *tier ) +{ + LloadBackend *b; + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + checked_lock( &b->b_mutex ); + if ( !b->b_retry_event ) { + b->b_retry_event = evtimer_new( daemon_base, backend_connect, b ); + if ( !b->b_retry_event ) { + Debug( LDAP_DEBUG_ANY, "tier_startup: " + "%s failed to allocate retry event\n", + tier->t_type.tier_name ); + return -1; + } + } + backend_retry( b ); + checked_unlock( &b->b_mutex ); + } + return LDAP_SUCCESS; +} + +int +tier_reset( LloadTier *tier, int shutdown ) +{ + LloadBackend *b; + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + epoch_t epoch = epoch_join(); + + checked_lock( &b->b_mutex ); + if ( shutdown ) { + b->b_numconns = b->b_numbindconns = 0; + } + backend_reset( b, 1 ); + backend_retry( b ); + checked_unlock( &b->b_mutex ); + + epoch_leave( epoch ); + } + return LDAP_SUCCESS; +} + +int +tier_destroy( LloadTier *tier ) +{ + while ( !LDAP_CIRCLEQ_EMPTY( &tier->t_backends ) ) { + LloadBackend *b = LDAP_CIRCLEQ_FIRST( &tier->t_backends ); + + lload_backend_destroy( b ); + } + +#ifdef BALANCER_MODULE + if ( tier->t_monitor ) { + BackendDB *be; + struct berval monitordn = BER_BVC("cn=monitor"); + int rc; + + be = select_backend( &monitordn, 0 ); + + /* FIXME: implement proper subsys shutdown in back-monitor or make + * backend just an entry, not a subsys */ + rc = tier->t_monitor->mss_destroy( be, tier->t_monitor ); + assert( rc == LDAP_SUCCESS ); + } +#endif /* BALANCER_MODULE */ + + ch_free( tier ); + return LDAP_SUCCESS; +} + +void +lload_tiers_destroy( void ) +{ + while ( !LDAP_STAILQ_EMPTY( &tiers ) ) { + LloadTier *tier = LDAP_STAILQ_FIRST( &tiers ); + + LDAP_STAILQ_REMOVE_HEAD( &tiers, t_next ); + tier->t_type.tier_destroy( tier ); + } +} + +void +lload_tiers_shutdown( void ) +{ + lload_tiers_reset( 1 ); +} + +void +lload_tiers_reset( int shutdown ) +{ + LloadTier *tier; + + LDAP_STAILQ_FOREACH ( tier, &tiers, t_next ) { + tier->t_type.tier_reset( tier, shutdown ); + } +} + +extern struct lload_tier_type roundrobin_tier; +extern struct lload_tier_type weighted_tier; + +struct { + char *name; + struct lload_tier_type *type; +} tier_types[] = { + { "roundrobin", &roundrobin_tier }, + { "weighted", &weighted_tier }, + + { NULL } +}; + +struct lload_tier_type * +lload_tier_find( char *name ) +{ + int i; + + for ( i = 0; tier_types[i].name; i++ ) { + if ( !strcasecmp( name, tier_types[i].name ) ) { + return tier_types[i].type; + } + } + return NULL; +} diff --git a/servers/lloadd/tier_roundrobin.c b/servers/lloadd/tier_roundrobin.c new file mode 100644 index 0000000000..0d924c7748 --- /dev/null +++ b/servers/lloadd/tier_roundrobin.c @@ -0,0 +1,135 @@ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2019 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "portable.h" + +#include "lload.h" + +static LloadTierInit roundrobin_init; +static LloadTierBackendCb roundrobin_add_backend; +static LloadTierBackendCb roundrobin_remove_backend; +static LloadTierSelect roundrobin_select; + +struct lload_tier_type roundrobin_tier; + +static LloadTier * +roundrobin_init( void ) +{ + LloadTier *tier; + + tier = ch_calloc( 1, sizeof(LloadTier) ); + + tier->t_type = roundrobin_tier; + ldap_pvt_thread_mutex_init( &tier->t_mutex ); + LDAP_CIRCLEQ_INIT( &tier->t_backends ); + + return tier; +} + +static int +roundrobin_add_backend( LloadTier *tier, LloadBackend *b ) +{ + assert( b->b_tier == tier ); + LDAP_CIRCLEQ_INSERT_TAIL( &tier->t_backends, b, b_next ); + if ( !tier->t_private ) { + tier->t_private = b; + } + tier->t_nbackends++; + return LDAP_SUCCESS; +} + +static int +roundrobin_remove_backend( LloadTier *tier, LloadBackend *b ) +{ + LloadBackend *next = LDAP_CIRCLEQ_LOOP_NEXT( &tier->t_backends, b, b_next ); + + assert_locked( &tier->t_mutex ); + assert_locked( &b->b_mutex ); + + assert( b->b_tier == tier ); + + LDAP_CIRCLEQ_REMOVE( &tier->t_backends, b, b_next ); + if ( b == tier->t_private ) { + if ( tier->t_nbackends ) { + tier->t_private = next; + } else { + assert( b == next ); + tier->t_private = NULL; + } + } + return LDAP_SUCCESS; +} + +static int +roundrobin_select( + LloadTier *tier, + LloadOperation *op, + LloadConnection **cp, + int *res, + char **message ) +{ + LloadBackend *b, *first, *next; + int rc = 0; + + checked_lock( &tier->t_mutex ); + first = b = tier->t_private; + checked_unlock( &tier->t_mutex ); + + if ( !first ) return rc; + + do { + int result; + + checked_lock( &b->b_mutex ); + next = LDAP_CIRCLEQ_LOOP_NEXT( &tier->t_backends, b, b_next ); + + result = backend_select( b, op, cp, res, message ); + checked_unlock( &b->b_mutex ); + + rc |= result; + if ( result && *cp ) { + /* + * Round-robin step: + * Rotate the queue to put this backend at the end. The race here + * is acceptable. + */ + checked_lock( &tier->t_mutex ); + tier->t_private = next; + checked_unlock( &tier->t_mutex ); + return rc; + } + + b = next; + } while ( b != first ); + + return rc; +} + +struct lload_tier_type roundrobin_tier = { + .tier_name = "roundrobin", + + .tier_init = roundrobin_init, + .tier_startup = tier_startup, + .tier_reset = tier_reset, + .tier_destroy = tier_destroy, + + .tier_oc = BER_BVC("olcBkLloadTierConfig"), + .tier_backend_oc = BER_BVC("olcBkLloadBackendConfig"), + + .tier_add_backend = roundrobin_add_backend, + .tier_remove_backend = roundrobin_remove_backend, + + .tier_select = roundrobin_select, +}; diff --git a/servers/lloadd/tier_weighted.c b/servers/lloadd/tier_weighted.c new file mode 100644 index 0000000000..b18c83d813 --- /dev/null +++ b/servers/lloadd/tier_weighted.c @@ -0,0 +1,224 @@ +/* $OpenLDAP$ */ +/* This work is part of OpenLDAP Software . + * + * Copyright 1998-2019 The OpenLDAP Foundation. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ + +#include "portable.h" + +#include + +#include "lload.h" +#include "lutil.h" + +static LloadTierInit weighted_init; +static LloadTierBackendCb weighted_add_backend; +static LloadTierBackendCb weighted_remove_backend; +static LloadTierSelect weighted_select; + +struct lload_tier_type weighted_tier; + +/* + * Linear Congruential Generator - we don't need + * high quality randomness, and we don't want to + * interfere with anyone else's use of srand(). + * + * The PRNG here cycles thru 941,955 numbers. + */ +static float weighted_seed; + +static void +weighted_srand( int seed ) +{ + weighted_seed = (float)seed / (float)RAND_MAX; +} + +static float +weighted_rand() +{ + float val = 9821.0 * weighted_seed + .211327; + weighted_seed = val - (int)val; + return weighted_seed; +} + +static void +weighted_shuffle( LloadBackend **b, int n ) +{ + int i, j, p; + uintptr_t total = 0, r; + + for ( i = 0; i < n; i++ ) + total += b[i]->b_weight; + + /* all weights are zero, do a straight Fisher-Yates shuffle */ + if ( !total ) { + while ( n ) { + LloadBackend *t; + i = weighted_rand() * n--; + t = b[n]; + b[n] = b[i]; + b[i] = t; + } + return; + } + + /* Do a shuffle per RFC2782 Page 4 */ + p = n; + for ( i = 0; i < n - 1; i++ ) { + r = weighted_rand() * total; + for ( j = 0; j < p; j++ ) { + r -= b[j]->b_weight; + if ( r <= 0 ) { + if ( j ) { + LloadBackend *t = b[0]; + b[0] = b[j]; + b[j] = t; + } + total -= b[0]->b_weight; + b++; + p--; + break; + } + } + /* TODO: once we have total == 0, should we jump over to the previous + * case? */ + } +} + +LloadTier * +weighted_init( void ) +{ + LloadTier *tier; + + tier = ch_calloc( 1, sizeof(LloadTier) ); + + tier->t_type = weighted_tier; + ldap_pvt_thread_mutex_init( &tier->t_mutex ); + LDAP_CIRCLEQ_INIT( &tier->t_backends ); + + weighted_srand( rand() ); + + return tier; +} + +int +weighted_add_backend( LloadTier *tier, LloadBackend *to_add ) +{ + LloadBackend *b; + uintptr_t added = 1; + + assert( to_add->b_tier == tier ); + + /* This requires us to use LDAP_CIRCLEQ_ENTRY_INIT() every time we have + * removed the backend from the list */ + if ( LDAP_CIRCLEQ_NEXT( to_add, b_next ) ) { + added = 0; + LDAP_CIRCLEQ_REMOVE( &tier->t_backends, to_add, b_next ); + } + + /* + * Keep it sorted. The only thing RFC 2782 specifies is that weight 0 + * entries are at the front of the list so they have a chance to be + * selected. + * + * Even with that in mind, there is a problem outlined in the RFC 2782 + * errata[0] where the ordering affects the likelihood of an entry being + * selected with weight 0 entries in the mix - they are an afterthought + * into the design after all. + * + * [0]. https://www.rfc-editor.org/errata/eid2984 + */ + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + if ( to_add->b_weight < b->b_weight ) { + LDAP_CIRCLEQ_INSERT_BEFORE( &tier->t_backends, b, to_add, b_next ); + goto done; + } + } + LDAP_CIRCLEQ_INSERT_TAIL( &tier->t_backends, to_add, b_next ); + +done: + tier->t_nbackends += added; + return LDAP_SUCCESS; +} + +static int +weighted_remove_backend( LloadTier *tier, LloadBackend *b ) +{ + assert_locked( &tier->t_mutex ); + assert_locked( &b->b_mutex ); + + assert( b->b_tier == tier ); + assert( tier->t_nbackends ); + + LDAP_CIRCLEQ_REMOVE( &tier->t_backends, b, b_next ); + LDAP_CIRCLEQ_ENTRY_INIT( b, b_next ); + tier->t_nbackends--; + + return LDAP_SUCCESS; +} + +int +weighted_select( + LloadTier *tier, + LloadOperation *op, + LloadConnection **cp, + int *res, + char **message ) +{ + LloadBackend *b, **sorted; + int rc = 0, i = 0; + + if ( !tier->t_nbackends ) return rc; + + sorted = ch_malloc( tier->t_nbackends * sizeof(LloadBackend *) ); + + LDAP_CIRCLEQ_FOREACH ( b, &tier->t_backends, b_next ) { + sorted[i++] = b; + } + + assert( i == tier->t_nbackends ); + + weighted_shuffle( sorted, tier->t_nbackends ); + + for ( i = 0; i < tier->t_nbackends; i++ ) { + int result; + + checked_lock( &sorted[i]->b_mutex ); + result = backend_select( sorted[i], op, cp, res, message ); + checked_unlock( &sorted[i]->b_mutex ); + + rc |= result; + if ( result && *cp ) { + break; + } + } + + ch_free( sorted ); + return rc; +} + +struct lload_tier_type weighted_tier = { + .tier_name = "weighted", + + .tier_init = weighted_init, + .tier_startup = tier_startup, + .tier_reset = tier_reset, + .tier_destroy = tier_destroy, + + .tier_oc = BER_BVC("olcBkLloadTierConfig"), + .tier_backend_oc = BER_BVC("olcBkLloadBackendConfig"), + + .tier_add_backend = weighted_add_backend, + .tier_remove_backend = weighted_remove_backend, + + .tier_select = weighted_select, +}; diff --git a/tests/data/lloadd-anon.conf b/tests/data/lloadd-anon.conf index a06799acd2..2e16bda15b 100644 --- a/tests/data/lloadd-anon.conf +++ b/tests/data/lloadd-anon.conf @@ -17,12 +17,17 @@ sockbuf_max_incoming_client 4194303 sockbuf_max_incoming_upstream 4194303 +tier roundrobin +# empty tier + +tier weighted backend-server uri=@URI2@ numconns=3 bindconns=2 retry=5000 max-pending-ops=5 conn-max-pending=3 + weight=10 backend-server uri=@URI3@ numconns=3 @@ -30,6 +35,7 @@ backend-server uri=@URI3@ retry=5000 max-pending-ops=5 conn-max-pending=3 + weight=1 backend-server uri=@URI4@ numconns=3 @@ -37,3 +43,4 @@ backend-server uri=@URI4@ retry=5000 max-pending-ops=5 conn-max-pending=3 + weight=5 diff --git a/tests/data/lloadd-backend-issues.conf b/tests/data/lloadd-backend-issues.conf index 6abf8c501b..2f37bde50e 100644 --- a/tests/data/lloadd-backend-issues.conf +++ b/tests/data/lloadd-backend-issues.conf @@ -22,6 +22,10 @@ bindconf binddn="cn=Manager,dc=example,dc=com" credentials=secret +tier roundrobin +# empty tier + +tier weighted # incorrect password (DB is empty) backend-server uri=@URI2@ numconns=3 @@ -29,6 +33,7 @@ backend-server uri=@URI2@ retry=500 max-pending-ops=5 conn-max-pending=3 + weight=10 # backend is often unresponsive backend-server uri=@URI3@ @@ -37,6 +42,7 @@ backend-server uri=@URI3@ retry=500 max-pending-ops=5 conn-max-pending=3 + weight=1 # unreachable backend (not running) backend-server uri=@URI4@ @@ -45,6 +51,7 @@ backend-server uri=@URI4@ retry=500 max-pending-ops=5 conn-max-pending=3 + weight=0 # backend that fails to resolve backend-server uri=ldap://does.not.resolve.example.com @@ -53,3 +60,4 @@ backend-server uri=ldap://does.not.resolve.example.com retry=500 max-pending-ops=5 conn-max-pending=3 + weight=5 diff --git a/tests/data/lloadd-sasl.conf b/tests/data/lloadd-sasl.conf index 6401d3f143..e7dfba340b 100644 --- a/tests/data/lloadd-sasl.conf +++ b/tests/data/lloadd-sasl.conf @@ -26,6 +26,10 @@ bindconf authzid="dn:cn=Manager,dc=example,dc=com" credentials=secret +tier roundrobin +# empty tier + +tier weighted backend-server uri=@URI2@ numconns=3 bindconns=3 diff --git a/tests/data/lloadd-tls.conf b/tests/data/lloadd-tls.conf index 00936c83a5..9499e95f10 100644 --- a/tests/data/lloadd-tls.conf +++ b/tests/data/lloadd-tls.conf @@ -34,6 +34,10 @@ bindconf credentials=secret tls_cacert=@TESTDIR@/tls/ca/certs/testsuiteCA.crt +tier roundrobin +# empty tier + +tier weighted backend-server uri=@URIP3@ starttls=critical numconns=3 diff --git a/tests/data/lloadd.conf b/tests/data/lloadd.conf index 99153657af..afd60bccfd 100644 --- a/tests/data/lloadd.conf +++ b/tests/data/lloadd.conf @@ -24,6 +24,10 @@ bindconf binddn="cn=Manager,dc=example,dc=com" credentials=secret +tier roundrobin +# empty tier + +tier weighted backend-server uri=@URI2@ numconns=3 bindconns=3 diff --git a/tests/data/lloadd/monitor.ldif b/tests/data/lloadd/monitor.ldif index 9e0f3fff73..dc18f771d4 100644 --- a/tests/data/lloadd/monitor.ldif +++ b/tests/data/lloadd/monitor.ldif @@ -26,7 +26,7 @@ olmRejectedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer @@ -58,10 +58,14 @@ olmRejectedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer -dn: cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor +objectClass: monitorContainer + +dn: cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monit + or objectClass: olmBalancerServer olmServerURI: ldap://localhost:9012/ olmActiveConnections: 4 @@ -71,8 +75,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 1,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 1,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -80,8 +84,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 3,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 3,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -89,8 +93,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 2,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 2,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -98,8 +102,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 4,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 4,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -136,10 +140,14 @@ olmRejectedOps: 0 olmCompletedOps: 2 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor +objectClass: monitorContainer + +dn: cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer -dn: cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monit + or objectClass: olmBalancerServer olmServerURI: ldap://localhost:9012/ olmActiveConnections: 4 @@ -149,8 +157,8 @@ olmReceivedOps: 2 olmCompletedOps: 2 olmFailedOps: 0 -dn: cn=Connection 1,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 1,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -158,8 +166,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 3,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 3,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -167,8 +175,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 2,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 2,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -176,8 +184,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 4,cn=first,cn=Backend Servers,cn=Load Balancer,cn=Backends,c - n=Monitor +dn: cn=Connection 4,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -185,7 +193,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Moni + tor objectClass: olmBalancerServer olmServerURI: ldap://localhost:9013/ olmActiveConnections: 9 @@ -195,8 +204,8 @@ olmReceivedOps: 2 olmCompletedOps: 2 olmFailedOps: 0 -dn: cn=Connection 5,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 5,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -204,8 +213,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 7,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 7,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -213,8 +222,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 8,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 8,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -222,8 +231,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 9,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 9,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -231,8 +240,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 6,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 6,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -240,8 +249,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 10,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 10,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -249,8 +258,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 11,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 11,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -258,8 +267,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 12,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 12,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -267,8 +276,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 13,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 13,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 diff --git a/tests/data/lloadd/test007-monitor.ldif b/tests/data/lloadd/test007-monitor.ldif index 87b86b21d7..3fb2363383 100644 --- a/tests/data/lloadd/test007-monitor.ldif +++ b/tests/data/lloadd/test007-monitor.ldif @@ -26,10 +26,14 @@ olmRejectedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer -dn: cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor +objectClass: monitorContainer + +dn: cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monit + or objectClass: olmBalancerServer olmServerURI: ldap://localhost:9012/ olmActiveConnections: 4 @@ -39,8 +43,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 1,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 1,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -48,8 +52,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 3,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 3,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -57,8 +61,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 2,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 2,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -66,8 +70,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 4,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 4,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -104,10 +108,14 @@ olmRejectedOps: 1 olmCompletedOps: 20 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor +objectClass: monitorContainer + +dn: cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer -dn: cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monit + or objectClass: olmBalancerServer olmServerURI: ldap://localhost:9012/ olmActiveConnections: 4 @@ -117,8 +125,8 @@ olmReceivedOps: 21 olmCompletedOps: 21 olmFailedOps: 0 -dn: cn=Connection 1,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 1,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -126,8 +134,8 @@ olmReceivedOps: 19 olmCompletedOps: 19 olmFailedOps: 0 -dn: cn=Connection 3,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 3,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -135,8 +143,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 2,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 2,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -144,8 +152,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 4,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 4,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -153,7 +161,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Moni + tor objectClass: olmBalancerServer olmServerURI: ldap://localhost:9013/ olmActiveConnections: 9 @@ -163,8 +172,8 @@ olmReceivedOps: 2 olmCompletedOps: 2 olmFailedOps: 0 -dn: cn=Connection 5,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 5,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -172,8 +181,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 7,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 7,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -181,8 +190,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 8,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 8,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -190,8 +199,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 9,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 9,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -199,8 +208,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 6,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 6,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -208,8 +217,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 10,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 10,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -217,8 +226,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 11,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 11,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -226,8 +235,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 12,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 12,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -235,8 +244,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 13,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 13,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -273,10 +282,14 @@ olmRejectedOps: 1 olmCompletedOps: 28 olmFailedOps: 0 -dn: cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor +objectClass: monitorContainer + +dn: cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monitor objectClass: monitorContainer -dn: cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Monit + or objectClass: olmBalancerServer olmServerURI: ldap://localhost:9012/ olmActiveConnections: 4 @@ -286,8 +299,8 @@ olmReceivedOps: 24 olmCompletedOps: 24 olmFailedOps: 0 -dn: cn=Connection 1,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 1,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -295,8 +308,8 @@ olmReceivedOps: 20 olmCompletedOps: 20 olmFailedOps: 0 -dn: cn=Connection 3,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 3,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -304,8 +317,8 @@ olmReceivedOps: 2 olmCompletedOps: 2 olmFailedOps: 0 -dn: cn=Connection 2,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 2,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -313,8 +326,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 4,cn=backend,cn=Backend Servers,cn=Load Balancer,cn=Backends - ,cn=Monitor +dn: cn=Connection 4,cn=backend,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=B + ackends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -322,7 +335,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backends,cn=Monitor +dn: cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn=Backends,cn=Moni + tor objectClass: olmBalancerServer olmServerURI: ldap://localhost:9013/ olmActiveConnections: 9 @@ -332,8 +346,8 @@ olmReceivedOps: 9 olmCompletedOps: 9 olmFailedOps: 0 -dn: cn=Connection 5,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 5,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -341,8 +355,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 7,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 7,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -350,8 +364,8 @@ olmReceivedOps: 5 olmCompletedOps: 5 olmFailedOps: 0 -dn: cn=Connection 8,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 8,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -359,8 +373,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 9,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 9,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: regular olmPendingOps: 0 @@ -368,8 +382,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 6,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backend - s,cn=Monitor +dn: cn=Connection 6,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn= + Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -377,8 +391,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 10,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 10,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -386,8 +400,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 11,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 11,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -395,8 +409,8 @@ olmReceivedOps: 1 olmCompletedOps: 1 olmFailedOps: 0 -dn: cn=Connection 12,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 12,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 @@ -404,8 +418,8 @@ olmReceivedOps: 0 olmCompletedOps: 0 olmFailedOps: 0 -dn: cn=Connection 13,cn=server 2,cn=Backend Servers,cn=Load Balancer,cn=Backen - ds,cn=Monitor +dn: cn=Connection 13,cn=server 2,cn=first,cn=Backend Tiers,cn=Load Balancer,cn + =Backends,cn=Monitor objectClass: olmBalancerConnection olmConnectionType: bind olmPendingOps: 0 diff --git a/tests/scripts/lloadd/test003-cnconfig b/tests/scripts/lloadd/test003-cnconfig index 377454cf77..0c008c7446 100755 --- a/tests/scripts/lloadd/test003-cnconfig +++ b/tests/scripts/lloadd/test003-cnconfig @@ -16,7 +16,7 @@ echo "running defines.sh" . $SRCDIR/scripts/defines.sh -mkdir -p $TESTDIR $DBDIR1 $DBDIR2 +mkdir -p $TESTDIR $CONF1.d $DBDIR1 $DBDIR2 $SLAPPASSWD -g -n >$CONFIGPWF echo "rootpw `$SLAPPASSWD -T $CONFIGPWF`" >$TESTDIR/configpw.conf @@ -157,10 +157,29 @@ fi echo "Deleting backends" -for i in 1 2 3 4 ; do - echo "cn={0}server "$i",olcBackend={0}lload,cn=config" +# weighted backends are sorted before we get to _cfadd to create their entries +for i in 3 2 4 1 ; do + echo "cn={0}server $i,cn={1}tier 2,olcBackend={0}lload,cn=config" $LDAPDELETE -H $URI6 -D cn=config -y $CONFIGPWF \ - "cn={0}server "$i",olcBackend={0}lload,cn=config" > /dev/null 2>&1 + "cn={0}server $i,cn={1}tier 2,olcBackend={0}lload,cn=config" > /dev/null 2>&1 + RC=$? + if test $RC != 0 ; then + echo "deleting server failed ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC + fi +done + +echo "Testing cn=config searching..." +$LDAPSEARCH -H $URI6 -D cn=config -y $CONFIGPWF \ + -s sub -b "olcBackend={0}lload,cn=config" '(objectclass=*)' > /dev/null 2>&1 + +echo "Deleting tiers" + +for i in 1 2; do + echo "cn={0}tier $i,olcBackend={0}lload,cn=config" + $LDAPDELETE -H $URI6 -D cn=config -y $CONFIGPWF \ + "cn={0}tier $i,olcBackend={0}lload,cn=config" > /dev/null 2>&1 RC=$? if test $RC != 0 ; then echo "deleting server failed ($RC)!" @@ -185,9 +204,35 @@ if test $RC != 52 ; then exit $RC fi -echo "Testing adding Server " +echo "Testing adding a tier" +$LDAPADD -D cn=config -H $URI6 -y $CONFIGPWF < $TESTOUT 2>&1 +dn: cn=roundrobin tier,olcBackend={0}lload,cn=config +objectClass: olcBkLloadTierConfig +olcBkLloadTierType: roundrobin +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for cn=roundrobin tier ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Testing adding a weighted tier" +$LDAPADD -D cn=config -H $URI6 -y $CONFIGPWF < $TESTOUT 2>&1 +dn: cn=weighted tier,olcBackend={0}lload,cn=config +objectClass: olcBkLloadTierConfig +olcBkLloadTierType: weighted +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for cn=weighted tier ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Testing adding a backend server" $LDAPADD -D cn=config -H $URI6 -y $CONFIGPWF < $TESTOUT 2>&1 -dn: cn=server 7,olcBackend={0}lload,cn=config +dn: cn=server 7,cn={1}weighted tier,olcBackend={0}lload,cn=config objectClass: olcBkLloadBackendConfig cn: server 7 olcBkLloadBackendUri: $URI3 @@ -196,6 +241,7 @@ olcBkLloadMaxPendingConns: 3 olcBkLloadMaxPendingOps: 5 olcBkLloadNumconns: 3 olcBkLloadRetry: 5000 +olcBkLloadWeight: 10 EOF RC=$? if test $RC != 0 ; then @@ -276,7 +322,7 @@ $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 dn: olcBackend={0}lload,cn=config changetype: modify replace: olcBkLloadMaxPDUPerCycle -olcBkLloadMaxPDUPerCycle: 2000 +olcBkLloadMaxPDUPerCycle: 0 EOF RC=$? @@ -374,7 +420,7 @@ fi echo "Testing backend attributes" echo "Testing olcBkLloadBindconns modify" $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 -dn: cn={0}server 7,olcBackend={0}lload,cn=config +dn: cn={0}server 7,cn={1}weighted tier,olcBackend={0}lload,cn=config changetype: modify replace: olcBkLloadBindconns olcBkLloadBindconns: 20 @@ -400,7 +446,7 @@ fi echo "Testing olcBkLloadMaxPendingConns modify" $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 -dn: cn={0}server 7,olcBackend={0}lload,cn=config +dn: cn={0}server 7,cn={1}weighted tier,olcBackend={0}lload,cn=config changetype: modify replace: olcBkLloadMaxPendingConns olcBkLloadMaxPendingConns: 30 diff --git a/tests/scripts/lloadd/test004-monitor b/tests/scripts/lloadd/test004-monitor index 795dee0278..3f2b35fb90 100755 --- a/tests/scripts/lloadd/test004-monitor +++ b/tests/scripts/lloadd/test004-monitor @@ -180,10 +180,24 @@ if test $RC != 0 ; then exit $RC fi -echo "Adding first backend server..." +echo "Adding first tier..." $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 dn: cn=first,olcBackend={0}lload,cn=config changetype: add +objectClass: olcBkLloadTierConfig +olcBkLloadTierType: roundrobin +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for backend ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + +echo "Adding first backend server..." +$LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 +dn: cn=backend,cn={0}first,olcBackend={0}lload,cn=config +changetype: add objectClass: olcBkLloadBackendConfig olcBkLloadBackendUri: $URI2 olcBkLloadMaxPendingConns: 3 @@ -191,6 +205,7 @@ olcBkLloadMaxPendingOps: 5 olcBkLloadRetry: 1000 olcBkLloadNumconns: 2 olcBkLloadBindconns: 2 +olcBkLloadWeight: 1 EOF RC=$? if test $RC != 0 ; then @@ -232,7 +247,7 @@ fi echo "Adding another backend server..." $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 -dn: cn=server 2,olcBackend={0}lload,cn=config +dn: cn=server 2,cn={0}first,olcBackend={0}lload,cn=config changetype: add objectClass: olcBkLloadBackendConfig olcBkLloadBackendUri: $URI3 @@ -241,6 +256,7 @@ olcBkLloadMaxPendingOps: 5 olcBkLloadRetry: 1000 olcBkLloadNumconns: 4 olcBkLloadBindconns: 5 +olcBkLloadWeight: 1 EOF RC=$? if test $RC != 0 ; then @@ -296,7 +312,7 @@ for i in 0 1 2 3 4 5; do if test $RC = 6 ; then break fi - echo "Waiting $SLEEP1 seconds until connections are established..." + echo "Waiting $SLEEP1 seconds until counters are updated..." sleep $SLEEP1 done if test $RC != 6 ; then diff --git a/tests/scripts/lloadd/test007-coherence b/tests/scripts/lloadd/test007-coherence index 00f55a280a..4a42085e12 100755 --- a/tests/scripts/lloadd/test007-coherence +++ b/tests/scripts/lloadd/test007-coherence @@ -28,8 +28,7 @@ if test $AC_lloadd = lloaddyes ; then fi # Monitor counts are unstable in the face of concurrency, since different -# clients may get different upstreams assigned for their operations. This might -# also change later when tiered load balancing is available. +# clients may get different upstreams assigned for their operations. # Another constraint is that some global counts are updated by the statistics # collection task scheduled to run every second. # @@ -203,9 +202,23 @@ if test $RC != 52 ; then exit $RC fi +echo "Adding first tier..." +$LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 +dn: cn=first,olcBackend={0}lload,cn=config +changetype: add +objectClass: olcBkLloadTierConfig +olcBkLloadTierType: roundrobin +EOF +RC=$? +if test $RC != 0 ; then + echo "ldapadd failed for backend ($RC)!" + test $KILLSERVERS != no && kill -HUP $KILLPIDS + exit $RC +fi + echo "Adding first backend server..." $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 -dn: cn=backend,olcBackend={0}lload,cn=config +dn: cn=backend,cn={0}first,olcBackend={0}lload,cn=config changetype: add objectClass: olcBkLloadBackendConfig olcBkLloadBackendUri: $URI2 @@ -257,7 +270,7 @@ fi echo "Adding another backend server..." $LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <> $TESTOUT 2>&1 -dn: cn=server 2,olcBackend={0}lload,cn=config +dn: cn=server 2,cn={0}first,olcBackend={0}lload,cn=config changetype: add objectClass: olcBkLloadBackendConfig olcBkLloadBackendUri: $URI3