ITS#9479 asyncmeta: fix hanging ops

Two separate problems
  1) ops that never got sent because of a pending Bind
  2) errors that never got returned because of an active op

diff --git a/servers/slapd/back-asyncmeta/back-asyncmeta.h b/servers/slapd/back-asyncmeta/back-asyncmeta.h
index 550054c1e4f76266f99fe32a88b0fc3044c2ffb8..85437bf2628bf0c497b81d98cda8d513740ae18d 100644 (file)
--- a/servers/slapd/back-asyncmeta/back-asyncmeta.h
+++ b/servers/slapd/back-asyncmeta/back-asyncmeta.h
@@ -590,6 +590,7 @@ extern LDAP_URLLIST_PROC    asyncmeta_back_default_urllist;
 #define        META_MSGID_NEED_BIND    (-2)
 #define        META_MSGID_CONNECTING   (-3)
 #define META_MSGID_UNDEFINED    (-4)
+#define META_MSGID_GOT_BIND     (-5)
 
 typedef enum meta_search_candidate_t {
        META_SEARCH_UNDEFINED = -2,

diff --git a/servers/slapd/back-asyncmeta/bind.c b/servers/slapd/back-asyncmeta/bind.c
index baef1643f94da773b6d312744aa56bb15ed87dbd..e268ce5b5022c6e06609158de0b27aee37e2d69f 100644 (file)
--- a/servers/slapd/back-asyncmeta/bind.c
+++ b/servers/slapd/back-asyncmeta/bind.c
@@ -1403,8 +1403,8 @@ asyncmeta_dobind_init(Operation *op, SlapReply *rs, bm_context_t *bc, a_metaconn
 
        meta_search_candidate_t retcode;
 
-       Debug( LDAP_DEBUG_TRACE, "%s >>> asyncmeta_dobind_init[%d]\n",
-               op->o_log_prefix, candidate );
+       Debug( LDAP_DEBUG_TRACE, "%s >>> asyncmeta_dobind_init[%d] msc %p\n",
+               op->o_log_prefix, candidate, msc );
 
        if ( mc->mc_authz_target == META_BOUND_ALL ) {
                return META_SEARCH_CANDIDATE;
@@ -1573,6 +1573,19 @@ retry_bind:
                Debug( asyncmeta_debug, "[%s] asyncmeta_dobind_init rc=%d msc: %p\n",
                      time_buf, rc, msc );
        }
+       if ( LogTest( LDAP_DEBUG_TRACE )) {
+               ber_socket_t s;
+               char sockname[LUTIL_ADDRLEN];
+               struct berval sockbv = BER_BVC( sockname );
+               Sockaddr addr;
+               socklen_t len = sizeof( addr );
+
+               ldap_get_option( msc->msc_ld, LDAP_OPT_DESC, &s );
+               getsockname( s, &addr.sa_addr, &len );
+               lutil_sockaddrstr( &addr, &sockbv );
+               Debug( LDAP_DEBUG_TRACE, "%s asyncmeta_dobind_init msc %p ld %p ldr %p fd %d addr %s\n",
+                       op->o_log_prefix, msc, msc->msc_ld, msc->msc_ldr, s, sockname );
+       }
 
        if (rc == LDAP_SERVER_DOWN ) {
                goto down;
@@ -1670,6 +1683,9 @@ asyncmeta_dobind_init_with_retry(Operation *op, SlapReply *rs, bm_context_t *bc,
        }
 
        if ( LDAP_BACK_CONN_ISBOUND( msc ) || LDAP_BACK_CONN_ISANON( msc ) ) {
+               if ( mc->pending_ops > 1 ) {
+                       asyncmeta_send_all_pending_ops( mc, candidate, op->o_threadctx, 1 );
+               }
                return META_SEARCH_CANDIDATE;
        }
 

diff --git a/servers/slapd/back-asyncmeta/meta_result.c b/servers/slapd/back-asyncmeta/meta_result.c
index 429857d4e81918639fc97dfd01cecf82976fefbd..2bd8759c7e3396892ffeb6aab1c3b092a4a820e0 100644 (file)
--- a/servers/slapd/back-asyncmeta/meta_result.c
+++ b/servers/slapd/back-asyncmeta/meta_result.c
@@ -642,7 +642,9 @@ asyncmeta_send_all_pending_ops(a_metaconn_t *mc, int candidate, void *ctx, int d
        for (bc = LDAP_STAILQ_FIRST(&mc->mc_om_list); bc; bc = onext) {
                meta_search_candidate_t ret;
                onext = LDAP_STAILQ_NEXT(bc, bc_next);
-               if (bc->candidates[candidate].sr_msgid != META_MSGID_NEED_BIND || bc->bc_active > 0 || bc->op->o_abandon > 0) {
+               if (bc->candidates[candidate].sr_msgid == META_MSGID_NEED_BIND)
+                       bc->candidates[candidate].sr_msgid = META_MSGID_GOT_BIND;
+               if (bc->candidates[candidate].sr_msgid != META_MSGID_GOT_BIND || bc->bc_active > 0 || bc->op->o_abandon > 0) {
                        continue;
                }
                bc->op->o_threadctx = ctx;
@@ -1380,6 +1382,7 @@ asyncmeta_op_read_error(a_metaconn_t *mc, int candidate, int error, void* ctx)
        Operation *op;
        SlapReply *rs;
        SlapReply *candidates;
+
        /* no outstanding ops, nothing to do but log */
        Debug( LDAP_DEBUG_TRACE,
               "asyncmeta_op_read_error: ldr=%p, err=%d\n",
@@ -1412,6 +1415,7 @@ asyncmeta_op_read_error(a_metaconn_t *mc, int candidate, int error, void* ctx)
                }
 
                if (bc->bc_active > 0) {
+                       bc->bc_invalid = 1;
                        continue;
                }
 
@@ -1619,22 +1623,21 @@ retry_bc:
 
        ldap_pvt_thread_mutex_lock( &mc->mc_om_mutex );
        rc = --mc->mc_active;
-       ldap_pvt_thread_mutex_unlock( &mc->mc_om_mutex );
        if (rc) {
                i++;
+               ldap_pvt_thread_mutex_unlock( &mc->mc_om_mutex );
                goto again;
        }
        slap_sl_mem_setctx(ctx, oldctx);
        if (mc->mc_conns) {
-               ldap_pvt_thread_mutex_lock( &mc->mc_om_mutex );
                for (i=0; i<ntargets; i++) {
                        if (!slapd_shutdown && !META_BACK_CONN_INVALID(msc)
                            && mc->mc_conns[i].msc_ldr && mc->mc_conns[i].conn) {
                                connection_client_enable(mc->mc_conns[i].conn);
                        }
                }
-               ldap_pvt_thread_mutex_unlock( &mc->mc_om_mutex );
        }
+       ldap_pvt_thread_mutex_unlock( &mc->mc_om_mutex );
        return NULL;
 }
 

diff --git a/servers/slapd/back-asyncmeta/search.c b/servers/slapd/back-asyncmeta/search.c
index 5321d007a1a3e4f965a486db33e486777a6cc570..37b9457ff2b0c7b6bd40b8bd4a4781f1fc501c5e 100644 (file)
--- a/servers/slapd/back-asyncmeta/search.c
+++ b/servers/slapd/back-asyncmeta/search.c
@@ -335,6 +335,7 @@ asyncmeta_back_search_start(
        LDAPControl             **ctrls = NULL;
        BerElement *ber = NULL;
        ber_int_t       msgid;
+       ber_socket_t s = -1;
 #ifdef SLAPD_META_CLIENT_PR
        LDAPControl             **save_ctrls = NULL;
 #endif /* SLAPD_META_CLIENT_PR */
@@ -572,7 +573,6 @@ done_pr:;
 
        if (ber) {
                struct timeval tv = {0, mt->mt_network_timeout*1000};
-               ber_socket_t s;
 
                if (!( LDAP_BACK_CONN_ISBOUND( msc )
                       || LDAP_BACK_CONN_ISANON( msc )) || msc->msc_ld == NULL ) {
@@ -664,7 +664,7 @@ done:;
        }
 
 doreturn:;
-       Debug( LDAP_DEBUG_TRACE, "%s <<< asyncmeta_back_search_start[%p]=%d\n", op->o_log_prefix, msc, candidates[candidate].sr_msgid );
+       Debug( LDAP_DEBUG_TRACE, "%s <<< asyncmeta_back_search_start[%p] (fd %d)=%d\n", op->o_log_prefix, msc, s, candidates[candidate].sr_msgid );
        return retcode;
 }
 
@@ -682,6 +682,7 @@ asyncmeta_back_search( Operation *op, SlapReply *rs )
        a_metaconn_t *mc;
        int msc_decr = 0;
        int max_pending_ops = (mi->mi_max_pending_ops == 0) ? META_BACK_CFG_MAX_PENDING_OPS : mi->mi_max_pending_ops;
+       int check_bind = 0;
 
        rs_assert_ready( rs );
        rs->sr_flags &= ~REP_ENTRY_MASK; /* paranoia, we can set rs = non-entry */
@@ -836,7 +837,8 @@ retry:
                case META_SEARCH_NEED_BIND:
                case META_SEARCH_BINDING:
                        Debug( LDAP_DEBUG_TRACE, "%s asyncmeta_back_search: BINDING "
-                              "cnd=\"%ld\" %p\n", op->o_log_prefix, i , &mc->mc_conns[i]);
+                              "cnd=\"%ld\" mc %p msc %p\n", op->o_log_prefix, i , mc, &mc->mc_conns[i]);
+                       check_bind++;
                        ncandidates++;
                        /* Todo add the context to the message queue but do not send the request
                         the receiver must send this when we are done binding */
@@ -915,6 +917,27 @@ retry:
                send_ldap_result(op, rs);
                goto finish;
        }
+
+       /* If we were processing many targets the result from a pending Bind
+        * on an earlier target may have arrived while we were sending to a
+        * later target. See if we can now send our pending request.
+        */
+       if ( check_bind ) {
+               for ( i = 0; i < mi->mi_ntargets; i++ ) {
+                       if ( candidates[ i ].sr_msgid == META_MSGID_GOT_BIND ) {
+                               rc = asyncmeta_back_search_start( op, rs, mc, bc, i, NULL, 0, 1 );
+                               if ( rc == META_SEARCH_ERR ) {
+                                       META_CANDIDATE_CLEAR( &candidates[i] );
+                                       candidates[ i ].sr_msgid = META_MSGID_IGNORE;
+                                       if ( META_BACK_ONERR_STOP( mi ) ) {
+                                               asyncmeta_handle_onerr_stop(op,rs,mc,bc,i);
+                                               goto finish;
+                                       }
+                               }
+                       }
+               }
+       }
+
        ldap_pvt_thread_mutex_lock( &mc->mc_om_mutex);
        for ( i = 0; i < mi->mi_ntargets; i++ ) {
                mc->mc_conns[i].msc_active--;