From: Shweta Sodani Date: Thu, 19 Feb 2026 14:19:28 +0000 (+0530) Subject: winbindd: Add a per-domain failover log message with reason X-Git-Tag: talloc-2.5.0~389 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f99b8ae404da8476cc5af99a6bb403e2a2047468;p=thirdparty%2Fsamba.git winbindd: Add a per-domain failover log message with reason Adding a new log message that describe for domain controller failover in winbindd_domain and cause of failover. (DC Connectivity, Kerberos, LDAP failures) Snippet of log message: print_failover_log: ======================================== DC FAILOVER: SUCCESSFULLY FAILED OVER ======================================== Domain Name: CORP Domain Alt Name: CORP.LOCAL Status: SUCCESS Failover Reason: DC_CONNECTIVITY Source DC (before failover): dc2.corp.local Destination DC (after failover): dc1.corp.local Current DC Name: dc1.corp.local Failover Start Time: 1771930355.281380 Failover End Time: 1771930355.613872 Elapsed Time (seconds): 0.332494 ======================================== Signed-off-by: Shweta Sodani Reviewed-by: Vinit Agnihotri Reviewed-by: Rabinarayan Panigrahi Reviewed-by: Xavi Hernandez Reviewed-by: Guenther Deschner Reviewed-by: Anoop C S Autobuild-User(master): Anoop C S Autobuild-Date(master): Wed Jun 24 19:24:13 UTC 2026 on atb-devel-224 --- diff --git a/source3/winbindd/winbindd.h b/source3/winbindd/winbindd.h index 0ecfad10254..62b1f703a00 100644 --- a/source3/winbindd/winbindd.h +++ b/source3/winbindd/winbindd.h @@ -44,6 +44,14 @@ #define WB_REPLACE_CHAR '_' +/* Enum to indicate the reason for DC failover */ +enum winbindd_failover_reason { + WINBINDD_FAILOVER_NONE = 0, /* No failover */ + WINBINDD_FAILOVER_DC_CONNECTIVITY, /* DC connectivity issue */ + WINBINDD_FAILOVER_KERBEROS, /* Kerberos authentication failure */ + WINBINDD_FAILOVER_LDAP /* LDAP operation failure */ +}; + struct winbind_internal_pipes; struct ads_struct; @@ -193,6 +201,11 @@ struct winbindd_domain { uint32_t sequence_number; NTSTATUS last_status; + /* DC failover metrics */ + struct timeval start_dc_time; + char *src_dc; + enum winbindd_failover_reason failover_reason; + /* The smb connection */ struct winbindd_cm_conn conn; diff --git a/source3/winbindd/winbindd_ads.c b/source3/winbindd/winbindd_ads.c index bb5e664a032..8fc3eafec73 100644 --- a/source3/winbindd/winbindd_ads.c +++ b/source3/winbindd/winbindd_ads.c @@ -160,6 +160,8 @@ static ADS_STATUS ads_cached_connection_connect(struct winbindd_domain *target_d status = ads_connect_creds(ads, creds); if (!ADS_ERR_OK(status)) { + /* LDAP connection failure - set failover reason if domain is available */ + set_domain_failover_state(target_domain, WINBINDD_FAILOVER_LDAP); DEBUG(1,("ads_connect for domain %s failed: %s\n", target_dom_name, ads_errstr(status))); goto out; diff --git a/source3/winbindd/winbindd_cm.c b/source3/winbindd/winbindd_cm.c index 3007361fe8f..42c0ba034ab 100644 --- a/source3/winbindd/winbindd_cm.c +++ b/source3/winbindd/winbindd_cm.c @@ -109,6 +109,7 @@ static bool set_dc_type_and_flags_trustinfo( struct winbindd_domain *domain ); static bool get_dcs(TALLOC_CTX *mem_ctx, struct winbindd_domain *domain, struct dc_name_ip **dcs, int *num_dcs, uint32_t request_flags); +static const char *failover_reason_to_string(enum winbindd_failover_reason); void winbind_msg_domain_offline(struct messaging_context *msg_ctx, void *private_data, @@ -1544,16 +1545,29 @@ static bool find_dc(TALLOC_CTX *mem_ctx, return false; } + /* + * Preferred DC connection failed. We are now certain that + * another DC will be selected. Set failover reason to + * DC connectivity. + */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY); + DBG_NOTICE("DC failover detected: Preferred DC failed, selecting alternative DC " + "(reason: DC connectivity)\n"); + lp_ctx = loadparm_init_s3(talloc_tos(), loadparm_s3_helpers()); if (lp_ctx == NULL) { DBG_ERR("loadparm_init_s3 failed\n"); + reset_domain_failover_state(domain); return false; } again: D_DEBUG("Retrieving a list of IP addresses for DCs.\n"); - if (!get_dcs(mem_ctx, domain, &dcs, &num_dcs, request_flags) || (num_dcs == 0)) + if (!get_dcs(mem_ctx, domain, &dcs, &num_dcs, request_flags) || (num_dcs == 0)) { + /* No DCs found, reset failover state */ + reset_domain_failover_state(domain); return False; + } D_DEBUG("Retrieved IP addresses for %d DCs.\n", num_dcs); for (i=0; ifailover_reason != WINBINDD_FAILOVER_NONE) { + DBG_DEBUG("DC connection successful after failover: %s -> %s " + "(reason: %s)\n", + domain->src_dc ? domain->src_dc : "unknown", + domain->dcname, + failover_reason_to_string(domain->failover_reason)); + } + return true; } @@ -1783,6 +1807,8 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain, found_dc = find_dc(mem_ctx, domain, request_flags, &xtp); if (!found_dc) { + /* Failover is happening - no DC found */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY); /* This is the one place where we will set the global winbindd offline state to true, if a "WINBINDD_OFFLINE" entry @@ -1803,6 +1829,9 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain, if (NT_STATUS_IS_OK(result)) { break; } + /* Connection preparation failed - failover is happening */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY); + if (!retry) { break; } @@ -1811,6 +1840,8 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain, if (!NT_STATUS_IS_OK(result)) { /* Ensure we setup the retry handler. */ set_domain_offline(domain); + /* Failover is happening - connection failed after retries */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY); goto out; } @@ -1977,6 +2008,111 @@ static bool connection_ok(struct winbindd_domain *domain) return True; } +/* Helper function to convert failover reason enum to string */ +static const char *failover_reason_to_string(enum winbindd_failover_reason reason) +{ + switch (reason) { + case WINBINDD_FAILOVER_NONE: + return "NONE"; + case WINBINDD_FAILOVER_DC_CONNECTIVITY: + return "DC_CONNECTIVITY"; + case WINBINDD_FAILOVER_KERBEROS: + return "KERBEROS"; + case WINBINDD_FAILOVER_LDAP: + return "LDAP"; + default: + return "UNKNOWN"; + } +} + +/* Helper function to set failover state and reason */ +void set_domain_failover_state(struct winbindd_domain *domain, + enum winbindd_failover_reason reason) +{ + /* Log a message if failure occur again */ + if (domain->failover_reason != WINBINDD_FAILOVER_NONE) { + DBG_DEBUG("Error encountered during DC failover. " + "Failed DC: %s " + "Failure reason: %s\n", + domain->dcname ? domain->dcname : "UNKNOWN", + failover_reason_to_string(reason)); + } + + /* Only set reason if it's not already set + * (preserve existing reason) */ + if (domain->failover_reason == WINBINDD_FAILOVER_NONE) { + domain->failover_reason = reason; + + /* Set src_dc to current DC name if not already set */ + if (domain->dcname != NULL) { + domain->src_dc = talloc_strdup(domain, domain->dcname); + if (domain->src_dc == NULL) { + DBG_ERR("talloc_strdup failed\n"); + } + } + } +} + +/* Helper function to reset failover state */ +void reset_domain_failover_state(struct winbindd_domain *domain) +{ + domain->failover_reason = WINBINDD_FAILOVER_NONE; + TALLOC_FREE(domain->src_dc); +} + +static void print_failover_log(struct winbindd_domain *domain, bool is_success) +{ + struct timeval current_time = timeval_current(); + double elapsed_seconds = timeval_elapsed(&domain->start_dc_time); + const char *failover_reason_str = failover_reason_to_string(domain->failover_reason); + char *log_message = NULL; + + if (domain->failover_reason == WINBINDD_FAILOVER_NONE) { + if (is_success) { + DBG_DEBUG("Successfully connected to " + "domain controller %s\n", + domain->dcname); + } + return; + } + + log_message = talloc_asprintf(talloc_tos(), + "========================================\n" + "DC FAILOVER: %s\n" + "========================================\n" + "Domain Name: %s\n" + "Domain Alt Name: %s\n" + "Status: %s\n" + "Failover Reason: %s\n" + "Source DC (before failover): %s\n" + "Destination DC (after failover): %s\n" + "Current DC Name: %s\n" + "Failover Start Time: %ld.%06ld\n" + "Current Time: %ld.%06ld\n" + "Elapsed Time (seconds): %.6f\n" + "========================================\n", + is_success ? "SUCCESSFULLY FAILED OVER" : "ATTEMPTING TO CONNECT", + domain->name ? domain->name : "N/A", + domain->alt_name ? domain->alt_name : "N/A", + is_success ? "SUCCESS" : "FAILED (Still trying to connect)", + failover_reason_str, + domain->src_dc ? domain->src_dc : "UNKNOWN", + domain->dcname ? domain->dcname : "UNKNOWN", + domain->dcname ? domain->dcname : "UNKNOWN", + (long)domain->start_dc_time.tv_sec, + (long)domain->start_dc_time.tv_usec, + (long)current_time.tv_sec, + (long)current_time.tv_usec, + elapsed_seconds); + if (log_message == NULL) { + DBG_ERR("talloc_asprintf failed\n"); + return; + } + + DBG_WARNING("\n%s\n", log_message); + TALLOC_FREE(log_message); +} + /* Initialize a new connection up to the RPC BIND. Bypass online status check so always does network calls. */ @@ -1984,6 +2120,8 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool { NTSTATUS result; bool skip_connection = domain->internal; + + domain->start_dc_time = timeval_current(); if (need_rw_dc && domain->rodc) { skip_connection = false; } @@ -2001,6 +2139,8 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool return NT_STATUS_OK; } + /* Connection is not OK - failover will happen */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY); invalidate_cm_connection(domain); if (!domain->primary && !domain->initialized) { @@ -2017,6 +2157,15 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool set_dc_type_and_flags(domain); } + if (NT_STATUS_IS_OK(result)) { + print_failover_log(domain, true); + /* Reset failover state on successful connection */ + if (domain->failover_reason != WINBINDD_FAILOVER_NONE) { + reset_domain_failover_state(domain); + } + } else { + print_failover_log(domain, false); + } return result; } diff --git a/source3/winbindd/winbindd_dual_srv.c b/source3/winbindd/winbindd_dual_srv.c index c48ca15dd2b..224013a775f 100644 --- a/source3/winbindd/winbindd_dual_srv.c +++ b/source3/winbindd/winbindd_dual_srv.c @@ -107,12 +107,15 @@ bool reset_cm_connection_on_error(struct winbindd_domain *domain, struct dcerpc_binding_handle *b, NTSTATUS status) { + bool ret = false; + if (NT_STATUS_EQUAL(status, NT_STATUS_ACCESS_DENIED) || NT_STATUS_EQUAL(status, NT_STATUS_RPC_SEC_PKG_ERROR) || NT_STATUS_EQUAL(status, NT_STATUS_NETWORK_ACCESS_DENIED)) { invalidate_cm_connection(domain); domain->conn.netlogon_force_reauth = true; - return true; + ret = true; + goto out; } if (NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT) || @@ -120,15 +123,24 @@ bool reset_cm_connection_on_error(struct winbindd_domain *domain, { invalidate_cm_connection(domain); /* We invalidated the connection. */ - return true; + ret = true; + goto out; } if (b != NULL && !dcerpc_binding_handle_is_connected(b)) { invalidate_cm_connection(domain); - return true; + ret = true; + goto out; + } + + /* Set failover flag if we're invalidating the connection */ +out: + if (ret) { + set_domain_failover_state(domain, + WINBINDD_FAILOVER_DC_CONNECTIVITY); } - return false; + return ret; } NTSTATUS _wbint_LookupSid(struct pipes_struct *p, struct wbint_LookupSid *r) diff --git a/source3/winbindd/winbindd_pam.c b/source3/winbindd/winbindd_pam.c index f83bde98484..b1aa4a7ad32 100644 --- a/source3/winbindd/winbindd_pam.c +++ b/source3/winbindd/winbindd_pam.c @@ -2510,6 +2510,8 @@ NTSTATUS _wbint_PamAuth(struct pipes_struct *p, NT_STATUS_EQUAL(result, NT_STATUS_IO_TIMEOUT) || NT_STATUS_EQUAL(result, NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND)) { DEBUG(10,("winbindd_dual_pam_auth_kerberos setting domain to offline\n")); + /* Set failover flag and reason for Kerberos failure */ + set_domain_failover_state(domain, WINBINDD_FAILOVER_KERBEROS); set_domain_offline( domain ); goto cached_logon; } diff --git a/source3/winbindd/winbindd_proto.h b/source3/winbindd/winbindd_proto.h index 1b6a4f5d115..750f7c7bd42 100644 --- a/source3/winbindd/winbindd_proto.h +++ b/source3/winbindd/winbindd_proto.h @@ -203,6 +203,9 @@ void winbind_msg_domain_online(struct messaging_context *msg_ctx, void set_domain_offline(struct winbindd_domain *domain); void set_domain_online_request(struct winbindd_domain *domain); +void set_domain_failover_state(struct winbindd_domain *domain, + enum winbindd_failover_reason reason); +void reset_domain_failover_state(struct winbindd_domain *domain); void winbind_add_failed_connection_entry( const struct winbindd_domain *domain, const char *server, diff --git a/source3/winbindd/winbindd_util.c b/source3/winbindd/winbindd_util.c index 3c83f96837d..4ce265b8d99 100644 --- a/source3/winbindd/winbindd_util.c +++ b/source3/winbindd/winbindd_util.c @@ -333,6 +333,8 @@ static NTSTATUS add_trusted_domain(const char *domain_name, domain->domain_type = trust_type; domain->domain_trust_attribs = trust_attribs; domain->routing_domain = routing_domain; + domain->start_dc_time = timeval_zero(); + reset_domain_failover_state(domain); sid_copy(&domain->sid, sid); /* Is this our primary domain ? */