]> git.ipfire.org Git - thirdparty/samba.git/commitdiff
winbindd: Add a per-domain failover log message with reason
authorShweta Sodani <ssodani@redhat.com>
Thu, 19 Feb 2026 14:19:28 +0000 (19:49 +0530)
committerAnoop C S <anoopcs@samba.org>
Wed, 24 Jun 2026 19:24:13 +0000 (19:24 +0000)
Adding a new log message that describe for domain controller failover
in winbindd_domain and cause of failover. (DC Connectivity, Kerberos,
LDAP failures)

Snippet of log message:

  print_failover_log:
  ========================================
  DC FAILOVER: SUCCESSFULLY FAILED OVER
  ========================================
  Domain Name: CORP
  Domain Alt Name: CORP.LOCAL
  Status: SUCCESS
  Failover Reason: DC_CONNECTIVITY
  Source DC (before failover): dc2.corp.local
  Destination DC (after failover): dc1.corp.local
  Current DC Name: dc1.corp.local
  Failover Start Time: 1771930355.281380
  Failover End Time: 1771930355.613872
  Elapsed Time (seconds): 0.332494
  ========================================

Signed-off-by: Shweta Sodani <ssodani@redhat.com>
Reviewed-by: Vinit Agnihotri <vagnihot@redhat.com>
Reviewed-by: Rabinarayan Panigrahi <rapanigr@redhat.com>
Reviewed-by: Xavi Hernandez <xhernandez@redhat.com>
Reviewed-by: Guenther Deschner <gd@samba.org>
Reviewed-by: Anoop C S <anoopcs@samba.org>
Autobuild-User(master): Anoop C S <anoopcs@samba.org>
Autobuild-Date(master): Wed Jun 24 19:24:13 UTC 2026 on atb-devel-224

source3/winbindd/winbindd.h
source3/winbindd/winbindd_ads.c
source3/winbindd/winbindd_cm.c
source3/winbindd/winbindd_dual_srv.c
source3/winbindd/winbindd_pam.c
source3/winbindd/winbindd_proto.h
source3/winbindd/winbindd_util.c

index 0ecfad10254686c62f3d65810cc17faadd9549a8..62b1f703a0070840f14b8abd2b2604ea9c742e53 100644 (file)
 
 #define WB_REPLACE_CHAR                '_'
 
+/* Enum to indicate the reason for DC failover */
+enum winbindd_failover_reason {
+       WINBINDD_FAILOVER_NONE = 0,           /* No failover */
+       WINBINDD_FAILOVER_DC_CONNECTIVITY,    /* DC connectivity issue */
+       WINBINDD_FAILOVER_KERBEROS,           /* Kerberos authentication failure */
+       WINBINDD_FAILOVER_LDAP                /* LDAP operation failure */
+};
+
 struct winbind_internal_pipes;
 struct ads_struct;
 
@@ -193,6 +201,11 @@ struct winbindd_domain {
        uint32_t sequence_number;
        NTSTATUS last_status;
 
+       /* DC failover metrics */
+       struct timeval start_dc_time;
+       char *src_dc;
+       enum winbindd_failover_reason failover_reason;
+
        /* The smb connection */
 
        struct winbindd_cm_conn conn;
index bb5e664a032d894d0dbe536826a0245efdfed71c..8fc3eafec73500ad866cc9cd9fbbceb9b02517ee 100644 (file)
@@ -160,6 +160,8 @@ static ADS_STATUS ads_cached_connection_connect(struct winbindd_domain *target_d
 
        status = ads_connect_creds(ads, creds);
        if (!ADS_ERR_OK(status)) {
+               /* LDAP connection failure - set failover reason if domain is available */
+               set_domain_failover_state(target_domain, WINBINDD_FAILOVER_LDAP);
                DEBUG(1,("ads_connect for domain %s failed: %s\n",
                         target_dom_name, ads_errstr(status)));
                goto out;
index 3007361fe8fd752b4982e82135087a82fcf1124b..42c0ba034ab2716f4b645873c340d8c4c4380d04 100644 (file)
@@ -109,6 +109,7 @@ static bool set_dc_type_and_flags_trustinfo( struct winbindd_domain *domain );
 static bool get_dcs(TALLOC_CTX *mem_ctx, struct winbindd_domain *domain,
                    struct dc_name_ip **dcs, int *num_dcs,
                    uint32_t request_flags);
+static const char *failover_reason_to_string(enum winbindd_failover_reason);
 
 void winbind_msg_domain_offline(struct messaging_context *msg_ctx,
                                void *private_data,
@@ -1544,16 +1545,29 @@ static bool find_dc(TALLOC_CTX *mem_ctx,
                return false;
        }
 
+       /*
+        * Preferred DC connection failed. We are now certain that
+        * another DC will be selected. Set failover reason to
+        * DC connectivity.
+        */
+       set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY);
+       DBG_NOTICE("DC failover detected: Preferred DC failed, selecting alternative DC "
+                "(reason: DC connectivity)\n");
+
        lp_ctx = loadparm_init_s3(talloc_tos(), loadparm_s3_helpers());
        if (lp_ctx == NULL) {
                DBG_ERR("loadparm_init_s3 failed\n");
+               reset_domain_failover_state(domain);
                return false;
        }
 
  again:
        D_DEBUG("Retrieving a list of IP addresses for DCs.\n");
-       if (!get_dcs(mem_ctx, domain, &dcs, &num_dcs, request_flags) || (num_dcs == 0))
+       if (!get_dcs(mem_ctx, domain, &dcs, &num_dcs, request_flags) || (num_dcs == 0)) {
+               /* No DCs found, reset failover state */
+               reset_domain_failover_state(domain);
                return False;
+       }
 
        D_DEBUG("Retrieved IP addresses for %d DCs.\n", num_dcs);
        for (i=0; i<num_dcs; i++) {
@@ -1590,6 +1604,8 @@ static bool find_dc(TALLOC_CTX *mem_ctx,
                        winbind_add_failed_connection_entry(domain,
                                dcs[i].name, NT_STATUS_UNSUCCESSFUL);
                }
+               /* All DCs failed, reset failover state */
+               reset_domain_failover_state(domain);
                return False;
        }
        talloc_reparent(NULL, mem_ctx, xtp);
@@ -1645,6 +1661,14 @@ return_transport:
 
        *ptransport = talloc_move(mem_ctx, &xtp);
 
+       if (domain->failover_reason != WINBINDD_FAILOVER_NONE) {
+               DBG_DEBUG("DC connection successful after failover: %s -> %s "
+                       "(reason: %s)\n",
+                       domain->src_dc ? domain->src_dc : "unknown",
+                       domain->dcname,
+                       failover_reason_to_string(domain->failover_reason));
+       }
+
        return true;
 }
 
@@ -1783,6 +1807,8 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
 
                found_dc = find_dc(mem_ctx, domain, request_flags, &xtp);
                if (!found_dc) {
+                       /* Failover is happening - no DC found */
+                       set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY);
                        /* This is the one place where we will
                           set the global winbindd offline state
                           to true, if a "WINBINDD_OFFLINE" entry
@@ -1803,6 +1829,9 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
                if (NT_STATUS_IS_OK(result)) {
                        break;
                }
+               /* Connection preparation failed - failover is happening */
+               set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY);
+
                if (!retry) {
                        break;
                }
@@ -1811,6 +1840,8 @@ static NTSTATUS cm_open_connection(struct winbindd_domain *domain,
        if (!NT_STATUS_IS_OK(result)) {
                /* Ensure we setup the retry handler. */
                set_domain_offline(domain);
+               /* Failover is happening - connection failed after retries */
+               set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY);
                goto out;
        }
 
@@ -1977,6 +2008,111 @@ static bool connection_ok(struct winbindd_domain *domain)
        return True;
 }
 
+/* Helper function to convert failover reason enum to string */
+static const char *failover_reason_to_string(enum winbindd_failover_reason reason)
+{
+       switch (reason) {
+       case WINBINDD_FAILOVER_NONE:
+               return "NONE";
+       case WINBINDD_FAILOVER_DC_CONNECTIVITY:
+               return "DC_CONNECTIVITY";
+       case WINBINDD_FAILOVER_KERBEROS:
+               return "KERBEROS";
+       case WINBINDD_FAILOVER_LDAP:
+               return "LDAP";
+       default:
+               return "UNKNOWN";
+       }
+}
+
+/* Helper function to set failover state and reason */
+void set_domain_failover_state(struct winbindd_domain *domain,
+                              enum winbindd_failover_reason reason)
+{
+       /* Log a message if failure occur again */
+       if (domain->failover_reason != WINBINDD_FAILOVER_NONE) {
+               DBG_DEBUG("Error encountered during DC failover. "
+                               "Failed DC: %s "
+                               "Failure reason: %s\n",
+                               domain->dcname ? domain->dcname : "UNKNOWN",
+                               failover_reason_to_string(reason));
+       }
+
+       /* Only set reason if it's not already set
+        * (preserve existing reason) */
+       if (domain->failover_reason == WINBINDD_FAILOVER_NONE) {
+               domain->failover_reason = reason;
+
+               /* Set src_dc to current DC name if not already set */
+               if (domain->dcname != NULL) {
+                       domain->src_dc = talloc_strdup(domain, domain->dcname);
+                       if (domain->src_dc == NULL) {
+                               DBG_ERR("talloc_strdup failed\n");
+                       }
+               }
+       }
+}
+
+/* Helper function to reset failover state */
+void reset_domain_failover_state(struct winbindd_domain *domain)
+{
+       domain->failover_reason = WINBINDD_FAILOVER_NONE;
+       TALLOC_FREE(domain->src_dc);
+}
+
+static void print_failover_log(struct winbindd_domain *domain, bool is_success)
+{
+       struct timeval current_time = timeval_current();
+       double elapsed_seconds = timeval_elapsed(&domain->start_dc_time);
+       const char *failover_reason_str = failover_reason_to_string(domain->failover_reason);
+       char *log_message = NULL;
+
+       if (domain->failover_reason == WINBINDD_FAILOVER_NONE) {
+               if (is_success) {
+                       DBG_DEBUG("Successfully connected to "
+                                               "domain controller %s\n",
+                                               domain->dcname);
+               }
+               return;
+       }
+
+       log_message = talloc_asprintf(talloc_tos(),
+               "========================================\n"
+               "DC FAILOVER: %s\n"
+               "========================================\n"
+               "Domain Name: %s\n"
+               "Domain Alt Name: %s\n"
+               "Status: %s\n"
+               "Failover Reason: %s\n"
+               "Source DC (before failover): %s\n"
+               "Destination DC (after failover): %s\n"
+               "Current DC Name: %s\n"
+               "Failover Start Time: %ld.%06ld\n"
+               "Current Time: %ld.%06ld\n"
+               "Elapsed Time (seconds): %.6f\n"
+               "========================================\n",
+               is_success ? "SUCCESSFULLY FAILED OVER" : "ATTEMPTING TO CONNECT",
+               domain->name ? domain->name : "N/A",
+               domain->alt_name ? domain->alt_name : "N/A",
+               is_success ? "SUCCESS" : "FAILED (Still trying to connect)",
+               failover_reason_str,
+               domain->src_dc ? domain->src_dc : "UNKNOWN",
+               domain->dcname ? domain->dcname : "UNKNOWN",
+               domain->dcname ? domain->dcname : "UNKNOWN",
+               (long)domain->start_dc_time.tv_sec,
+               (long)domain->start_dc_time.tv_usec,
+               (long)current_time.tv_sec,
+               (long)current_time.tv_usec,
+               elapsed_seconds);
+       if (log_message == NULL) {
+               DBG_ERR("talloc_asprintf failed\n");
+               return;
+       }
+
+       DBG_WARNING("\n%s\n", log_message);
+       TALLOC_FREE(log_message);
+}
+
 /* Initialize a new connection up to the RPC BIND.
    Bypass online status check so always does network calls. */
 
@@ -1984,6 +2120,8 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool
 {
        NTSTATUS result;
        bool skip_connection = domain->internal;
+
+       domain->start_dc_time = timeval_current();
        if (need_rw_dc && domain->rodc) {
                skip_connection = false;
        }
@@ -2001,6 +2139,8 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool
                return NT_STATUS_OK;
        }
 
+       /* Connection is not OK - failover will happen */
+       set_domain_failover_state(domain, WINBINDD_FAILOVER_DC_CONNECTIVITY);
        invalidate_cm_connection(domain);
 
        if (!domain->primary && !domain->initialized) {
@@ -2017,6 +2157,15 @@ static NTSTATUS init_dc_connection_network(struct winbindd_domain *domain, bool
                set_dc_type_and_flags(domain);
        }
 
+       if (NT_STATUS_IS_OK(result)) {
+               print_failover_log(domain, true);
+               /* Reset failover state on successful connection */
+               if (domain->failover_reason != WINBINDD_FAILOVER_NONE) {
+                       reset_domain_failover_state(domain);
+               }
+       } else {
+               print_failover_log(domain, false);
+       }
        return result;
 }
 
index c48ca15dd2b25c8461e07a2545b55e0862cb73f6..224013a775f5b7b5e7bd585bab47b864afe5dc01 100644 (file)
@@ -107,12 +107,15 @@ bool reset_cm_connection_on_error(struct winbindd_domain *domain,
                                  struct dcerpc_binding_handle *b,
                                  NTSTATUS status)
 {
+       bool ret = false;
+
        if (NT_STATUS_EQUAL(status, NT_STATUS_ACCESS_DENIED) ||
            NT_STATUS_EQUAL(status, NT_STATUS_RPC_SEC_PKG_ERROR) ||
            NT_STATUS_EQUAL(status, NT_STATUS_NETWORK_ACCESS_DENIED)) {
                invalidate_cm_connection(domain);
                domain->conn.netlogon_force_reauth = true;
-               return true;
+               ret = true;
+               goto out;
        }
 
        if (NT_STATUS_EQUAL(status, NT_STATUS_IO_TIMEOUT) ||
@@ -120,15 +123,24 @@ bool reset_cm_connection_on_error(struct winbindd_domain *domain,
        {
                invalidate_cm_connection(domain);
                /* We invalidated the connection. */
-               return true;
+               ret = true;
+               goto out;
        }
 
        if (b != NULL && !dcerpc_binding_handle_is_connected(b)) {
                invalidate_cm_connection(domain);
-               return true;
+               ret = true;
+               goto out;
+       }
+
+       /* Set failover flag if we're invalidating the connection */
+out:
+       if (ret) {
+               set_domain_failover_state(domain,
+                                       WINBINDD_FAILOVER_DC_CONNECTIVITY);
        }
 
-       return false;
+       return ret;
 }
 
 NTSTATUS _wbint_LookupSid(struct pipes_struct *p, struct wbint_LookupSid *r)
index f83bde98484b7b51a06306c3385194cc2872a816..b1aa4a7ad3218687131b0fca632b90d07793f304 100644 (file)
@@ -2510,6 +2510,8 @@ NTSTATUS _wbint_PamAuth(struct pipes_struct *p,
                    NT_STATUS_EQUAL(result, NT_STATUS_IO_TIMEOUT) ||
                    NT_STATUS_EQUAL(result, NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND)) {
                        DEBUG(10,("winbindd_dual_pam_auth_kerberos setting domain to offline\n"));
+                       /* Set failover flag and reason for Kerberos failure */
+                       set_domain_failover_state(domain, WINBINDD_FAILOVER_KERBEROS);
                        set_domain_offline( domain );
                        goto cached_logon;
                }
index 1b6a4f5d1158051aec416b48889de7d2e3d81d48..750f7c7bd425ff8554dc366acf28943b35352c21 100644 (file)
@@ -203,6 +203,9 @@ void winbind_msg_domain_online(struct messaging_context *msg_ctx,
 
 void set_domain_offline(struct winbindd_domain *domain);
 void set_domain_online_request(struct winbindd_domain *domain);
+void set_domain_failover_state(struct winbindd_domain *domain,
+                              enum winbindd_failover_reason reason);
+void reset_domain_failover_state(struct winbindd_domain *domain);
 void winbind_add_failed_connection_entry(
        const struct winbindd_domain *domain,
        const char *server,
index 3c83f96837d5e9ba9018909884d8a0e06e53c375..4ce265b8d99de73e379645a5178c82a4aa87572d 100644 (file)
@@ -333,6 +333,8 @@ static NTSTATUS add_trusted_domain(const char *domain_name,
        domain->domain_type = trust_type;
        domain->domain_trust_attribs = trust_attribs;
        domain->routing_domain = routing_domain;
+       domain->start_dc_time = timeval_zero();
+       reset_domain_failover_state(domain);
        sid_copy(&domain->sid, sid);
 
        /* Is this our primary domain ? */