Added failover ack queueing. Failover update messages are no longer

immediately responded to, but are placed on a queue (attached to
the dhcp_failover_state_t structure).  Updated leases are committed
only when the queue is drained, giving a significant performance
increase.

The toack queue is drained:
  a) when the queue size exceeds half the partner's max-unacked-updates
     setting, or
  b) two seconds after the last update message is received.

Added 'refcnt' and 'next' fields to the failover_message_t structure,
to allow messages to be placed on the toack queue.

On the sending side, leases are no longer committed immediately after
being acked.  Rather, the commit is deferred until no further acks are
pending.  This also gives a major gain in performance.

diff --git a/includes/dhcpd.h b/includes/dhcpd.h
index 9ef6c4dcd50abf60b97556c93e6a501fbd058a6a..05a44e2fca37e33f682102fa32a9483f08e99732 100644 (file)
--- a/includes/dhcpd.h
+++ b/includes/dhcpd.h
@@ -2385,6 +2385,8 @@ void dhcp_failover_send_contact (void *);
 isc_result_t dhcp_failover_send_state (dhcp_failover_state_t *);
 isc_result_t dhcp_failover_send_updates (dhcp_failover_state_t *);
 int dhcp_failover_queue_update (struct lease *, int);
+int dhcp_failover_send_acks (dhcp_failover_state_t *);
+int dhcp_failover_queue_ack (dhcp_failover_state_t *, failover_message_t *msg);
 void dhcp_failover_ack_queue_remove (dhcp_failover_state_t *, struct lease *);
 isc_result_t dhcp_failover_state_set_value PROTO ((omapi_object_t *,
                                                   omapi_object_t *,
@@ -2434,7 +2436,7 @@ isc_result_t dhcp_failover_send_disconnect PROTO ((omapi_object_t *,
 isc_result_t dhcp_failover_send_bind_update (dhcp_failover_state_t *,
                                             struct lease *);
 isc_result_t dhcp_failover_send_bind_ack (dhcp_failover_state_t *,
-                                         struct lease *, failover_message_t *,
+                                         failover_message_t *,
                                          int, const char *);
 isc_result_t dhcp_failover_send_poolreq (dhcp_failover_state_t *);
 isc_result_t dhcp_failover_send_poolresp (dhcp_failover_state_t *, int);

diff --git a/includes/failover.h b/includes/failover.h
index 36b7ffbd59558e0b4fbe19f1fa8817649192671b..fbd9bedd9db52cbe824e0ea98f971f51a7c19d72 100644 (file)
--- a/includes/failover.h
+++ b/includes/failover.h
@@ -161,7 +161,10 @@ typedef struct {
 /* Failover server flags. */
 #define FTF_STARTUP    1
 
-typedef struct {
+typedef struct failover_message {
+       int refcnt;
+       struct failover_message *next;
+
        u_int8_t type;
 
        u_int8_t binding_status;
@@ -303,6 +306,13 @@ typedef struct _dhcp_failover_state {
                                           lease, send an UPDDONE message. */
        int cur_unacked_updates;        /* Number of updates we've sent
                                           that have not yet been acked. */
+
+                                       /* List of messages which we haven't
+                                          acked yet. */
+       failover_message_t *toack_queue_head;
+       failover_message_t *toack_queue_tail;
+       int pending_acks;               /* Number of messages in the toack
+                                          queue. */
 } dhcp_failover_state_t;
 
 #define DHCP_FAILOVER_VERSION          1

diff --git a/server/failover.c b/server/failover.c
index 98a73d64b32d8f1827c756f8eecba2f1a54fdee3..398f6e65449c3c60fc9a64a85f1b44f731b6b041 100644 (file)
--- a/server/failover.c
+++ b/server/failover.c
@@ -43,7 +43,7 @@
 
 #ifndef lint
 static char copyright[] =
-"$Id: failover.c,v 1.34 2001/02/15 22:14:41 neild Exp $ Copyright (c) 1999-2001 The Internet Software Consortium.  All rights reserved.\n";
+"$Id: failover.c,v 1.35 2001/02/22 00:15:54 neild Exp $ Copyright (c) 1999-2001 The Internet Software Consortium.  All rights reserved.\n";
 #endif /* not lint */
 
 #include "dhcpd.h"
@@ -57,6 +57,12 @@ static isc_result_t do_a_failover_option (omapi_object_t *,
                                          dhcp_failover_link_t *);
 dhcp_failover_listener_t *failover_listeners;
 
+static isc_result_t failover_message_reference (failover_message_t **,
+                                               failover_message_t *,
+                                               const char *file, int line);
+static isc_result_t failover_message_dereference (failover_message_t **,
+                                                 const char *file, int line);
+
 void dhcp_failover_startup ()
 {
        dhcp_failover_state_t *state;
@@ -339,13 +345,15 @@ isc_result_t dhcp_failover_link_signal (omapi_object_t *h,
                link -> state = dhcp_flink_message_wait;
                link -> imsg = dmalloc (sizeof (failover_message_t), MDL);
                if (!link -> imsg) {
+                       status = ISC_R_NOMEMORY;
                      dhcp_flink_fail:
                        if (link -> imsg) {
-                               dfree (link -> imsg, MDL);
-                               link -> imsg = (failover_message_t *)0;
+                               failover_message_dereference (&link->imsg,
+                                                             MDL);
                        }
                        link -> state = dhcp_flink_disconnected;
-                       log_info ("message length wait: no memory");
+                       log_info ("message length wait: %s",
+                                 isc_result_totext (status));
                        omapi_disconnect (c, 1);
                        /* XXX just blow away the protocol state now?
                           XXX or will disconnect blow it away? */
@@ -357,8 +365,10 @@ isc_result_t dhcp_failover_link_signal (omapi_object_t *h,
                link -> imsg_count = 0; /* Bytes read. */
                
                /* Maximum of 2048 bytes in any failover message. */
-               if (link -> imsg_len > DHCP_FAILOVER_MAX_MESSAGE_SIZE)
+               if (link -> imsg_len > DHCP_FAILOVER_MAX_MESSAGE_SIZE) {
+                       status = ISC_R_UNEXPECTED;
                        goto dhcp_flink_fail;
+               }
 
                if ((omapi_connection_require (c, link -> imsg_len - 2U)) !=
                    ISC_R_SUCCESS)
@@ -409,7 +419,8 @@ isc_result_t dhcp_failover_link_signal (omapi_object_t *h,
                                
                /* Now start sucking options off the wire. */
                while (link -> imsg_count < link -> imsg_len) {
-                       if (do_a_failover_option (c, link) != ISC_R_SUCCESS)
+                       status = do_a_failover_option (c, link);
+                       if (status != ISC_R_SUCCESS)
                                goto dhcp_flink_fail;
                }
 
@@ -497,8 +508,7 @@ isc_result_t dhcp_failover_link_signal (omapi_object_t *h,
                omapi_signal ((omapi_object_t *)link -> state_object,
                              "message", link);
                link -> state = dhcp_flink_message_length_wait;
-               dfree (link -> imsg, MDL);
-               link -> imsg = (failover_message_t *)0;
+               failover_message_dereference (&link -> imsg, MDL);
                /* XXX This is dangerous because we could get into a tight
                   XXX loop reading input without servicing any other stuff.
                   XXX There needs to be a way to relinquish control but
@@ -787,8 +797,7 @@ isc_result_t dhcp_failover_link_destroy (omapi_object_t *h,
                return ISC_R_INVALIDARG;
        link = (dhcp_failover_link_t *)h;
        if (link -> imsg) {
-               dfree (link -> imsg, file, line);
-               link -> imsg = (failover_message_t *)0;
+               failover_message_dereference (&link -> imsg, MDL);
        }
        if (link -> state_object)
                dhcp_failover_state_dereference (&link -> state_object, MDL);
@@ -2110,6 +2119,75 @@ int dhcp_failover_queue_update (struct lease *lease, int immediate)
        return 1;
 }
 
+int dhcp_failover_send_acks (dhcp_failover_state_t *state) {
+       failover_message_t *msg = (failover_message_t *)0;
+
+       /* Must commit all leases prior to acking them. */
+       if (!commit_leases ())
+               return 0;
+
+       while (state -> toack_queue_head) {
+               failover_message_reference
+                       (&msg, state -> toack_queue_head, MDL);
+               failover_message_dereference
+                       (&state -> toack_queue_head, MDL);
+               if (msg -> next) {
+                       failover_message_reference
+                               (&state -> toack_queue_head, msg -> next, MDL);
+               }
+
+               dhcp_failover_send_bind_ack (state, msg, 0, (const char *)0);
+
+               failover_message_dereference (&msg, MDL);
+       }
+
+       failover_message_dereference (&state -> toack_queue_tail, MDL);
+       state -> pending_acks = 0;
+
+       return 1;
+}
+
+void dhcp_failover_toack_queue_timeout (void *vs) {
+       dhcp_failover_state_t *state = vs;
+       dhcp_failover_send_acks (state);
+}
+
+/* Queue an ack for a message.  There is currently no way to queue a
+   negative ack -- these need to be sent directly. */
+
+int dhcp_failover_queue_ack (dhcp_failover_state_t *state,
+                            failover_message_t *msg)
+{
+       if (state -> toack_queue_head) {
+               failover_message_reference
+                       (&state -> toack_queue_tail -> next, msg, MDL);
+               failover_message_dereference (&state -> toack_queue_tail, MDL);
+       } else {
+               failover_message_reference (&state -> toack_queue_head,
+                                           msg, MDL);
+       }
+       failover_message_reference (&state -> toack_queue_tail, msg, MDL);
+
+       state -> pending_acks++;
+
+       /* Flush the toack queue whenever we exceed half the number of
+          allowed unacked updates. */
+       if (state -> pending_acks >= state -> partner.max_flying_updates / 2) {
+               dhcp_failover_send_acks (state);
+       }
+
+       /* Schedule a timeout to flush the ack queue. */
+       if (state -> pending_acks > 0) {
+               add_timeout (cur_time + 2,
+                            dhcp_failover_toack_queue_timeout, state,
+                            (tvref_t)dhcp_failover_state_reference,
+                            (tvunref_t)dhcp_failover_state_dereference);
+       }
+
+
+       return 1;
+}
+
 void dhcp_failover_ack_queue_remove (dhcp_failover_state_t *state,
                                     struct lease *lease)
 {
@@ -2127,14 +2205,15 @@ void dhcp_failover_ack_queue_remove (dhcp_failover_state_t *state,
                } else {
                        lease_dereference (&state -> ack_queue_tail, MDL);
                }
-               lease -> flags &= ~ON_ACK_QUEUE;
-               state -> cur_unacked_updates--;
-               return;
-       }
-       for (lp = state -> ack_queue_head;
-            lp && lp -> next_pending != lease; lp = lp -> next_pending)
-               ;
-       if (lp) {
+       } else {
+               for (lp = state -> ack_queue_head;
+                    lp && lp -> next_pending != lease;
+                    lp = lp -> next_pending)
+                       ;
+
+               if (!lp)
+                       return;
+
                lease_dereference (&lp -> next_pending, MDL);
                if (lease -> next_pending) {
                        lease_reference (&lp -> next_pending,
@@ -2145,8 +2224,18 @@ void dhcp_failover_ack_queue_remove (dhcp_failover_state_t *state,
                        lease_reference (&state -> ack_queue_tail, lp, MDL);
                }
        }
+
        lease -> flags &= ~ON_ACK_QUEUE;
        state -> cur_unacked_updates--;
+
+       /*
+        * When updating leases as a result of an ack, we defer the commit
+        * for performance reasons.  When there are no more acks pending,
+        * do a commit.
+        */
+       if (state -> cur_unacked_updates == 0) {
+               commit_leases();
+       }
 }
 
 isc_result_t dhcp_failover_state_set_value (omapi_object_t *h,
@@ -3604,7 +3693,6 @@ isc_result_t dhcp_failover_send_bind_update (dhcp_failover_state_t *state,
 /* Send a Bind ACK message. */
 
 isc_result_t dhcp_failover_send_bind_ack (dhcp_failover_state_t *state,
-                                         struct lease *lease,
                                          failover_message_t *msg,
                                          int reason, const char *message)
 {
@@ -3768,7 +3856,7 @@ isc_result_t dhcp_failover_send_update_request (dhcp_failover_state_t *state)
 #if defined (DEBUG_FAILOVER_MESSAGES)  
        char obuf [64];
        unsigned obufix = 0;
-       
+
 # define FMA obuf, &obufix, sizeof obuf
        failover_print (FMA, "(updreq");
 #else
@@ -3873,6 +3961,12 @@ isc_result_t dhcp_failover_send_update_done (dhcp_failover_state_t *state)
                log_debug ("%s", obuf);
        }
 #endif
+
+       /* There may be uncommitted leases at this point (since
+          dhcp_failover_process_bind_ack() doesn't commit leases);
+          commit the lease file. */
+       commit_leases();
+
        return status;
 }
 
@@ -3966,8 +4060,8 @@ isc_result_t dhcp_failover_process_bind_update (dhcp_failover_state_t *state,
                                         msg -> binding_status);
 #endif
                                dhcp_failover_send_bind_ack
-                                       (state, lease,
-                                        msg, FTR_FATAL_CONFLICT, outbuf);
+                                       (state, msg, FTR_FATAL_CONFLICT,
+                                        outbuf);
                                goto out;
                        }
                }
@@ -3975,13 +4069,14 @@ isc_result_t dhcp_failover_process_bind_update (dhcp_failover_state_t *state,
        }
 
        /* Try to install the new information. */
-       if (!supersede_lease (lease, lt, 1, 0, 0)) {
+       if (!supersede_lease (lease, lt, 0, 0, 0) ||
+           !write_lease (lease)) {
                message = "database update failed";
              bad:
-               dhcp_failover_send_bind_ack (state, lease, msg,
-                                            reason, message);
+               dhcp_failover_send_bind_ack (state, msg, reason, message);
        } else
-               dhcp_failover_send_bind_ack (state, lease, msg, 0, 0);
+               dhcp_failover_queue_ack (state, msg);
+
       out:
        if (lt)
                lease_dereference (&lt, MDL);
@@ -4034,7 +4129,8 @@ isc_result_t dhcp_failover_process_bind_ack (dhcp_failover_state_t *state,
        }
 
        /* Try to install the new information. */
-       supersede_lease (lease, lt, 1, 0, 0);
+       supersede_lease (lease, lt, 0, 0, 0);
+       write_lease (lease);
 
       unqueue:
        dhcp_failover_ack_queue_remove (state, lease);
@@ -4548,6 +4644,25 @@ int lease_mine_to_reallocate (struct lease *lease)
        return 1;
 }
 
+static isc_result_t failover_message_reference (failover_message_t **mp,
+                                               failover_message_t *m,
+                                               const char *file, int line)
+{
+       *mp = m;
+       m -> refcnt++;
+       return ISC_R_SUCCESS;
+}
+
+static isc_result_t failover_message_dereference (failover_message_t **mp,
+                                                 const char *file, int line)
+{
+       (*mp) -> refcnt--;
+       if ((*mp) -> refcnt == 0) {
+               dfree (*mp, MDL);
+       }
+       return ISC_R_SUCCESS;
+}
+
 OMAPI_OBJECT_ALLOC (dhcp_failover_state, dhcp_failover_state_t,
                    dhcp_type_failover_state)
 OMAPI_OBJECT_ALLOC (dhcp_failover_listener, dhcp_failover_listener_t,