lib-http: client: Implemented support for connection failure backoff.

diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c
index 2e1e99baa167565ca8f0fae436b6fb6bb382db93..f26fe8ad5f66e803384104a0efb57558bf7555d4 100644 (file)
--- a/src/lib-http/http-client-peer.c
+++ b/src/lib-http/http-client-peer.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "net.h"
+#include "time-util.h"
 #include "str.h"
 #include "hash.h"
 #include "array.h"
@@ -78,7 +79,8 @@ int http_client_peer_addr_cmp
  */
 
 static void
-http_client_peer_connect(struct http_client_peer *peer, unsigned int count)
+http_client_peer_do_connect(struct http_client_peer *peer,
+       unsigned int count)
 {
        unsigned int i;
 
@@ -89,6 +91,42 @@ http_client_peer_connect(struct http_client_peer *peer, unsigned int count)
        }
 }
 
+
+static void
+http_client_peer_connect_backoff(struct http_client_peer *peer)
+{
+       i_assert(peer->to_backoff != NULL);
+
+       http_client_peer_debug(peer,
+               "Backoff timer expired");
+
+       timeout_remove(&peer->to_backoff);
+       http_client_peer_do_connect(peer, 1);
+}
+
+static void
+http_client_peer_connect(struct http_client_peer *peer, unsigned int count)
+{
+       if (peer->to_backoff != NULL)
+               return;
+
+       if (peer->last_failure.tv_sec > 0) {
+               int backoff_time_spent =
+                       timeval_diff_msecs(&ioloop_timeval, &peer->last_failure);
+               if (backoff_time_spent < (int)peer->backoff_time_msecs) {
+                       http_client_peer_debug(peer,
+                               "Starting backoff timer for %d msecs",
+                               peer->backoff_time_msecs - backoff_time_spent);
+                       peer->to_backoff = timeout_add
+                               ((unsigned int)(peer->backoff_time_msecs - backoff_time_spent),
+                                       http_client_peer_connect_backoff, peer);
+                       return;
+               }
+       }
+
+       http_client_peer_do_connect(peer, count);
+}
+
 bool http_client_peer_is_connected(struct http_client_peer *peer)
 {
        struct http_client_connection *const *conn_idx;
@@ -228,7 +266,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
        i_assert(idle == 0);
 
        /* determine how many new connections we can set up */
-       if (peer->last_connect_failed && working_conn_count > 0 &&
+       if (peer->last_failure.tv_sec > 0 && working_conn_count > 0 &&
            working_conn_count == connecting) {
                /* don't create new connections until the existing ones have
                   finished connecting successfully. */
@@ -381,6 +419,8 @@ void http_client_peer_free(struct http_client_peer **_peer)
 
        if (peer->to_req_handling != NULL)
                timeout_remove(&peer->to_req_handling);
+       if (peer->to_backoff != NULL)
+               timeout_remove(&peer->to_backoff);
 
        /* make a copy of the connection array; freed connections modify it */
        t_array_init(&conns, array_count(&peer->conns));
@@ -471,7 +511,11 @@ void http_client_peer_connection_success(struct http_client_peer *peer)
 {
        struct http_client_queue *const *queue;
 
-       peer->last_connect_failed = FALSE;
+       peer->last_failure.tv_sec = peer->last_failure.tv_usec = 0;
+       peer->backoff_time_msecs = 0;
+
+       if (peer->to_backoff != NULL)
+               timeout_remove(&peer->to_backoff);
 
        array_foreach(&peer->queues, queue) {
                http_client_queue_connection_success(*queue, &peer->addr);
@@ -483,6 +527,7 @@ void http_client_peer_connection_success(struct http_client_peer *peer)
 void http_client_peer_connection_failure(struct http_client_peer *peer,
                                         const char *reason)
 {
+       const struct http_client_settings *set = &peer->client->set;
        struct http_client_queue *const *queue;
        unsigned int num_urgent;
 
@@ -490,7 +535,15 @@ void http_client_peer_connection_failure(struct http_client_peer *peer,
 
        http_client_peer_debug(peer, "Failed to make connection");
 
-       peer->last_connect_failed = TRUE;
+       peer->last_failure = ioloop_timeval;
+
+       if (array_count(&peer->conns) == 1) {
+               if (peer->backoff_time_msecs == 0)
+                       peer->backoff_time_msecs = set->connect_backoff_time_msecs;
+               else
+                       peer->backoff_time_msecs *= 2;
+       }
+
        if (array_count(&peer->conns) > 1) {
                /* if there are other connections attempting to connect, wait
                   for them before failing the requests. remember that we had
@@ -553,5 +606,9 @@ void http_client_peer_switch_ioloop(struct http_client_peer *peer)
                peer->to_req_handling =
                        io_loop_move_timeout(&peer->to_req_handling);
        }
+       if (peer->to_backoff != NULL) {
+               peer->to_backoff =
+                       io_loop_move_timeout(&peer->to_backoff);
+       }
 }
 

diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index 3a942155d388831281a9c9281a2c23f0263a7391..35badd4a4fa91e6d72853bed7d52362873c4cc78 100644 (file)
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -11,8 +11,9 @@
 
 #define HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
 #define HTTP_CLIENT_CONNECT_TIMEOUT_MSECS (1000*30)
-#define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5)
 #define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2)
+#define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5)
+#define HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS (100)
 
 enum http_response_payload_type;
 
@@ -159,10 +160,14 @@ struct http_client_peer {
        /* zero time-out for consolidating request handling */
        struct timeout *to_req_handling;
 
+       /* connection retry */
+       struct timeval last_failure;
+       struct timeout *to_backoff;
+       unsigned int backoff_time_msecs;
+
        unsigned int destroyed:1;        /* peer is being destroyed */
        unsigned int no_payload_sync:1;  /* expect: 100-continue failed before */
        unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */
-       unsigned int last_connect_failed:1;
        unsigned int allows_pipelining:1;/* peer is known to allow persistent
                                             connections */
 };

diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c
index be368ddf04afc444ceab913566cb040e4f0138d0..1adb39d6839e93aa6ea577d718dc7cddcd4b35bc 100644 (file)
--- a/src/lib-http/http-client.c
+++ b/src/lib-http/http-client.c
@@ -121,6 +121,10 @@ struct http_client *http_client_init(const struct http_client_settings *set)
                (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
        client->set.max_attempts = set->max_attempts;
        client->set.max_connect_attempts = set->max_connect_attempts;
+       client->set.connect_backoff_time_msecs =
+               set->connect_backoff_time_msecs == 0 ?
+                       HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS :
+                       set->connect_backoff_time_msecs;
        client->set.no_auto_redirect = set->no_auto_redirect;
        client->set.no_ssl_tunnel = set->no_ssl_tunnel;
        client->set.max_redirects = set->max_redirects;

diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h
index 21ccff92c9ce81f31474e7ea16edf801494d0723..ee2caf906ccb9cdd627e1d1edfb940a82a77fc25 100644 (file)
--- a/src/lib-http/http-client.h
+++ b/src/lib-http/http-client.h
@@ -92,6 +92,9 @@ struct http_client_settings {
         */
        unsigned int max_connect_attempts;
 
+       /* Initial backoff time; doubled at each connection failure */
+       unsigned int connect_backoff_time_msecs;
+
        /* response header limits */
        struct http_header_limits response_hdr_limits;