]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
http-client: Changed struct http_client_host_port into a struct http_client_queue...
authorStephan Bosch <stephan@rename-it.nl>
Fri, 22 Nov 2013 20:07:41 +0000 (22:07 +0200)
committerStephan Bosch <stephan@rename-it.nl>
Fri, 22 Nov 2013 20:07:41 +0000 (22:07 +0200)
Peer and request objects now reference the queue object directly rather
than the host object. This way, there is no need to find the matching
host:port in the host anymore. This makes the queueing structure more
intuitive and more efficient. This is a first step towards support for
connecting to HTTP services through unix sockets or directing requests at
specific hosts (so not from the URL). This patch also fixes a potential
timeout leak (to_connect) in http_client_host_port (now http_client_queue)
and makes sure it is moved during switch_ioloop(). Finally it updates the
structure comment at the top of http-client.c.

src/lib-http/Makefile.am
src/lib-http/http-client-host.c
src/lib-http/http-client-peer.c
src/lib-http/http-client-private.h
src/lib-http/http-client-queue.c [new file with mode: 0644]
src/lib-http/http-client-request.c
src/lib-http/http-client.c

index 802a767b7bf5d4e5d3b1428ad792ad11589dc565..702c1e3eea29829af968cdaf2f53c9be0219a662 100644 (file)
@@ -21,6 +21,7 @@ libhttp_la_SOURCES = \
        http-client-request.c \
        http-client-connection.c \
        http-client-peer.c \
+       http-client-queue.c \
        http-client-host.c \
        http-client.c
 
index 427e4b2e90439880e4052d3e76eae83aacc82e54..1dbf2cd839c1a34c8c197c5d24e6e63905fa801d 100644 (file)
@@ -37,313 +37,20 @@ http_client_host_debug(struct http_client_host *host,
        }
 }
 
-/*
- * Host:port
- */
-
-static void
-http_client_host_port_connection_setup(struct http_client_host_port *hport);
-
-static struct http_client_host_port *
-http_client_host_port_find(struct http_client_host *host,
-       const struct http_client_peer_addr *addr)
-{
-       struct http_client_host_port *hport;
-
-       array_foreach_modifiable(&host->ports, hport) {
-               if (hport->addr.type == addr->type && hport->addr.port == addr->port &&
-                   null_strcmp(hport->addr.https_name, addr->https_name) == 0)
-                       return hport;
-       }
-
-       return NULL;
-}
-
-static struct http_client_host_port *
-http_client_host_port_init(struct http_client_host *host,
-       const struct http_client_peer_addr *addr)
-{
-       struct http_client_host_port *hport;
-
-       hport = http_client_host_port_find(host, addr);
-       if (hport == NULL) {
-               hport = array_append_space(&host->ports);
-               hport->host = host;
-               hport->addr = *addr;
-               hport->https_name = i_strdup(addr->https_name);
-               hport->addr.https_name = hport->https_name;
-               hport->ips_connect_idx = 0;
-               i_array_init(&hport->request_queue, 16);
-       }
-
-       return hport;
-}
-
-static void http_client_host_port_error(struct http_client_host_port *hport,
-       unsigned int status, const char *error)
-{
-       struct http_client_request **req;
-
-       /* abort all pending requests */
-       array_foreach_modifiable(&hport->request_queue, req) {
-               http_client_request_error(*req, status, error);
-       }
-       array_clear(&hport->request_queue);
-}
-
-static void http_client_host_port_deinit(struct http_client_host_port *hport)
-{
-       http_client_host_port_error
-               (hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
-       i_free(hport->https_name);
-       if (array_is_created(&hport->pending_peers))
-               array_free(&hport->pending_peers);
-       array_free(&hport->request_queue);
-}
-
-static void
-http_client_host_port_drop_request(struct http_client_host_port *hport,
-       struct http_client_request *req)
-{
-       ARRAY_TYPE(http_client_request) *req_arr = &hport->request_queue;
-       struct http_client_request **req_idx;
-
-       array_foreach_modifiable(req_arr, req_idx) {
-               if (*req_idx == req) {
-                       array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1);
-                       break;
-               }
-       }
-}
-
-static bool
-http_client_hport_is_last_connect_ip(struct http_client_host_port *hport)
-{
-       i_assert(hport->ips_connect_idx < hport->host->ips_count);
-       i_assert(hport->ips_connect_start_idx < hport->host->ips_count);
-
-       /* we'll always go through all the IPs. we don't necessarily start
-          connecting from the first IP, so we'll need to treat the IPs as
-          a ring buffer where we automatically wrap back to the first IP
-          when necessary. */
-       return (hport->ips_connect_idx + 1) % hport->host->ips_count ==
-               hport->ips_connect_start_idx;
-}
-
-static void
-http_client_host_port_soft_connect_timeout(struct http_client_host_port *hport)
-{
-       struct http_client_host *host = hport->host;
-       const struct http_client_peer_addr *addr = &hport->addr;
-
-       if (hport->to_connect != NULL)
-               timeout_remove(&hport->to_connect);
-
-       if (http_client_hport_is_last_connect_ip(hport)) {
-               /* no more IPs to try */
-               return;
-       }
-
-       /* if our our previous connection attempt takes longer than the
-          soft_connect_timeout, we start a connection attempt to the next IP in
-          parallel */
-       http_client_host_debug(host, "Connection to %s%s is taking a long time; "
-               "starting parallel connection attempt to next IP",
-               http_client_peer_addr2str(addr), addr->https_name == NULL ? "" :
-                       t_strdup_printf(" (SSL=%s)", addr->https_name)); 
-
-       /* next IP */
-       hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count;
-
-       /* setup connection to new peer (can start new soft timeout) */
-       http_client_host_port_connection_setup(hport);
-}
-
-static void
-http_client_host_port_connection_setup(struct http_client_host_port *hport)
-{
-       struct http_client_host *host = hport->host;
-       struct http_client_peer *peer = NULL;
-       const struct http_client_peer_addr *addr = &hport->addr;
-       unsigned int num_requests = array_count(&hport->request_queue);
-
-       if (num_requests == 0)
-               return;
-
-       /* update our peer address */
-       hport->addr.ip = host->ips[hport->ips_connect_idx];
-
-       http_client_host_debug(host, "Setting up connection to %s%s "
-               "(%u requests pending)", http_client_peer_addr2str(addr),
-               (addr->https_name == NULL ? "" :
-                       t_strdup_printf(" (SSL=%s)", addr->https_name)), num_requests);
-
-       /* create/get peer */
-       peer = http_client_peer_get(host->client, addr);
-       http_client_peer_add_host(peer, host);
-
-       /* handle requests; creates new connections when needed/possible */
-       http_client_peer_trigger_request_handler(peer);
-
-       if (!http_client_peer_is_connected(peer)) {
-               unsigned int msecs;
-
-               /* not already connected, wait for connections */
-               if (!array_is_created(&hport->pending_peers))
-                       i_array_init(&hport->pending_peers, 8);
-               array_append(&hport->pending_peers, &peer, 1);                  
-
-               /* start soft connect time-out (but only if we have another IP left) */
-               msecs = host->client->set.soft_connect_timeout_msecs;
-               if (!http_client_hport_is_last_connect_ip(hport) && msecs > 0 &&
-                   hport->to_connect == NULL) {
-                       hport->to_connect =
-                               timeout_add(msecs, http_client_host_port_soft_connect_timeout, hport);
-               }
-       }
-}
-
-static unsigned int
-http_client_host_get_ip_idx(struct http_client_host *host,
-                           const struct ip_addr *ip)
-{
-       unsigned int i;
-
-       for (i = 0; i < host->ips_count; i++) {
-               if (net_ip_compare(&host->ips[i], ip))
-                       return i;
-       }
-       i_unreached();
-}
-
-static void
-http_client_host_port_connection_success(struct http_client_host_port *hport,
-                                        const struct http_client_peer_addr *addr)
-{
-       /* we achieved at least one connection the the addr->ip */
-       hport->ips_connect_start_idx =
-               http_client_host_get_ip_idx(hport->host, &addr->ip);
-
-       /* stop soft connect time-out */
-       if (hport->to_connect != NULL)
-               timeout_remove(&hport->to_connect);
-
-       /* drop all other attempts to the hport. note that we get here whenever
-          a connection is successfully created, so pending_peers array
-          may be empty. */
-       if (array_is_created(&hport->pending_peers) &&
-               array_count(&hport->pending_peers) > 0) {
-               struct http_client_peer *const *peer_idx;
-
-               array_foreach(&hport->pending_peers, peer_idx) {
-                       if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
-                               /* don't drop any connections to the successfully
-                                  connected peer, even if some of the connections
-                                  are pending. they may be intended for urgent
-                                  requests. */
-                               continue;
-                       }
-                       /* remove this host from the peer; if this was the last/only host, the
-                          peer will be freed, closing all connections.
-                        */
-                       http_client_peer_remove_host(*peer_idx, hport->host);
-               }
-               array_clear(&hport->pending_peers);
-       }
-}
-
-static bool
-http_client_host_port_connection_failure(struct http_client_host_port *hport,
-       const struct http_client_peer_addr *addr, const char *reason)
-{
-       struct http_client_host *host = hport->host;
-
-       if (array_is_created(&hport->pending_peers) &&
-               array_count(&hport->pending_peers) > 0) {
-               struct http_client_peer *const *peer_idx;
-
-               /* we're still doing the initial connections to this hport. if
-                  we're also doing parallel connections with soft timeouts
-                  (pending_peer_count>1), wait for them to finish
-                  first. */
-               array_foreach(&hport->pending_peers, peer_idx) {
-                       if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
-                               array_delete(&hport->pending_peers,
-                                       array_foreach_idx(&hport->pending_peers, peer_idx), 1);
-                               break;
-                       }
-               }
-               if (array_count(&hport->pending_peers) > 0)
-                       return TRUE;
-       }
-
-       /* one of the connections failed. if we're not using soft timeouts,
-          we need to try to connect to the next IP. if we are using soft
-          timeouts, we've already tried all of the IPs by now. */
-       if (hport->to_connect != NULL)
-               timeout_remove(&hport->to_connect);
-
-       if (http_client_hport_is_last_connect_ip(hport)) {
-               /* all IPs failed, but retry all of them again on the
-                  next request. */
-               hport->ips_connect_idx = hport->ips_connect_start_idx =
-                       (hport->ips_connect_idx + 1) % host->ips_count;
-               http_client_host_port_error(hport,
-                       HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
-               return FALSE;
-       }
-       hport->ips_connect_idx = (hport->ips_connect_idx + 1) % host->ips_count;
-       http_client_host_port_connection_setup(hport);
-       return TRUE;
-}
-
 /*
  * Host
  */
 
-void http_client_host_connection_success(struct http_client_host *host,
-       const struct http_client_peer_addr *addr)
-{
-       struct http_client_host_port *hport;
-
-       http_client_host_debug(host, "Successfully connected to %s",
-               http_client_peer_addr2str(addr));
-
-       hport = http_client_host_port_find(host, addr);
-       if (hport == NULL)
-               return;
-
-       http_client_host_port_connection_success(hport, addr);
-}
-
-void http_client_host_connection_failure(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, const char *reason)
-{
-       struct http_client_host_port *hport;
-
-       http_client_host_debug(host, "Failed to connect to %s: %s",
-               http_client_peer_addr2str(addr), reason);
-
-       hport = http_client_host_port_find(host, addr);
-       if (hport == NULL)
-               return;
-
-       if (!http_client_host_port_connection_failure(hport, addr, reason)) {
-               /* failed definitively for currently queued requests */
-               if (host->client->ioloop != NULL)
-                       io_loop_stop(host->client->ioloop);
-       }
-}
-
 static void
-http_client_host_lookup_failure(struct http_client_host *host, const char *error)
+http_client_host_lookup_failure(struct http_client_host *host,
+                             const char *error)
 {
-       struct http_client_host_port *hport;
+       struct http_client_queue *const *queue_idx;
 
        error = t_strdup_printf("Failed to lookup host %s: %s",
                                host->name, error);
-       array_foreach_modifiable(&host->ports, hport) {
-               http_client_host_port_error(hport,
+       array_foreach_modifiable(&host->queues, queue_idx) {
+               http_client_queue_fail(*queue_idx,
                        HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, error);
        }
 }
@@ -352,7 +59,7 @@ static void
 http_client_host_dns_callback(const struct dns_lookup_result *result,
                              struct http_client_host *host)
 {
-       struct http_client_host_port *hport;
+       struct http_client_queue *const *queue_idx;
        unsigned int requests = 0;
 
        host->dns_lookup = NULL;
@@ -373,11 +80,12 @@ http_client_host_dns_callback(const struct dns_lookup_result *result,
        /* FIXME: make DNS result expire */
 
        /* make connections to requested ports */
-       array_foreach_modifiable(&host->ports, hport) {
-               unsigned int count = array_count(&hport->request_queue);
-               hport->ips_connect_idx = hport->ips_connect_start_idx = 0;
+       array_foreach_modifiable(&host->queues, queue_idx) {
+               struct http_client_queue *queue = *queue_idx;
+               unsigned int count = array_count(&queue->request_queue);
+               queue->ips_connect_idx = queue->ips_connect_start_idx = 0;
                if (count > 0)
-                       http_client_host_port_connection_setup(hport);
+                       http_client_queue_connection_setup(queue);
                requests += count;
        }
 
@@ -440,7 +148,7 @@ struct http_client_host *http_client_host_get
                host = i_new(struct http_client_host, 1);
                host->client = client;
                host->name = i_strdup(hostname);
-               i_array_init(&host->ports, 4);
+               i_array_init(&host->queues, 4);
                i_array_init(&host->delayed_failing_requests, 1);
 
                hostname = host->name;
@@ -461,7 +169,7 @@ struct http_client_host *http_client_host_get
 void http_client_host_submit_request(struct http_client_host *host,
        struct http_client_request *req)
 {
-       struct http_client_host_port *hport;
+       struct http_client_queue *queue;
        const struct http_url *host_url = req->host_url;
        struct http_client_peer_addr addr;
        const char *error;
@@ -478,12 +186,9 @@ void http_client_host_submit_request(struct http_client_host *host,
 
        http_client_request_get_peer_addr(req, &addr);
 
-       /* add request to host (grouped by tcp port) */
-       hport = http_client_host_port_init(host, &addr);
-       if (req->urgent)
-               array_insert(&hport->request_queue, 0, &req, 1);
-       else
-               array_append(&hport->request_queue, &req, 1);
+       /* add request to queue (grouped by tcp port) */
+       queue = http_client_queue_create(host, &addr);
+       http_client_queue_submit_request(queue, req);
 
        /* start DNS lookup if necessary */
        if (host->ips_count == 0 && host->dns_lookup == NULL)   
@@ -492,86 +197,14 @@ void http_client_host_submit_request(struct http_client_host *host,
        /* make a connection if we have an IP already */
        if (host->ips_count == 0)
                return;
-       i_assert(hport->ips_connect_idx < host->ips_count);
-       http_client_host_port_connection_setup(hport);
-}
-
-struct http_client_request *
-http_client_host_claim_request(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, bool no_urgent)
-{
-       struct http_client_host_port *hport;
-       struct http_client_request *const *requests;
-       struct http_client_request *req;
-       unsigned int i, count;
-
-       hport = http_client_host_port_find(host, addr);
-       if (hport == NULL)
-               return NULL;
-
-       requests = array_get(&hport->request_queue, &count);
-       if (count == 0)
-               return NULL;
-       i = 0;
-       if (requests[0]->urgent && no_urgent) {
-               for (; requests[i]->urgent; i++) {
-                       if (i == count)
-                               return NULL;
-               }
-       }
-       req = requests[i];
-       array_delete(&hport->request_queue, i, 1);
-
-       http_client_host_debug(host,
-               "Connection to peer %s claimed request %s %s",
-               http_client_peer_addr2str(addr), http_client_request_label(req),
-               (req->urgent ? "(urgent)" : ""));
-
-       return req;
-}
-
-unsigned int http_client_host_requests_pending(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, unsigned int *num_urgent_r)
-{
-       struct http_client_host_port *hport;
-       struct http_client_request *const *requests;
-       unsigned int count, i;
-
-       *num_urgent_r = 0;
-
-       hport = http_client_host_port_find(host, addr);
-       if (hport == NULL)
-               return 0;
-
-       requests = array_get(&hport->request_queue, &count);
-       for (i = 0; i < count; i++) {
-               if (requests[i]->urgent)
-                       (*num_urgent_r)++;
-               else
-                       break;
-       }
-       return count;
-}
-
-void http_client_host_drop_request(struct http_client_host *host,
-       struct http_client_request *req)
-{
-       struct http_client_host_port *hport;
-       struct http_client_peer_addr addr;
-
-       http_client_request_get_peer_addr(req, &addr);
-
-       hport = http_client_host_port_find(host, &addr);
-       if (hport == NULL)
-               return;
 
-       http_client_host_port_drop_request(hport, req);
+       http_client_queue_connection_setup(queue);
 }
 
 void http_client_host_free(struct http_client_host **_host)
 {
        struct http_client_host *host = *_host;
-       struct http_client_host_port *hport;
+       struct http_client_queue *const *queue_idx;
        struct http_client_request *req, *const *reqp;
        const char *hostname = host->name;
 
@@ -584,10 +217,10 @@ void http_client_host_free(struct http_client_host **_host)
                dns_lookup_abort(&host->dns_lookup);
 
        /* drop request queues */
-       array_foreach_modifiable(&host->ports, hport) {
-               http_client_host_port_deinit(hport);
+       array_foreach(&host->queues, queue_idx) {
+               http_client_queue_free(*queue_idx);
        }
-       array_free(&host->ports);
+       array_free(&host->queues);
 
        while (array_count(&host->delayed_failing_requests) > 0) {
                reqp = array_idx(&host->delayed_failing_requests, 0);
@@ -605,12 +238,15 @@ void http_client_host_free(struct http_client_host **_host)
 
 void http_client_host_switch_ioloop(struct http_client_host *host)
 {
-       struct http_client_request **req;
+       struct http_client_queue *const *queue_idx;
+       struct http_client_request *const *req_idx;
 
        if (host->dns_lookup != NULL)
                dns_lookup_switch_ioloop(host->dns_lookup);
-       array_foreach_modifiable(&host->delayed_failing_requests, req) {
-               (*req)->to_delayed_error =
-                       io_loop_move_timeout(&(*req)->to_delayed_error);
+       array_foreach(&host->queues, queue_idx)
+               http_client_queue_switch_ioloop(*queue_idx);
+       array_foreach(&host->delayed_failing_requests, req_idx) {
+               (*req_idx)->to_delayed_error =
+                       io_loop_move_timeout(&(*req_idx)->to_delayed_error);
        }
 }
index 8b1978f3fb97addd8f49629499776eef02361c9b..03cb7c5b9de1277707bfb4898f1af24518f30ec7 100644 (file)
@@ -114,11 +114,11 @@ static unsigned int
 http_client_peer_requests_pending(struct http_client_peer *peer,
                                  unsigned int *num_urgent_r)
 {
-       struct http_client_host *const *host;
+       struct http_client_queue *const *queue;
        unsigned int num_requests = 0, num_urgent = 0, requests, urgent;
 
-       array_foreach(&peer->hosts, host) {
-               requests = http_client_host_requests_pending(*host, &peer->addr, &urgent);
+       array_foreach(&peer->queues, queue) {
+               requests = http_client_queue_requests_pending(*queue, &urgent);
 
                num_requests += requests;
                num_urgent += urgent;
@@ -356,7 +356,7 @@ http_client_peer_create(struct http_client *client,
        peer->addr = *addr;
        peer->https_name = i_strdup(addr->https_name);
        peer->addr.https_name = peer->https_name;
-       i_array_init(&peer->hosts, 16);
+       i_array_init(&peer->queues, 16);
        i_array_init(&peer->conns, 16);
 
        hash_table_insert
@@ -391,7 +391,7 @@ void http_client_peer_free(struct http_client_peer **_peer)
 
        i_assert(array_count(&peer->conns) == 0);
        array_free(&peer->conns);
-       array_free(&peer->hosts);
+       array_free(&peer->queues);
 
        hash_table_remove
                (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
@@ -415,34 +415,35 @@ http_client_peer_get(struct http_client *client,
        return peer;
 }
 
-bool http_client_peer_have_host(struct http_client_peer *peer,
-                               struct http_client_host *host)
+bool http_client_peer_have_queue(struct http_client_peer *peer,
+                               struct http_client_queue *queue)
 {
-       struct http_client_host *const *host_idx;
+       struct http_client_queue *const *queue_idx;
 
-       array_foreach(&peer->hosts, host_idx) {
-               if (*host_idx == host)
+       array_foreach(&peer->queues, queue_idx) {
+               if (*queue_idx == queue)
                        return TRUE;
        }
        return FALSE;
 }
 
-void http_client_peer_add_host(struct http_client_peer *peer,
-                              struct http_client_host *host)
+void http_client_peer_link_queue(struct http_client_peer *peer,
+                              struct http_client_queue *queue)
 {
-       if (!http_client_peer_have_host(peer, host))
-               array_append(&peer->hosts, &host, 1);
+       if (!http_client_peer_have_queue(peer, queue))
+               array_append(&peer->queues, &queue, 1);
 }
 
-void http_client_peer_remove_host(struct http_client_peer *peer,
-                               struct http_client_host *host)
+void http_client_peer_unlink_queue(struct http_client_peer *peer,
+                               struct http_client_queue *queue)
 {
-       struct http_client_host *const *host_idx;
+       struct http_client_queue *const *queue_idx;
 
-       array_foreach(&peer->hosts, host_idx) {
-               if (*host_idx == host) {
-                       array_delete(&peer->hosts, array_foreach_idx(&peer->hosts, host_idx), 1);
-                       if (array_count(&peer->hosts) == 0)
+       array_foreach(&peer->queues, queue_idx) {
+               if (*queue_idx == queue) {
+                       array_delete(&peer->queues,
+                               array_foreach_idx(&peer->queues, queue_idx), 1);
+                       if (array_count(&peer->queues) == 0)
                                http_client_peer_free(&peer);
                        return;
                }
@@ -452,12 +453,12 @@ void http_client_peer_remove_host(struct http_client_peer *peer,
 struct http_client_request *
 http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent)
 {
-       struct http_client_host *const *host_idx;
+       struct http_client_queue *const *queue_idx;
        struct http_client_request *req;
 
-       array_foreach(&peer->hosts, host_idx) {
-               if ((req=http_client_host_claim_request
-                       (*host_idx, &peer->addr, no_urgent)) != NULL) {
+       array_foreach(&peer->queues, queue_idx) {
+               if ((req=http_client_queue_claim_request
+                       (*queue_idx, &peer->addr, no_urgent)) != NULL) {
                        req->peer = peer;
                        return req;
                }
@@ -468,12 +469,12 @@ http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent)
 
 void http_client_peer_connection_success(struct http_client_peer *peer)
 {
-       struct http_client_host *const *host;
+       struct http_client_queue *const *queue;
 
        peer->last_connect_failed = FALSE;
 
-       array_foreach(&peer->hosts, host) {
-               http_client_host_connection_success(*host, &peer->addr);
+       array_foreach(&peer->queues, queue) {
+               http_client_queue_connection_success(*queue, &peer->addr);
        }
 
        http_client_peer_trigger_request_handler(peer);
@@ -482,7 +483,7 @@ void http_client_peer_connection_success(struct http_client_peer *peer)
 void http_client_peer_connection_failure(struct http_client_peer *peer,
                                         const char *reason)
 {
-       struct http_client_host *const *host;
+       struct http_client_queue *const *queue;
        unsigned int num_urgent;
 
        i_assert(array_count(&peer->conns) > 0);
@@ -500,8 +501,8 @@ void http_client_peer_connection_failure(struct http_client_peer *peer,
                   failed. a second connect will probably also fail, so just
                   try another IP for the hosts(s) or abort all requests if this
                   was the only/last option. */
-               array_foreach(&peer->hosts, host) {
-                       http_client_host_connection_failure(*host, &peer->addr, reason);
+               array_foreach(&peer->queues, queue) {
+                       http_client_queue_connection_failure(*queue, &peer->addr, reason);
                }
        }
        if (array_count(&peer->conns) == 0 &&
index db74ec3dd84ce2db64e4230d559eeb10aa2db312..872cc18db08b8b50a5f395f262ddfe812138ab56 100644 (file)
 enum http_response_payload_type;
 
 struct http_client_host;
-struct http_client_host_port;
+struct http_client_queue;
 struct http_client_peer;
 struct http_client_connection;
 
 ARRAY_DEFINE_TYPE(http_client_host, struct http_client_host *);
-ARRAY_DEFINE_TYPE(http_client_host_port, struct http_client_host_port);
+ARRAY_DEFINE_TYPE(http_client_queue, struct http_client_queue *);
 ARRAY_DEFINE_TYPE(http_client_peer, struct http_client_peer *);
 ARRAY_DEFINE_TYPE(http_client_connection, struct http_client_connection *);
 ARRAY_DEFINE_TYPE(http_client_request, struct http_client_request *);
@@ -59,6 +59,7 @@ struct http_client_request {
 
        struct http_client *client;
        struct http_client_host *host;
+       struct http_client_queue *queue;
        struct http_client_peer *peer;
        struct http_client_connection *conn;
 
@@ -101,8 +102,71 @@ struct http_client_request {
        unsigned int ssl_tunnel:1;
 };
 
-struct http_client_host_port {
+struct http_client_connection {
+       struct connection conn;
+       struct http_client_peer *peer;
+       struct http_client *client;
+       unsigned int refcount;
+
+       const char *label;
+
+       unsigned int id; // DEBUG: identify parallel connections
+       int connect_errno;
+       struct timeval connect_start_timestamp;
+       struct timeval connected_timestamp;
+       struct http_client_request *connect_request;
+
+       struct ssl_iostream *ssl_iostream;
+       struct http_response_parser *http_parser;
+       struct timeout *to_connect, *to_input, *to_idle, *to_response;
+       struct timeout *to_requests;
+
+       struct http_client_request *pending_request;
+       struct istream *incoming_payload;
+       struct io *io_req_payload;
+
+       /* requests that have been sent, waiting for response */
+       ARRAY_TYPE(http_client_request) request_wait_list;
+
+       unsigned int connected:1;           /* connection is connected */
+       unsigned int tunneling:1;          /* last sent request turns this
+                                             connection into tunnel */
+       unsigned int connect_succeeded:1;
+       unsigned int closing:1;
+       unsigned int close_indicated:1;
+       unsigned int output_locked:1;       /* output is locked; no pipelining */
+       unsigned int payload_continue:1;    /* received 100-continue for current
+                                               request */
+};
+
+struct http_client_peer {
+       struct http_client_peer_addr addr;
+       char *https_name;
+
+       struct http_client *client;
+       struct http_client_peer *prev, *next;
+
+       /* queues using this peer */
+       ARRAY_TYPE(http_client_queue) queues;
+
+       /* active connections to this peer */
+       ARRAY_TYPE(http_client_connection) conns;
+
+       /* zero time-out for consolidating request handling */
+       struct timeout *to_req_handling;
+
+       unsigned int destroyed:1;        /* peer is being destroyed */
+       unsigned int no_payload_sync:1;  /* expect: 100-continue failed before */
+       unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */
+       unsigned int last_connect_failed:1;
+       unsigned int allows_pipelining:1;/* peer is known to allow persistent
+                                            connections */
+};
+
+struct http_client_queue {
+       struct http_client *client;
        struct http_client_host *host;
+       char *name;
 
        struct http_client_peer_addr addr;
        char *https_name;
@@ -138,73 +202,12 @@ struct http_client_host {
        ARRAY(struct http_client_request *) delayed_failing_requests;
 
        /* requests are managed on a per-port basis */
-       ARRAY_TYPE(http_client_host_port) ports;
+       ARRAY_TYPE(http_client_queue) queues;
 
        /* active DNS lookup */
        struct dns_lookup *dns_lookup;
 };
 
-struct http_client_peer {
-       struct http_client_peer_addr addr;
-       char *https_name;
-
-       struct http_client *client;
-       struct http_client_peer *prev, *next;
-
-       /* hosts served through this peer */
-       ARRAY_TYPE(http_client_host) hosts;
-
-       /* active connections to this peer */
-       ARRAY_TYPE(http_client_connection) conns;
-
-       /* zero time-out for consolidating request handling */
-       struct timeout *to_req_handling;
-
-       unsigned int destroyed:1;        /* peer is being destroyed */
-       unsigned int no_payload_sync:1;  /* expect: 100-continue failed before */
-       unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */
-       unsigned int last_connect_failed:1;
-       unsigned int allows_pipelining:1;/* peer is known to allow persistent
-                                            connections */
-};
-
-struct http_client_connection {
-       struct connection conn;
-       struct http_client_peer *peer;
-       struct http_client *client;
-       unsigned int refcount;
-
-       const char *label;
-
-       unsigned int id; // DEBUG: identify parallel connections
-       int connect_errno;
-       struct timeval connect_start_timestamp;
-       struct timeval connected_timestamp;
-       struct http_client_request *connect_request;
-
-       struct ssl_iostream *ssl_iostream;
-       struct http_response_parser *http_parser;
-       struct timeout *to_connect, *to_input, *to_idle, *to_response;
-       struct timeout *to_requests;
-
-       struct http_client_request *pending_request;
-       struct istream *incoming_payload;
-       struct io *io_req_payload;
-
-       /* requests that have been sent, waiting for response */
-       ARRAY_TYPE(http_client_request) request_wait_list;
-
-       unsigned int connected:1;           /* connection is connected */
-       unsigned int tunneling:1;          /* last sent request turns this
-                                             connection into tunnel */
-       unsigned int connect_succeeded:1;
-       unsigned int closing:1;
-       unsigned int close_indicated:1;
-       unsigned int output_locked:1;       /* output is locked; no pipelining */
-       unsigned int payload_continue:1;    /* received 100-continue for current
-                                               request */
-};
-
 struct http_client {
        pool_t pool;
 
@@ -222,66 +225,6 @@ struct http_client {
        unsigned int pending_requests;
 };
 
-static inline const char *
-http_client_peer_addr2str(const struct http_client_peer_addr *addr)
-{
-       if (addr->ip.family == AF_INET6)
-               return t_strdup_printf("[%s]:%u", net_ip2addr(&addr->ip), addr->port);
-       return t_strdup_printf("%s:%u", net_ip2addr(&addr->ip), addr->port);
-}
-
-static inline const char *
-http_client_request_label(struct http_client_request *req)
-{
-       if (req->label == NULL) {
-               return t_strdup_printf("[%s %s%s]",
-                       req->method, http_url_create(&req->origin_url), req->target);
-       }
-       return req->label;
-}
-
-static inline void
-http_client_request_get_peer_addr(const struct http_client_request *req,
-       struct http_client_peer_addr *addr)
-{
-       const struct http_url *host_url = req->host_url;
-       
-       memset(addr, 0, sizeof(*addr));
-       if (req->connect_direct) {
-               addr->type = HTTP_CLIENT_PEER_ADDR_RAW;
-               addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
-       } else if (host_url->have_ssl) {
-               if (req->ssl_tunnel)
-                       addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL;
-               else
-                       addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS;
-               addr->https_name = host_url->host_name;
-               addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
-       } else {
-               addr->type = HTTP_CLIENT_PEER_ADDR_HTTP;
-               addr->port = (host_url->have_port ? host_url->port : HTTP_DEFAULT_PORT);
-       }
-}
-
-static inline const char *
-http_client_connection_label(struct http_client_connection *conn)
-{
-       return t_strdup_printf("%s%s [%d]",
-               http_client_peer_addr2str(&conn->peer->addr),
-               (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL ?
-                       " (tunnel)" : ""), conn->id);
-}
-
-static inline const char *
-http_client_peer_label(struct http_client_peer *peer)
-{
-       if (peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) {
-               return t_strconcat
-                       (http_client_peer_addr2str(&peer->addr), " (tunnel)", NULL);
-       }
-       return http_client_peer_addr2str(&peer->addr);
-}
-
 int http_client_init_ssl_ctx(struct http_client *client, const char **error_r);
 
 void http_client_request_ref(struct http_client_request *req);
@@ -337,12 +280,12 @@ struct http_client_peer *
        http_client_peer_get(struct http_client *client,
                const struct http_client_peer_addr *addr);
 void http_client_peer_free(struct http_client_peer **_peer);
-bool http_client_peer_have_host(struct http_client_peer *peer,
-                               struct http_client_host *host);
-void http_client_peer_add_host(struct http_client_peer *peer,
-       struct http_client_host *host);
-void http_client_peer_remove_host(struct http_client_peer *peer,
-                               struct http_client_host *host);
+bool http_client_peer_have_queue(struct http_client_peer *peer,
+                               struct http_client_queue *queue);
+void http_client_peer_link_queue(struct http_client_peer *peer,
+       struct http_client_queue *queue);
+void http_client_peer_unlink_queue(struct http_client_peer *peer,
+                               struct http_client_queue *queue);
 struct http_client_request *
        http_client_peer_claim_request(struct http_client_peer *peer,
                bool no_urgent);
@@ -355,29 +298,112 @@ bool http_client_peer_is_connected(struct http_client_peer *peer);
 unsigned int http_client_peer_idle_connections(struct http_client_peer *peer);
 void http_client_peer_switch_ioloop(struct http_client_peer *peer);
 
+struct http_client_queue *
+http_client_queue_create(struct http_client_host *host,
+       const struct http_client_peer_addr *addr);
+void http_client_queue_free(struct http_client_queue *queue);
+void http_client_queue_fail(struct http_client_queue *queue,
+       unsigned int status, const char *error);
+void http_client_queue_connection_setup(struct http_client_queue *queue);
+void http_client_queue_submit_request(struct http_client_queue *queue,
+       struct http_client_request *req);
+void
+http_client_queue_drop_request(struct http_client_queue *queue,
+       struct http_client_request *req);
+struct http_client_request *
+http_client_queue_claim_request(struct http_client_queue *queue,
+       const struct http_client_peer_addr *addr, bool no_urgent);
+unsigned int
+http_client_queue_requests_pending(struct http_client_queue *queue,
+       unsigned int *num_urgent_r);
+void
+http_client_queue_connection_success(struct http_client_queue *queue,
+                                        const struct http_client_peer_addr *addr);
+bool
+http_client_queue_connection_failure(struct http_client_queue *queue,
+       const struct http_client_peer_addr *addr, const char *reason);
+void http_client_queue_switch_ioloop(struct http_client_queue *queue);
+
 struct http_client_host *
 http_client_host_get(struct http_client *client,
        const struct http_url *host_url);
 void http_client_host_free(struct http_client_host **_host);
 void http_client_host_submit_request(struct http_client_host *host,
        struct http_client_request *req);
-struct http_client_request *
-http_client_host_claim_request(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, bool no_urgent);
-struct http_client_request *
-http_client_host_claim_connect_request(struct http_client_host *host,
-       const struct http_client_peer_addr *addr);
-void http_client_host_connection_success(struct http_client_host *host,
-       const struct http_client_peer_addr *addr);
-void http_client_host_connection_failure(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, const char *reason);
-unsigned int http_client_host_requests_pending(struct http_client_host *host,
-       const struct http_client_peer_addr *addr, unsigned int *num_urgent_r);
-unsigned int
-http_client_host_connect_requests_pending(struct http_client_host *host,
-       const struct http_client_peer_addr *addr);
-void http_client_host_drop_request(struct http_client_host *host,
-       struct http_client_request *req);
 void http_client_host_switch_ioloop(struct http_client_host *host);
 
+static inline const char *
+http_client_peer_addr2str(const struct http_client_peer_addr *addr)
+{
+       if (addr->ip.family == AF_INET6)
+               return t_strdup_printf("[%s]:%u", net_ip2addr(&addr->ip), addr->port);
+       return t_strdup_printf("%s:%u", net_ip2addr(&addr->ip), addr->port);
+}
+
+static inline const char *
+http_client_request_label(struct http_client_request *req)
+{
+       if (req->label == NULL) {
+               return t_strdup_printf("[%s %s%s]",
+                       req->method, http_url_create(&req->origin_url), req->target);
+       }
+       return req->label;
+}
+
+static inline void
+http_client_request_get_peer_addr(const struct http_client_request *req,
+       struct http_client_peer_addr *addr)
+{
+       const struct http_url *host_url = req->host_url;
+       
+       memset(addr, 0, sizeof(*addr));
+       if (req->connect_direct) {
+               addr->type = HTTP_CLIENT_PEER_ADDR_RAW;
+               addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
+       } else if (host_url->have_ssl) {
+               if (req->ssl_tunnel)
+                       addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL;
+               else
+                       addr->type = HTTP_CLIENT_PEER_ADDR_HTTPS;
+               addr->https_name = host_url->host_name;
+               addr->port = (host_url->have_port ? host_url->port : HTTPS_DEFAULT_PORT);
+       } else {
+               addr->type = HTTP_CLIENT_PEER_ADDR_HTTP;
+               addr->port = (host_url->have_port ? host_url->port : HTTP_DEFAULT_PORT);
+       }
+}
+
+static inline const char *
+http_client_connection_label(struct http_client_connection *conn)
+{
+       return t_strdup_printf("%s%s [%d]",
+               http_client_peer_addr2str(&conn->peer->addr),
+               (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL ?
+                       " (tunnel)" : ""), conn->id);
+}
+
+static inline const char *
+http_client_peer_label(struct http_client_peer *peer)
+{
+       if (peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) {
+               return t_strconcat
+                       (http_client_peer_addr2str(&peer->addr), " (tunnel)", NULL);
+       }
+       return http_client_peer_addr2str(&peer->addr);
+}
+
+static inline unsigned int
+http_client_host_get_ip_idx(struct http_client_host *host,
+                           const struct ip_addr *ip)
+{
+       unsigned int i;
+
+       for (i = 0; i < host->ips_count; i++) {
+               if (net_ip_compare(&host->ips[i], ip))
+                       return i;
+       }
+       i_unreached();
+}
+
+
 #endif
diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c
new file mode 100644 (file)
index 0000000..b48903d
--- /dev/null
@@ -0,0 +1,375 @@
+/* Copyright (c) 2013 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "llist.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "dns-lookup.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_queue_debug(struct http_client_queue *queue,
+       const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_queue_debug(struct http_client_queue *queue,
+       const char *format, ...)
+{
+       va_list args;
+
+       if (queue->client->set.debug) {
+
+               va_start(args, format); 
+               i_debug("http-client: queue %s: %s", 
+                       queue->name, t_strdup_vprintf(format, args));
+               va_end(args);
+       }
+}
+
+/*
+ * Queue
+ */
+
+static struct http_client_queue *
+http_client_queue_find(struct http_client_host *host,
+       const struct http_client_peer_addr *addr)
+{
+       struct http_client_queue *const *queue_idx;
+
+       array_foreach_modifiable(&host->queues, queue_idx) {
+               struct http_client_queue *queue = *queue_idx;
+
+               if (queue->addr.type == addr->type && queue->addr.port == addr->port &&
+                   null_strcmp(queue->addr.https_name, addr->https_name) == 0)
+                       return queue;
+       }
+
+       return NULL;
+}
+
+struct http_client_queue *
+http_client_queue_create(struct http_client_host *host,
+       const struct http_client_peer_addr *addr)
+{
+       struct http_client_queue *queue;
+
+       queue = http_client_queue_find(host, addr);
+       if (queue == NULL) {
+               char *name;
+
+               switch (addr->type) {
+               case HTTP_CLIENT_PEER_ADDR_RAW:
+                       name = i_strdup_printf("raw://%s:%u", host->name, addr->port);
+                       break;
+               case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
+               case HTTP_CLIENT_PEER_ADDR_HTTPS:
+                       name = i_strdup_printf("https://%s:%u", host->name, addr->port);
+                       break;
+               case HTTP_CLIENT_PEER_ADDR_HTTP:
+                       name = i_strdup_printf("http://%s:%u", host->name, addr->port);
+                       break;
+               default:
+                       i_unreached();
+               }
+
+               queue = i_new(struct http_client_queue, 1);
+               queue->client = host->client;
+               queue->host = host;
+               queue->addr = *addr;
+               queue->https_name = i_strdup(addr->https_name);
+               queue->addr.https_name = queue->https_name;
+               queue->name = name;
+               queue->ips_connect_idx = 0;
+               i_array_init(&queue->request_queue, 16);
+               array_append(&host->queues, &queue, 1);
+       }
+
+       return queue;
+}
+
+void http_client_queue_free(struct http_client_queue *queue)
+{
+       http_client_queue_fail
+               (queue, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
+       i_free(queue->https_name);
+       if (array_is_created(&queue->pending_peers))
+               array_free(&queue->pending_peers);
+       array_free(&queue->request_queue);
+       if (queue->to_connect != NULL)
+               timeout_remove(&queue->to_connect);
+       i_free(queue->name);
+       i_free(queue);
+}
+
+void http_client_queue_fail(struct http_client_queue *queue,
+       unsigned int status, const char *error)
+{
+       struct http_client_request **req;
+
+       /* abort all pending requests */
+       array_foreach_modifiable(&queue->request_queue, req) {
+               http_client_request_error(*req, status, error);
+       }
+       array_clear(&queue->request_queue);
+}
+
+void
+http_client_queue_drop_request(struct http_client_queue *queue,
+       struct http_client_request *req)
+{
+       ARRAY_TYPE(http_client_request) *req_arr = &queue->request_queue;
+       struct http_client_request **req_idx;
+
+       array_foreach_modifiable(req_arr, req_idx) {
+               if (*req_idx == req) {
+                       array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1);
+                       break;
+               }
+       }
+}
+
+static bool
+http_client_queue_is_last_connect_ip(struct http_client_queue *queue)
+{
+       struct http_client_host *host = queue->host;
+
+       i_assert(queue->ips_connect_idx < host->ips_count);
+       i_assert(queue->ips_connect_start_idx < host->ips_count);
+
+       /* we'll always go through all the IPs. we don't necessarily start
+          connecting from the first IP, so we'll need to treat the IPs as
+          a ring buffer where we automatically wrap back to the first IP
+          when necessary. */
+       return (queue->ips_connect_idx + 1) % host->ips_count ==
+               queue->ips_connect_start_idx;
+}
+
+static void
+http_client_queue_soft_connect_timeout(struct http_client_queue *queue)
+{
+       struct http_client_host *host = queue->host;
+       const struct http_client_peer_addr *addr = &queue->addr;
+
+       if (queue->to_connect != NULL)
+               timeout_remove(&queue->to_connect);
+
+       if (http_client_queue_is_last_connect_ip(queue)) {
+               /* no more IPs to try */
+               return;
+       }
+
+       /* if our our previous connection attempt takes longer than the
+          soft_connect_timeout, we start a connection attempt to the next IP in
+          parallel */
+       http_client_queue_debug(queue, "Connection to %s%s is taking a long time; "
+               "starting parallel connection attempt to next IP",
+               http_client_peer_addr2str(addr), addr->https_name == NULL ? "" :
+                       t_strdup_printf(" (SSL=%s)", addr->https_name)); 
+
+       /* next IP */
+       queue->ips_connect_idx = (queue->ips_connect_idx + 1) % host->ips_count;
+
+       /* setup connection to new peer (can start new soft timeout) */
+       http_client_queue_connection_setup(queue);
+}
+
+void http_client_queue_connection_setup(struct http_client_queue *queue)
+{
+       struct http_client_host *host = queue->host;
+       struct http_client_peer *peer = NULL;
+       const struct http_client_peer_addr *addr = &queue->addr;
+       unsigned int num_requests = array_count(&queue->request_queue);
+
+       if (num_requests == 0)
+               return;
+
+       /* update our peer address */
+       i_assert(queue->ips_connect_idx < host->ips_count);
+       queue->addr.ip = host->ips[queue->ips_connect_idx];
+
+       http_client_queue_debug(queue, "Setting up connection to %s%s "
+               "(%u requests pending)", http_client_peer_addr2str(addr),
+               (addr->https_name == NULL ? "" :
+                       t_strdup_printf(" (SSL=%s)", addr->https_name)), num_requests);
+
+       /* create/get peer */
+       peer = http_client_peer_get(queue->client, addr);
+       http_client_peer_link_queue(peer, queue);
+
+       /* handle requests; creates new connections when needed/possible */
+       http_client_peer_trigger_request_handler(peer);
+
+       if (!http_client_peer_is_connected(peer)) {
+               unsigned int msecs;
+
+               /* not already connected, wait for connections */
+               if (!array_is_created(&queue->pending_peers))
+                       i_array_init(&queue->pending_peers, 8);
+               array_append(&queue->pending_peers, &peer, 1);                  
+
+               /* start soft connect time-out (but only if we have another IP left) */
+               msecs = host->client->set.soft_connect_timeout_msecs;
+               if (!http_client_queue_is_last_connect_ip(queue) && msecs > 0 &&
+                       queue->to_connect == NULL) {
+                       queue->to_connect =
+                               timeout_add(msecs, http_client_queue_soft_connect_timeout, queue);
+               }
+       }
+}
+
+void
+http_client_queue_connection_success(struct http_client_queue *queue,
+                                        const struct http_client_peer_addr *addr)
+{
+       /* we achieved at least one connection the the addr->ip */
+       queue->ips_connect_start_idx =
+               http_client_host_get_ip_idx(queue->host, &addr->ip);
+
+       /* stop soft connect time-out */
+       if (queue->to_connect != NULL)
+               timeout_remove(&queue->to_connect);
+
+       /* drop all other attempts to the hport. note that we get here whenever
+          a connection is successfully created, so pending_peers array
+          may be empty. */
+       if (array_is_created(&queue->pending_peers) &&
+               array_count(&queue->pending_peers) > 0) {
+               struct http_client_peer *const *peer_idx;
+
+               array_foreach(&queue->pending_peers, peer_idx) {
+                       if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
+                               /* don't drop any connections to the successfully
+                                  connected peer, even if some of the connections
+                                  are pending. they may be intended for urgent
+                                  requests. */
+                               continue;
+                       }
+                       /* unlink this queue from the peer; if this was the last/only queue, the
+                          peer will be freed, closing all connections.
+                        */
+                       http_client_peer_unlink_queue(*peer_idx, queue);
+               }
+               array_clear(&queue->pending_peers);
+       }
+}
+
+bool
+http_client_queue_connection_failure(struct http_client_queue *queue,
+       const struct http_client_peer_addr *addr, const char *reason)
+{
+       struct http_client_host *host = queue->host;
+
+       if (array_is_created(&queue->pending_peers) &&
+               array_count(&queue->pending_peers) > 0) {
+               struct http_client_peer *const *peer_idx;
+
+               /* we're still doing the initial connections to this hport. if
+                  we're also doing parallel connections with soft timeouts
+                  (pending_peer_count>1), wait for them to finish
+                  first. */
+               array_foreach(&queue->pending_peers, peer_idx) {
+                       if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
+                               array_delete(&queue->pending_peers,
+                                       array_foreach_idx(&queue->pending_peers, peer_idx), 1);
+                               break;
+                       }
+               }
+               if (array_count(&queue->pending_peers) > 0)
+                       return TRUE;
+       }
+
+       /* one of the connections failed. if we're not using soft timeouts,
+          we need to try to connect to the next IP. if we are using soft
+          timeouts, we've already tried all of the IPs by now. */
+       if (queue->to_connect != NULL)
+               timeout_remove(&queue->to_connect);
+
+       if (http_client_queue_is_last_connect_ip(queue)) {
+               /* all IPs failed, but retry all of them again on the
+                  next request. */
+               queue->ips_connect_idx = queue->ips_connect_start_idx =
+                       (queue->ips_connect_idx + 1) % host->ips_count;
+               http_client_queue_fail(queue,
+                       HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
+               return FALSE;
+       }
+       queue->ips_connect_idx = (queue->ips_connect_idx + 1) % host->ips_count;
+       http_client_queue_connection_setup(queue);
+       return TRUE;
+}
+
+void http_client_queue_submit_request(struct http_client_queue *queue,
+       struct http_client_request *req)
+{
+       req->queue = queue;
+
+       if (req->urgent)
+               array_insert(&queue->request_queue, 0, &req, 1);
+       else
+               array_append(&queue->request_queue, &req, 1);
+}
+
+struct http_client_request *
+http_client_queue_claim_request(struct http_client_queue *queue,
+       const struct http_client_peer_addr *addr, bool no_urgent)
+{
+       struct http_client_request *const *requests;
+       struct http_client_request *req;
+       unsigned int i, count;
+
+       requests = array_get(&queue->request_queue, &count);
+       if (count == 0)
+               return NULL;
+       i = 0;
+       if (requests[0]->urgent && no_urgent) {
+               for (; requests[i]->urgent; i++) {
+                       if (i == count)
+                               return NULL;
+               }
+       }
+       req = requests[i];
+       array_delete(&queue->request_queue, i, 1);
+
+       http_client_queue_debug(queue,
+               "Connection to peer %s claimed request %s %s",
+               http_client_peer_addr2str(addr), http_client_request_label(req),
+               (req->urgent ? "(urgent)" : ""));
+
+       return req;
+}
+
+unsigned int
+http_client_queue_requests_pending(struct http_client_queue *queue,
+       unsigned int *num_urgent_r)
+{
+       struct http_client_request *const *requests;
+       unsigned int count, i;
+
+       *num_urgent_r = 0;
+
+       requests = array_get(&queue->request_queue, &count);
+       for (i = 0; i < count; i++) {
+               if (requests[i]->urgent)
+                       (*num_urgent_r)++;
+               else
+                       break;
+       }
+       return count;
+}
+
+void http_client_queue_switch_ioloop(struct http_client_queue *queue)
+{
+       if (queue->to_connect != NULL)
+               queue->to_connect = io_loop_move_timeout(&queue->to_connect);
+}
index e941168403a058a52c5f86a078ee4fcb94b74d4d..d260a289cbbeabcb5c67e40a32e67d6dec4f4eb4 100644 (file)
@@ -771,8 +771,8 @@ void http_client_request_abort(struct http_client_request **_req)
                return;
        req->callback = NULL;
        req->state = HTTP_REQUEST_STATE_ABORTED;
-       if (req->host != NULL)
-               http_client_host_drop_request(req->host, req);
+       if (req->queue != NULL)
+               http_client_queue_drop_request(req->queue, req);
        http_client_request_unref(_req);
 }
 
@@ -848,6 +848,7 @@ void http_client_request_redirect(struct http_client_request *req,
        }
        
        req->host = NULL;
+       req->queue = NULL;
        req->conn = NULL;
 
        origin_url = http_url_create(&req->origin_url);
@@ -920,7 +921,7 @@ void http_client_request_resubmit(struct http_client_request *req)
        req->conn = NULL;
        req->peer = NULL;
        req->state = HTTP_REQUEST_STATE_QUEUED;
-       http_client_host_submit_request(req->host, req);
+       http_client_queue_submit_request(req->queue, req);
 }
 
 void http_client_request_retry(struct http_client_request *req,
index 39e2d2f007cf8dff6e97bff62b9c71bf732490c6..8dbf7a6ca4e52ab9fa5d22b6977d31f17ec2642a 100644 (file)
 #define HTTP_DEFAULT_PORT 80
 #define HTTPS_DEFAULT_PORT 443
 
-/* FIXME: This implementation not yet finished. The essence works: it is
-   possible to submit requests through the client. Responses are dumped to
-   stdout
-
-    Structure so far:
-    Client - Acts much like a browser; it is not dedicated to a single host.
-             Client can accept requests to different hosts, which can be served
-             at different IPs. Redirects can be handled in the background by
-             making a new connection. Connections to new hosts are created once
-             needed for servicing a request.
-    Requests - Semantics are similar to imapc commands. Create a request, 
-               optionally modify some aspects of it and finally submit it.
-    Hosts - We maintain a 'cache' of hosts for which we have looked up IPs.
-            Requests are first queued in the host struct on a per-port basis.
-    Peers - Group connections to the same ip/port (== peer_addr).
-    Connections - Actual connections to a server. Once a connection is ready to
-                  handle requests, it claims a request from a host object. One
-                  connection hand service multiple hosts and one host can have
-                  multiple associated connections, possibly to different ips and
-                  ports.
-
-  TODO: lots of cleanup, authentication, ssl, timeouts,         rawlog etc.
+/* Structure:
+
+        http-client:
+
+        Acts much like a browser; it is not dedicated to a single host. Client can
+   accept requests to different hosts, which can be served at different IPs.
+   Redirects are handled in the background by making a new connection.
+   Connections to new hosts are created once needed for servicing a request.
+
+        http-client-request:
+
+        The request semantics are similar to imapc commands. Create a request, 
+   optionally modify some aspects of it and finally submit it. Once finished,
+   a callback is called with the returned response.
+
+   http-client-host:
+
+   We maintain a 'cache' of hosts for which we have looked up IPs. One host
+   can have multiple IPs.
+
+   http-client-queue:
+
+   Requests are queued in a queue object. These queues are maintained for each
+   host:port target and listed in the host object. The queue object is
+   responsible for starting connection attempts to TCP port at the various IPs
+   known for the host.
+
+   http-client-peer:
+
+   The peer object groups multiple connections to the same ip/port
+   (== peer_addr).
+
+   http-client-connection:
+
+   This is an actual connection to a server. Once a connection is ready to
+   handle requests, it claims a request from a host object. One connection can
+   service multiple hosts and one host can have multiple associated connections,
+   possibly to different ips and ports.
+
  */
 
 /*