{
va_list args;
- if (conn->client->set.debug) {
-
+ if (conn->debug || conn->ppool->peer->cctx->set.debug) {
va_start(args, format);
i_debug("http-client: conn %s: %s",
conn->label, t_strdup_vprintf(format, args));
http_client_connection_retry_requests(struct http_client_connection *conn,
unsigned int status, const char *error)
{
- const struct http_client_settings *set = &conn->client->set;
+ const struct http_client_settings *set = &conn->peer->client->set;
struct http_client_request *req, **req_idx;
if (!array_is_created(&conn->request_wait_list))
http_client_connection_server_close(struct http_client_connection **_conn)
{
struct http_client_connection *conn = *_conn;
+ struct http_client_peer *peer = conn->peer;
+ struct http_client *client = peer->client;
struct http_client_request *req, **req_idx;
http_client_connection_debug(conn,
}
array_clear(&conn->request_wait_list);
- if (conn->client->ioloop != NULL)
- io_loop_stop(conn->client->ioloop);
+ if (client != NULL && client->ioloop != NULL)
+ io_loop_stop(client->ioloop);
http_client_connection_close(_conn);
}
int http_client_connection_check_ready(struct http_client_connection *conn)
{
+ const struct http_client_settings *set = &conn->peer->client->set;
int ret;
if (conn->in_req_callback) {
if (!conn->connected || conn->output_locked || conn->output_broken ||
conn->close_indicated || conn->tunneling ||
http_client_connection_count_pending(conn) >=
- conn->client->set.max_pipelined_requests)
+ set->max_pipelined_requests)
return 0;
if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) {
return 1;
}
+static void
+http_client_connection_detach_peer(struct http_client_connection *conn)
+{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_connection *const *conn_idx;
+ ARRAY_TYPE(http_client_connection) *conn_arr;
+ bool found = FALSE;
+
+ if (peer == NULL)
+ return;
+
+ http_client_peer_ref(peer);
+ conn_arr = &peer->conns;
+ array_foreach(conn_arr, conn_idx) {
+ if (*conn_idx == conn) {
+ array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
+ found = TRUE;
+ break;
+ }
+ }
+
+ i_assert(found);
+
+ conn->peer = NULL;
+ http_client_connection_debug(conn, "Detached peer");
+
+ if (conn->connect_succeeded)
+ http_client_peer_connection_lost(peer, conn->lost_prematurely);
+ http_client_peer_unref(&peer);
+}
+
static void
http_client_connection_idle_timeout(struct http_client_connection *conn)
{
http_client_connection_close(&conn);
}
+void http_client_connection_lost_peer(struct http_client_connection *conn)
+{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client *client = peer->client;
+ const struct http_client_settings *set = &client->set;
+ struct http_client_peer_pool *ppool = conn->ppool;
+ struct http_client_peer_shared *pshared = ppool->peer;
+ unsigned int timeout, count;
+
+ if (!conn->connected) {
+ http_client_connection_unref(&conn);
+ return;
+ }
+
+ i_assert(!conn->in_req_callback);
+
+ if (conn->to_idle == NULL) {
+ count = array_count(&ppool->conns);
+ i_assert(count > 0);
+
+ /* set timeout for this connection */
+ if (count > http_client_peer_shared_max_connections(pshared)) {
+ /* instant death for (urgent) connections above limit */
+ timeout = 0;
+ } else {
+ unsigned int idle_count = array_count(&ppool->idle_conns);
+
+ /* kill duplicate connections quicker;
+ linearly based on the number of connections */
+ i_assert(count >= idle_count + 1);
+ timeout = (set->max_parallel_connections - idle_count) *
+ (set->max_idle_time_msecs / set->max_parallel_connections);
+ }
+
+ http_client_connection_debug(conn,
+ "Lost peer; going idle (timeout = %u msecs)",
+ timeout);
+
+ conn->to_idle =
+ timeout_add(timeout, http_client_connection_idle_timeout, conn);
+ array_append(&ppool->idle_conns, &conn, 1);
+ } else {
+ http_client_connection_debug(conn,
+ "Lost peer; already idle");
+ }
+
+ http_client_connection_detach_peer(conn);
+}
+
void http_client_connection_check_idle(struct http_client_connection *conn)
{
+ struct http_client_peer *peer;
struct http_client_peer_pool *ppool = conn->ppool;
+ struct http_client *client;
+ const struct http_client_settings *set;
unsigned int timeout, count;
+ peer = conn->peer;
+ if (peer == NULL) {
+ i_assert(conn->to_idle != NULL);
+ return;
+ }
+
+ if (conn->to_idle != NULL) {
+ /* timeout already set */
+ return;
+ }
+
+ client = peer->client;
+ set = &client->set;
+
if (conn->connected &&
array_is_created(&conn->request_wait_list) &&
array_count(&conn->request_wait_list) == 0 &&
!conn->in_req_callback &&
conn->incoming_payload == NULL &&
- conn->client->set.max_idle_time_msecs > 0) {
+ set->max_idle_time_msecs > 0) {
- if (conn->to_idle != NULL) {
- /* timeout already set */
- return;
- }
+ i_assert(peer != NULL);
+ client = peer->client;
- if (conn->client->ioloop != NULL)
- io_loop_stop(conn->client->ioloop);
+ if (client->ioloop != NULL)
+ io_loop_stop(client->ioloop);
- count = array_count(&conn->peer->conns);
+ count = array_count(&peer->conns);
i_assert(count > 0);
/* set timeout for this connection */
- if (count > conn->client->set.max_parallel_connections) {
+ if (count > set->max_parallel_connections) {
/* instant death for (urgent) connections above limit */
timeout = 0;
} else {
/* kill duplicate connections quicker;
linearly based on the number of connections */
i_assert(array_count(&ppool->conns) >= idle_count + 1);
- timeout = (conn->client->set.max_parallel_connections - idle_count) *
- (conn->client->set.max_idle_time_msecs /
- conn->client->set.max_parallel_connections);
+ timeout = (set->max_parallel_connections - idle_count) *
+ (set->max_idle_time_msecs / set->max_parallel_connections);
}
http_client_connection_debug(conn,
timeout_add(timeout, http_client_connection_idle_timeout, conn);
array_append(&ppool->idle_conns, &conn, 1);
-
- } else {
- /* there should be no idle timeout */
- i_assert(conn->to_idle == NULL);
}
}
}
}
-void http_client_connection_use_idle(struct http_client_connection *conn,
+void http_client_connection_claim_idle(struct http_client_connection *conn,
struct http_client_peer *peer)
{
- http_client_connection_debug(conn, "Reusing idle connection");
+ http_client_connection_debug(conn, "Claimed as idle");
i_assert(peer->ppool == conn->ppool);
http_client_connection_stop_idle(conn);
+
+ if (conn->peer == NULL || conn->peer != peer) {
+ http_client_connection_detach_peer(conn);
+
+ conn->peer = peer;
+ conn->debug = peer->client->set.debug;
+ array_append(&peer->conns, &conn, 1);
+ }
}
static void
void http_client_connection_start_request_timeout(
struct http_client_connection *conn)
{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client *client = peer->client;
+ const struct http_client_settings *set = &client->set;
unsigned int timeout_msecs =
- conn->client->set.request_timeout_msecs;
+ set->request_timeout_msecs;
if (conn->pending_request != NULL)
return;
static void
http_client_connection_continue_timeout(struct http_client_connection *conn)
{
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
struct http_client_request *const *wait_reqs;
struct http_client_request *req;
unsigned int wait_count;
i_assert(conn->pending_request == NULL);
timeout_remove(&conn->to_response);
- conn->peer->no_payload_sync = TRUE;
+ pshared->no_payload_sync = TRUE;
http_client_connection_debug(conn,
"Expected 100-continue response timed out; sending payload anyway");
int http_client_connection_next_request(struct http_client_connection *conn)
{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
struct http_client_request *req = NULL;
const char *error;
bool pipelined;
/* claim request, but no urgent request can be second in line */
pipelined = array_count(&conn->request_wait_list) > 0 ||
conn->pending_request != NULL;
- req = http_client_peer_claim_request(conn->peer, pipelined);
+ req = http_client_peer_claim_request(peer, pipelined);
if (req == NULL)
return 0;
http_client_connection_stop_idle(conn);
req->payload_sync_continue = FALSE;
- if (conn->peer->no_payload_sync)
+ if (pshared->no_payload_sync)
req->payload_sync = FALSE;
/* add request to wait list and add a reference */
an HTTP/1.0 intermediary, such a client SHOULD NOT wait for an
indefinite period before sending the message body.
*/
- if (req->payload_sync && !conn->peer->seen_100_response) {
+ if (req->payload_sync && !pshared->seen_100_response) {
i_assert(!pipelined);
i_assert(req->payload_chunked || req->payload_size > 0);
i_assert(conn->to_response == NULL);
struct http_client_request *req,
struct http_response *response)
{
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
struct istream *payload;
bool retrying;
if (conn->incoming_payload == NULL && conn->conn.input != NULL) {
i_assert(conn->conn.io != NULL ||
- conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW);
+ pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW);
return http_client_connection_unref(&conn);
}
http_client_connection_unref(&conn);
{
struct http_client_connection *conn =
(struct http_client_connection *)_conn;
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
struct http_response response;
struct http_client_request *const *reqs;
struct http_client_request *req = NULL, *req_ref;
i_stream_get_name(conn->conn.input),
stream_errno != 0 ?
i_stream_get_error(conn->conn.input) : "EOF");
- http_client_peer_connection_failure(conn->peer, error);
+ http_client_connection_failure(conn, error);
http_client_connection_debug(conn, "%s", error);
http_client_connection_close(&conn);
return;
continue;
}
- conn->peer->no_payload_sync = FALSE;
- conn->peer->seen_100_response = TRUE;
+ pshared->no_payload_sync = FALSE;
+ pshared->seen_100_response = TRUE;
req->payload_sync_continue = TRUE;
http_client_connection_debug(conn,
/* drop Expect: continue */
req->payload_sync = FALSE;
conn->output_locked = FALSE;
- conn->peer->no_payload_sync = TRUE;
+ pshared->no_payload_sync = TRUE;
if (http_client_request_try_retry(req))
handled = TRUE;
/* redirection */
if (finished > 0) {
/* connection still alive after (at least one) request;
we can pipeline -> mark for subsequent connections */
- conn->peer->allows_pipelining = TRUE;
+ pshared->allows_pipelining = TRUE;
/* room for new requests */
- if (http_client_connection_check_ready(conn) > 0)
- http_client_peer_trigger_request_handler(conn->peer);
+ if (peer != NULL &&
+ http_client_connection_check_ready(conn) > 0)
+ http_client_peer_trigger_request_handler(peer);
}
}
static void
http_client_connection_ready(struct http_client_connection *conn)
{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_pool *ppool = conn->ppool;
+ struct http_client_peer_shared *pshared = ppool->peer;
+ ARRAY_TYPE(http_client_connection) *conn_arr;
+ struct http_client_connection *const *conn_idx;
+ const struct http_client_settings *set = &peer->client->set;
+
http_client_connection_debug(conn, "Ready for requests");
/* connected */
conn->last_ioloop = current_ioloop;
timeout_remove(&conn->to_connect);
+ /* remove from pending list */
+ conn_arr = &ppool->pending_conns;
+ array_foreach(conn_arr, conn_idx) {
+ if (*conn_idx == conn) {
+ array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
+ break;
+ }
+ }
+
/* indicate connection success */
conn->connect_succeeded = TRUE;
- http_client_peer_connection_success(conn->peer);
+ http_client_peer_connection_success(peer);
/* start raw log */
- if (conn->client->set.rawlog_dir != NULL) {
- iostream_rawlog_create(conn->client->set.rawlog_dir,
+ if (ppool->rawlog_dir != NULL) {
+ iostream_rawlog_create(ppool->rawlog_dir,
&conn->conn.input, &conn->conn.output);
}
/* direct tunneling connections handle connect requests just by providing a
raw connection */
- if (conn->peer->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) {
+ if (pshared->addr.type == HTTP_CLIENT_PEER_ADDR_RAW) {
struct http_client_request *req;
+
req = http_client_peer_claim_request(conn->peer, FALSE);
if (req != NULL) {
/* start protocol I/O */
conn->http_parser = http_response_parser_init
- (conn->conn.input, &conn->client->set.response_hdr_limits, 0);
+ (conn->conn.input, &set->response_hdr_limits, 0);
o_stream_set_flush_callback(conn->conn.output,
http_client_connection_output, conn);
}
http_client_connection_ssl_handshaked(const char **error_r, void *context)
{
struct http_client_connection *conn = context;
- const char *error, *host = conn->peer->addr.a.tcp.https_name;
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
+ const struct http_client_settings *set = &peer->client->set;
+ const char *error, *host = pshared->addr.a.tcp.https_name;
if (ssl_iostream_check_cert_validity(conn->ssl_iostream, host, &error) == 0)
http_client_connection_debug(conn, "SSL handshake successful");
- else if (conn->client->set.ssl->allow_invalid_cert) {
+ else if (set->ssl->allow_invalid_cert) {
http_client_connection_debug(conn, "SSL handshake successful, "
"ignoring invalid certificate: %s", error);
} else {
http_client_connection_ssl_init(struct http_client_connection *conn,
const char **error_r)
{
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_pool *ppool = conn->ppool;
+ struct http_client_peer_shared *pshared = ppool->peer;
+ const struct http_client_settings *set = &peer->client->set;
struct ssl_iostream_settings ssl_set;
+ struct ssl_iostream_context *ssl_ctx = ppool->ssl_ctx;
const char *error;
- i_assert(conn->client->ssl_ctx != NULL);
+ i_assert(ssl_ctx != NULL);
i_zero(&ssl_set);
- if (!conn->client->set.ssl->allow_invalid_cert) {
+ if (!set->ssl->allow_invalid_cert) {
ssl_set.verbose_invalid_cert = TRUE;
}
- if (conn->client->set.debug)
+ if (set->debug)
http_client_connection_debug(conn, "Starting SSL handshake");
- if (io_stream_create_ssl_client(conn->client->ssl_ctx,
- conn->peer->addr.a.tcp.https_name, &ssl_set,
+ if (io_stream_create_ssl_client(ssl_ctx,
+ pshared->addr.a.tcp.https_name, &ssl_set,
&conn->conn.input, &conn->conn.output,
&conn->ssl_iostream, &error) < 0) {
*error_r = t_strdup_printf(
{
struct http_client_connection *conn =
(struct http_client_connection *)_conn;
- const struct http_client_settings *set = &conn->client->set;
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
+ const struct http_client_settings *set = &peer->client->set;
const char *error;
if (!success) {
set->socket_recv_buffer_size);
}
- if (http_client_peer_addr_is_https(&conn->peer->addr)) {
+ if (http_client_peer_addr_is_https(&pshared->addr)) {
if (http_client_connection_ssl_init(conn, &error) < 0) {
http_client_connection_debug(conn, "%s", error);
http_client_connection_failure(conn, error);
}
static void
-http_client_connection_connect(struct http_client_connection *conn)
+http_client_connection_connect(struct http_client_connection *conn,
+ unsigned int timeout_msecs)
{
- unsigned int msecs;
-
conn->connect_start_timestamp = ioloop_timeval;
if (connection_client_connect(&conn->conn) < 0) {
conn->connect_errno = errno;
/* don't use connection.h timeout because we want this timeout
to include also the SSL handshake */
- msecs = conn->client->set.connect_timeout_msecs;
- if (msecs == 0)
- msecs = conn->client->set.request_timeout_msecs;
- if (msecs > 0) {
- conn->to_connect =
- timeout_add(msecs, http_client_connect_timeout, conn);
+ if (timeout_msecs > 0) {
+ conn->to_connect = timeout_add(timeout_msecs,
+ http_client_connect_timeout, conn);
}
}
static void
http_client_connect_tunnel_timeout(struct http_client_connection *conn)
{
- const char *error, *name = http_client_peer_addr2str(&conn->peer->addr);
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
+ const char *error, *name = http_client_peer_addr2str(&pshared->addr);
unsigned int msecs;
msecs = timeval_diff_msecs(&ioloop_timeval,
name, msecs/1000, msecs%1000);
http_client_connection_debug(conn, "%s", error);
- http_client_peer_connection_failure(conn->peer, error);
+ http_client_connection_failure(conn, error);
http_client_connection_close(&conn);
}
http_client_connection_tunnel_response(const struct http_response *response,
struct http_client_connection *conn)
{
- struct http_client_context *cctx = conn->client->cctx;
+ struct http_client_peer_shared *pshared = conn->ppool->peer;
+ struct http_client_context *cctx = pshared->cctx;
struct http_client_tunnel tunnel;
- const char *name = http_client_peer_addr2str(&conn->peer->addr);
+ const char *name = http_client_peer_addr2str(&pshared->addr);
struct http_client_request *req = conn->connect_request;
conn->connect_request = NULL;
static void
http_client_connection_connect_tunnel(struct http_client_connection *conn,
- const struct ip_addr *ip, in_port_t port)
+ const struct ip_addr *ip, in_port_t port,
+ unsigned int timeout_msecs)
{
- unsigned int msecs;
+ struct http_client *client = conn->peer->client;
conn->connect_start_timestamp = ioloop_timeval;
conn->connect_request = http_client_request_connect_ip
- (conn->client, ip, port, http_client_connection_tunnel_response, conn);
+ (client, ip, port, http_client_connection_tunnel_response, conn);
http_client_request_set_urgent(conn->connect_request);
http_client_request_submit(conn->connect_request);
/* don't use connection.h timeout because we want this timeout
to include also the SSL handshake */
- msecs = conn->client->set.connect_timeout_msecs;
- if (msecs == 0)
- msecs = conn->client->set.request_timeout_msecs;
- if (msecs > 0) {
- conn->to_connect =
- timeout_add(msecs, http_client_connect_tunnel_timeout, conn);
+ if (timeout_msecs > 0) {
+ conn->to_connect = timeout_add(timeout_msecs,
+ http_client_connect_tunnel_timeout, conn);
}
}
struct http_client_connection *
http_client_connection_create(struct http_client_peer *peer)
{
- struct http_client_context *cctx = peer->client->cctx;
+ struct http_client_peer_shared *pshared = peer->shared;
struct http_client_peer_pool *ppool = peer->ppool;
+ struct http_client_context *cctx = pshared->cctx;
+ struct http_client *client = peer->client;
+ const struct http_client_settings *set = &client->set;
struct http_client_connection *conn;
static unsigned int id = 0;
- const struct http_client_peer_addr *addr = &peer->addr;
+ const struct http_client_peer_addr *addr = &pshared->addr;
const char *conn_type = "UNKNOWN";
+ unsigned int timeout_msecs;
- switch (peer->addr.type) {
+ switch (pshared->addr.type) {
case HTTP_CLIENT_PEER_ADDR_HTTP:
conn_type = "HTTP";
break;
break;
}
+ timeout_msecs = set->connect_timeout_msecs;
+ if (timeout_msecs == 0)
+ timeout_msecs = set->request_timeout_msecs;
+
conn = i_new(struct http_client_connection, 1);
conn->refcount = 1;
- conn->client = peer->client;
conn->id = id++;
conn->ppool = ppool;
conn->peer = peer;
- if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW)
+ conn->debug = client->set.debug;
+ if (pshared->addr.type != HTTP_CLIENT_PEER_ADDR_RAW)
i_array_init(&conn->request_wait_list, 16);
conn->io_wait_timer = io_wait_timer_add();
conn->label = i_strdup_printf("%s [%d]",
- http_client_peer_label(peer), conn->id);
+ http_client_peer_shared_label(pshared), conn->id);
- switch (peer->addr.type) {
+ switch (pshared->addr.type) {
case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
http_client_connection_connect_tunnel
- (conn, &addr->a.tcp.ip, addr->a.tcp.port);
+ (conn, &addr->a.tcp.ip, addr->a.tcp.port, timeout_msecs);
break;
case HTTP_CLIENT_PEER_ADDR_UNIX:
connection_init_client_unix(cctx->conn_list, &conn->conn,
addr->a.un.path);
conn->connect_initialized = TRUE;
- http_client_connection_connect(conn);
+ http_client_connection_connect(conn, timeout_msecs);
break;
default:
connection_init_client_ip(cctx->conn_list, &conn->conn,
&addr->a.tcp.ip, addr->a.tcp.port);
conn->connect_initialized = TRUE;
- http_client_connection_connect(conn);
+ http_client_connection_connect(conn, timeout_msecs);
}
- array_append(&peer->conns, &conn, 1);
+ array_append(&ppool->pending_conns, &conn, 1);
array_append(&ppool->conns, &conn, 1);
+ array_append(&peer->conns, &conn, 1);
http_client_peer_pool_ref(ppool);
- http_client_peer_ref(peer);
http_client_connection_debug(conn,
"%s connection created (%d parallel connections exist)%s",
static void
http_client_connection_disconnect(struct http_client_connection *conn)
{
- struct http_client_peer *peer = conn->peer;
struct http_client_peer_pool *ppool = conn->ppool;
ARRAY_TYPE(http_client_connection) *conn_arr;
struct http_client_connection *const *conn_idx;
timeout_remove(&conn->to_requests);
timeout_remove(&conn->to_connect);
timeout_remove(&conn->to_input);
- timeout_remove(&conn->to_idle);
timeout_remove(&conn->to_response);
- /* remove this connection from the list */
+ /* remove this connection from the lists */
conn_arr = &ppool->conns;
array_foreach(conn_arr, conn_idx) {
if (*conn_idx == conn) {
break;
}
}
- conn_arr = &peer->conns;
+ conn_arr = &ppool->pending_conns;
array_foreach(conn_arr, conn_idx) {
if (*conn_idx == conn) {
array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
}
}
- if (conn->connect_succeeded)
- http_client_peer_connection_lost(peer, conn->lost_prematurely);
+ http_client_connection_detach_peer(conn);
http_client_connection_stop_idle(conn); // FIXME: needed?
}
bool http_client_connection_unref(struct http_client_connection **_conn)
{
struct http_client_connection *conn = *_conn;
- struct http_client_peer *peer = conn->peer;
struct http_client_peer_pool *ppool = conn->ppool;
i_assert(conn->refcount > 0);
i_free(conn);
http_client_peer_pool_unref(&ppool);
- http_client_peer_unref(&peer);
return FALSE;
}
http_client_connection_unref(_conn);
}
-void http_client_connection_peer_closed(struct http_client_connection **_conn)
-{
- struct http_client_connection *conn = *_conn;
-
- http_client_connection_debug(conn, "Peer closed");
- http_client_connection_disconnect(conn);
-
- if (http_client_connection_unref(_conn))
- conn->peer = NULL;
-}
-
void http_client_connection_switch_ioloop(struct http_client_connection *conn)
{
if (conn->io_req_payload != NULL)
#include "http-client-private.h"
+static void
+http_client_peer_connect_backoff(struct http_client_peer *peer);
+
+static void
+http_client_peer_shared_connection_success(
+ struct http_client_peer_shared *pshared);
+static void
+http_client_peer_shared_connection_failure(
+ struct http_client_peer_shared *pshared, const char *reason,
+ unsigned int pending);
+static void
+http_client_peer_connection_failed_any(struct http_client_peer *peer,
+ const char *reason);
+
/*
* Logging
*/
http_client_peer_pool_debug(struct http_client_peer_pool *ppool,
const char *format, ...)
{
- struct http_client_peer *peer = ppool->peer;
+ struct http_client_peer_shared *pshared = ppool->peer;
va_list args;
- if (peer->client->cctx->set.debug) {
+ if (pshared->cctx->set.debug) {
va_start(args, format);
i_debug("http-client: peer %s: %s",
- http_client_peer_label(peer), t_strdup_vprintf(format, args));
+ http_client_peer_shared_label(pshared), t_strdup_vprintf(format, args));
+ va_end(args);
+ }
+}
+
+/* Peer (shared) */
+
+static inline void
+http_client_peer_shared_debug(struct http_client_peer_shared *pshared,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_peer_shared_debug(struct http_client_peer_shared *pshared,
+ const char *format, ...)
+{
+ va_list args;
+
+ if (pshared->cctx->set.debug) {
+ va_start(args, format);
+ i_debug("http-client: peer %s (shared): %s",
+ http_client_peer_shared_label(pshared), t_strdup_vprintf(format, args));
va_end(args);
}
}
http_client_peer_debug(struct http_client_peer *peer,
const char *format, ...)
{
+ struct http_client_peer_shared *pshared = peer->shared;
+ struct http_client *client = peer->client;
va_list args;
- if (peer->client->set.debug) {
+ if (client->set.debug) {
va_start(args, format);
i_debug("http-client: peer %s: %s",
- http_client_peer_label(peer), t_strdup_vprintf(format, args));
+ http_client_peer_shared_label(pshared), t_strdup_vprintf(format, args));
va_end(args);
}
}
*/
static struct http_client_peer_pool *
-http_client_peer_pool_create(struct http_client_peer *peer)
+http_client_peer_pool_create(struct http_client_peer_shared *pshared,
+ struct ssl_iostream_context *ssl_ctx, const char *rawlog_dir)
{
struct http_client_peer_pool *ppool;
ppool = i_new(struct http_client_peer_pool, 1);
ppool->refcount = 1;
- ppool->peer = peer;
+ ppool->peer = pshared;
+
+ http_client_peer_shared_ref(pshared);
i_array_init(&ppool->conns, 16);
+ i_array_init(&ppool->pending_conns, 16);
i_array_init(&ppool->idle_conns, 16);
+ DLLIST_PREPEND(&pshared->pools_list, ppool);
+
+ ppool->ssl_ctx = ssl_ctx;
+ ppool->rawlog_dir = i_strdup(rawlog_dir);
+
http_client_peer_pool_debug(ppool, "Peer pool created");
return ppool;
array_foreach_modifiable(&conns, conn)
http_client_connection_unref(conn);
i_assert(array_count(&ppool->idle_conns) == 0);
+ i_assert(array_count(&ppool->pending_conns) == 0);
i_assert(array_count(&ppool->conns) == 0);
http_client_peer_pool_unref(_ppool);
void http_client_peer_pool_unref(struct http_client_peer_pool **_ppool)
{
struct http_client_peer_pool *ppool = *_ppool;
+ struct http_client_peer_shared *pshared = ppool->peer;
*_ppool = NULL;
i_assert(array_count(&ppool->idle_conns) == 0);
i_assert(array_count(&ppool->conns) == 0);
array_free(&ppool->idle_conns);
+ array_free(&ppool->pending_conns);
array_free(&ppool->conns);
+ DLLIST_REMOVE(&pshared->pools_list, ppool);
+
+ i_free(ppool->rawlog_dir);
i_free(ppool);
+ http_client_peer_shared_unref(&pshared);
+}
+
+static struct http_client_peer_pool *
+http_client_peer_pool_get(struct http_client_peer_shared *pshared,
+ struct http_client *client)
+{
+ struct http_client_peer_pool *ppool;
+ struct ssl_iostream_context *ssl_ctx = client->ssl_ctx;
+ const char *rawlog_dir = client->set.rawlog_dir;
+
+ i_assert(!http_client_peer_addr_is_https(&pshared->addr) ||
+ ssl_ctx != NULL);
+
+ ppool = pshared->pools_list;
+ while (ppool != NULL) {
+ if (ppool->ssl_ctx == ssl_ctx &&
+ null_strcmp(ppool->rawlog_dir, rawlog_dir) == 0)
+ break;
+ ppool = ppool->next;
+ }
+
+ if (ppool == NULL) {
+ ppool = http_client_peer_pool_create
+ (pshared, ssl_ctx, rawlog_dir);
+ } else {
+ http_client_peer_pool_debug(ppool, "Peer pool reused");
+ http_client_peer_pool_ref(ppool);
+ }
+
+ return ppool;
+}
+
+static void
+http_client_peer_pool_connection_success(
+ struct http_client_peer_pool *ppool)
+{
+ http_client_peer_pool_debug(ppool,
+ "Successfully connected (connections=%u)",
+ array_count(&ppool->conns));
+
+ http_client_peer_shared_connection_success(ppool->peer);
+}
+
+static void
+http_client_peer_pool_connection_failure(
+ struct http_client_peer_pool *ppool, const char *reason)
+{
+ unsigned int pending;
+
+ /* count number of pending connections */
+ pending = array_count(&ppool->pending_conns);
+ i_assert(pending > 0);
+
+ http_client_peer_pool_debug(ppool,
+ "Failed to make connection "
+ "(connections=%u, connecting=%u)",
+ array_count(&ppool->conns), pending);
+
+ http_client_peer_shared_connection_failure(ppool->peer,
+ reason, pending);
}
/*
- * Peer
+ * Peer (shared)
*/
-static void
-http_client_peer_do_connect(struct http_client_peer *peer,
- unsigned int count);
+static struct http_client_peer_shared *
+http_client_peer_shared_create(struct http_client_context *cctx,
+ const struct http_client_peer_addr *addr)
+{
+ struct http_client_peer_shared *pshared;
+
+ pshared = i_new(struct http_client_peer_shared, 1);
+ pshared->refcount = 1;
+ pshared->cctx = cctx;
+
+ pshared->addr = *addr;
+ switch (addr->type) {
+ case HTTP_CLIENT_PEER_ADDR_RAW:
+ case HTTP_CLIENT_PEER_ADDR_HTTP:
+ i_assert(pshared->addr.a.tcp.ip.family != 0);
+ break;
+ case HTTP_CLIENT_PEER_ADDR_HTTPS:
+ case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
+ i_assert(pshared->addr.a.tcp.ip.family != 0);
+ pshared->addr_name = i_strdup(addr->a.tcp.https_name);
+ pshared->addr.a.tcp.https_name = pshared->addr_name;
+ break;
+ case HTTP_CLIENT_PEER_ADDR_UNIX:
+ pshared->addr_name = i_strdup(addr->a.un.path);
+ pshared->addr.a.un.path = pshared->addr_name;
+ break;
+ default:
+ break;
+ }
+
+ hash_table_insert(cctx->peers,
+ (const struct http_client_peer_addr *)&pshared->addr, pshared);
+ DLLIST_PREPEND(&cctx->peers_list, pshared);
+
+ pshared->backoff_initial_time_msecs =
+ cctx->set.connect_backoff_time_msecs;
+ pshared->backoff_max_time_msecs =
+ cctx->set.connect_backoff_max_time_msecs;
+
+ http_client_peer_shared_debug(pshared, "Peer created");
+ return pshared;
+}
+
+void http_client_peer_shared_ref(struct http_client_peer_shared *pshared)
+{
+ pshared->refcount++;
+}
+
+void http_client_peer_shared_unref(struct http_client_peer_shared **_pshared)
+{
+ struct http_client_peer_shared *pshared = *_pshared;
+
+ *_pshared = NULL;
+
+ i_assert(pshared->refcount > 0);
+ if (--pshared->refcount > 0)
+ return;
+
+ http_client_peer_shared_debug(pshared, "Peer destroy");
+
+ i_assert(pshared->pools_list == NULL);
+
+ /* unlist in client */
+ hash_table_remove(pshared->cctx->peers,
+ (const struct http_client_peer_addr *)&pshared->addr);
+ DLLIST_REMOVE(&pshared->cctx->peers_list, pshared);
+
+ if (pshared->to_backoff != NULL)
+ timeout_remove(&pshared->to_backoff);
+
+ i_free(pshared->addr_name);
+ i_free(pshared->label);
+ i_free(pshared);
+}
+
+static struct http_client_peer_shared *
+http_client_peer_shared_get(struct http_client_context *cctx,
+ const struct http_client_peer_addr *addr)
+{
+ struct http_client_peer_shared *pshared;
+
+ pshared = hash_table_lookup(cctx->peers, addr);
+ if (pshared == NULL) {
+ pshared = http_client_peer_shared_create(cctx, addr);
+ } else {
+ http_client_peer_shared_debug(pshared, "Peer reused");
+ http_client_peer_shared_ref(pshared);
+ }
+
+ return pshared;
+}
+
+void http_client_peer_shared_close(struct http_client_peer_shared **_pshared)
+{
+ struct http_client_peer_shared *pshared = *_pshared;
+ struct http_client_peer_pool *pool, *next;
+
+ http_client_peer_shared_ref(pshared);
+ pool = pshared->pools_list;
+ while (pool != NULL) {
+ next = pool->next;
+ http_client_peer_pool_close(&pool);
+ pool = next;
+ }
+ http_client_peer_shared_unref(_pshared);
+}
const char *
-http_client_peer_label(struct http_client_peer *peer)
+http_client_peer_shared_label(struct http_client_peer_shared *pshared)
{
- if (peer->label == NULL) {
- switch (peer->addr.type) {
+ if (pshared->label == NULL) {
+ switch (pshared->addr.type) {
case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
- peer->label = i_strconcat
- (http_client_peer_addr2str(&peer->addr), " (tunnel)", NULL);
+ pshared->label = i_strconcat
+ (http_client_peer_addr2str(&pshared->addr), " (tunnel)", NULL);
break;
default:
- peer->label = i_strdup
- (http_client_peer_addr2str(&peer->addr));
+ pshared->label = i_strdup
+ (http_client_peer_addr2str(&pshared->addr));
}
}
- return peer->label;
+ return pshared->label;
}
static void
-http_client_peer_connect_backoff(struct http_client_peer *peer)
+http_client_peer_shared_connect_backoff(
+ struct http_client_peer_shared *pshared)
{
- i_assert(peer->to_backoff != NULL);
+ struct http_client_peer *peer;
- http_client_peer_debug(peer,
- "Backoff timer expired");
+ i_assert(pshared->to_backoff != NULL);
- timeout_remove(&peer->to_backoff);
+ http_client_peer_shared_debug(pshared, "Backoff timer expired");
- if (array_count(&peer->queues) == 0) {
- http_client_peer_close(&peer);
- return;
- }
+ timeout_remove(&pshared->to_backoff);
- http_client_peer_do_connect(peer, 1);
+ peer = pshared->peers_list;
+ while (peer != NULL) {
+ http_client_peer_connect_backoff(peer);
+ peer = peer->shared_next;
+ }
}
static bool
-http_client_peer_start_backoff_timer(struct http_client_peer *peer)
+http_client_peer_shared_start_backoff_timer(
+ struct http_client_peer_shared *pshared)
{
- if (peer->to_backoff != NULL)
+ if (pshared->to_backoff != NULL)
return TRUE;
- if (peer->last_failure.tv_sec > 0) {
+ if (pshared->last_failure.tv_sec > 0) {
int backoff_time_spent =
- timeval_diff_msecs(&ioloop_timeval, &peer->last_failure);
-
- if (backoff_time_spent < (int)peer->backoff_time_msecs) {
- http_client_peer_debug(peer,
- "Starting backoff timer for %d msecs",
- peer->backoff_time_msecs - backoff_time_spent);
- peer->to_backoff = timeout_add
- ((unsigned int)(peer->backoff_time_msecs - backoff_time_spent),
- http_client_peer_connect_backoff, peer);
+ timeval_diff_msecs(&ioloop_timeval, &pshared->last_failure);
+
+ if (backoff_time_spent < (int)pshared->backoff_current_time_msecs) {
+ unsigned int new_time = (unsigned int)
+ (pshared->backoff_current_time_msecs - backoff_time_spent);
+ http_client_peer_shared_debug(pshared,
+ "Starting backoff timer for %d msecs", new_time);
+ pshared->to_backoff = timeout_add(new_time,
+ http_client_peer_shared_connect_backoff, pshared);
return TRUE;
}
- http_client_peer_debug(peer,
+ http_client_peer_shared_debug(pshared,
"Backoff time already exceeded by %d msecs",
- backoff_time_spent - peer->backoff_time_msecs);
+ backoff_time_spent - pshared->backoff_current_time_msecs);
}
return FALSE;
}
static void
-http_client_peer_increase_backoff_timer(struct http_client_peer *peer)
+http_client_peer_shared_increase_backoff_timer(
+ struct http_client_peer_shared *pshared)
{
- const struct http_client_settings *set = &peer->client->set;
-
- if (peer->backoff_time_msecs == 0)
- peer->backoff_time_msecs = set->connect_backoff_time_msecs;
+ if (pshared->backoff_current_time_msecs == 0)
+ pshared->backoff_current_time_msecs = pshared->backoff_initial_time_msecs;
else
- peer->backoff_time_msecs *= 2;
- if (peer->backoff_time_msecs > set->connect_backoff_max_time_msecs)
- peer->backoff_time_msecs = set->connect_backoff_max_time_msecs;
+ pshared->backoff_current_time_msecs *= 2;
+ if (pshared->backoff_current_time_msecs >
+ pshared->backoff_max_time_msecs) {
+ pshared->backoff_current_time_msecs = pshared->backoff_max_time_msecs;
+ }
+}
+
+static void
+http_client_peer_shared_reset_backoff_timer(
+ struct http_client_peer_shared *pshared)
+{
+ pshared->backoff_current_time_msecs = 0;
+
+ timeout_remove(&pshared->to_backoff);
+}
+
+static void
+http_client_peer_shared_connection_success(
+ struct http_client_peer_shared *pshared)
+{
+ pshared->last_failure.tv_sec = pshared->last_failure.tv_usec = 0;
+ http_client_peer_shared_reset_backoff_timer(pshared);
+}
+
+static void
+http_client_peer_shared_connection_failure(
+ struct http_client_peer_shared *pshared, const char *reason,
+ unsigned int pending)
+{
+ pshared->last_failure = ioloop_timeval;
+
+ /* manage backoff timer only when this was the only attempt */
+ if (pending == 1)
+ http_client_peer_shared_increase_backoff_timer(pshared);
+
+ if (pending > 1) {
+ /* if there are other connections attempting to connect, wait
+ for them before failing the requests. remember that we had
+ trouble with connecting so in future we don't try to create
+ more than one connection until connects work again. */
+ } else {
+ struct http_client_peer *peer;
+
+ /* this was the only/last connection and connecting to it
+ failed. notify all interested peers about the failure */
+ peer = pshared->peers_list;
+ while (peer != NULL) {
+ struct http_client_peer *peer_next = peer->shared_next;
+ http_client_peer_connection_failed_any(peer, reason);
+ peer = peer_next;
+ }
+ }
}
static void
-http_client_peer_reset_backoff_timer(struct http_client_peer *peer)
+http_client_peer_shared_connection_lost(
+ struct http_client_peer_shared *pshared,
+ bool premature)
{
- peer->backoff_time_msecs = 0;
+ /* update backoff timer if the connection was lost prematurely.
+ this prevents reconnecting immediately to a server that is
+ misbehaving by disconnecting before sending a response.
+ */
+ if (premature) {
+ pshared->last_failure = ioloop_timeval;
+ http_client_peer_shared_increase_backoff_timer(pshared);
+ }
+}
- timeout_remove(&peer->to_backoff);
+void http_client_peer_shared_switch_ioloop(
+ struct http_client_peer_shared *pshared)
+{
+ if (pshared->to_backoff != NULL) {
+ pshared->to_backoff =
+ io_loop_move_timeout(&pshared->to_backoff);
+ }
}
+unsigned int
+http_client_peer_shared_max_connections(
+ struct http_client_peer_shared *pshared)
+{
+ struct http_client_peer *peer;
+ unsigned int max_conns = 0;
+
+ peer = pshared->peers_list;
+ while (peer != NULL) {
+ max_conns += peer->client->set.max_parallel_connections;
+ peer = peer->shared_next;
+ }
+
+ return max_conns;
+}
+
+/*
+ * Peer
+ */
+
+static void
+http_client_peer_drop(struct http_client_peer **_peer);
+
static struct http_client_peer *
http_client_peer_create(struct http_client *client,
- const struct http_client_peer_addr *addr)
+ struct http_client_peer_shared *pshared)
{
struct http_client_peer *peer;
peer = i_new(struct http_client_peer, 1);
peer->refcount = 1;
peer->client = client;
- peer->addr = *addr;
-
- switch (addr->type) {
- case HTTP_CLIENT_PEER_ADDR_RAW:
- case HTTP_CLIENT_PEER_ADDR_HTTP:
- i_assert(peer->addr.a.tcp.ip.family != 0);
- break;
- case HTTP_CLIENT_PEER_ADDR_HTTPS:
- case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL:
- i_assert(peer->addr.a.tcp.ip.family != 0);
- i_assert(client->ssl_ctx != NULL);
- peer->addr_name = i_strdup(addr->a.tcp.https_name);
- peer->addr.a.tcp.https_name = peer->addr_name;
- break;
- case HTTP_CLIENT_PEER_ADDR_UNIX:
- peer->addr_name = i_strdup(addr->a.un.path);
- peer->addr.a.un.path = peer->addr_name;
- break;
- default:
- break;
- }
+ peer->shared = pshared;
i_array_init(&peer->queues, 16);
i_array_init(&peer->conns, 16);
- hash_table_insert
- (client->peers, (const struct http_client_peer_addr *)&peer->addr, peer);
- DLLIST_PREPEND(&client->peers_list, peer);
-
- peer->ppool = http_client_peer_pool_create(peer);
+ DLLIST_PREPEND_FULL
+ (&client->peers_list, peer, client_prev, client_next);
+ DLLIST_PREPEND_FULL
+ (&pshared->peers_list, peer, shared_prev, shared_next);
+ pshared->peers_count++;
+
+ http_client_peer_shared_ref(pshared);
+ peer->ppool = http_client_peer_pool_get(pshared, client);
+
+ /* choose backoff times */
+ if (pshared->peers_list == NULL ||
+ client->set.connect_backoff_time_msecs <
+ pshared->backoff_initial_time_msecs) {
+ pshared->backoff_initial_time_msecs =
+ client->set.connect_backoff_time_msecs;
+ }
+ if (pshared->peers_list == NULL ||
+ client->set.connect_backoff_max_time_msecs >
+ pshared->backoff_max_time_msecs) {
+ pshared->backoff_max_time_msecs =
+ client->set.connect_backoff_max_time_msecs;
+ }
http_client_peer_debug(peer, "Peer created");
return peer;
static void
http_client_peer_disconnect(struct http_client_peer *peer)
{
+ struct http_client_queue *const *queue;
+ struct http_client *client = peer->client;
+ struct http_client_peer_shared *pshared = peer->shared;
struct http_client_connection **conn;
ARRAY_TYPE(http_client_connection) conns;
- struct http_client_queue *const *queue;
if (peer->disconnected)
return;
/* make a copy of the connection array; freed connections modify it */
t_array_init(&conns, array_count(&peer->conns));
- array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
- array_foreach_modifiable(&conns, conn) {
- http_client_connection_peer_closed(conn);
- }
+ array_copy(&conns.arr, 0, &peer->conns.arr,
+ 0, array_count(&peer->conns));
+ array_foreach_modifiable(&conns, conn)
+ http_client_connection_lost_peer(*conn);
i_assert(array_count(&peer->conns) == 0);
if (peer->to_req_handling != NULL)
timeout_remove(&peer->to_req_handling);
- if (peer->to_backoff != NULL)
- timeout_remove(&peer->to_backoff);
/* unlist in client */
- hash_table_remove
- (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
- DLLIST_REMOVE(&peer->client->peers_list, peer);
+ DLLIST_REMOVE_FULL
+ (&client->peers_list, peer, client_prev, client_next);
+ /* unlist in peer */
+ DLLIST_REMOVE_FULL
+ (&pshared->peers_list, peer, shared_prev, shared_next);
+ pshared->peers_count--;
/* unlink all queues */
array_foreach(&peer->queues, queue)
{
struct http_client_peer *peer = *_peer;
struct http_client_peer_pool *ppool = peer->ppool;
-
- i_assert(peer->refcount > 0);
+ struct http_client_peer_shared *pshared = peer->shared;
*_peer = NULL;
+ i_assert(peer->refcount > 0);
if (--peer->refcount > 0)
return TRUE;
http_client_peer_debug(peer, "Peer destroy");
http_client_peer_disconnect(peer);
- http_client_peer_pool_unref(&ppool);
i_assert(array_count(&peer->queues) == 0);
array_free(&peer->conns);
array_free(&peer->queues);
- i_free(peer->addr_name);
- i_free(peer->label);
i_free(peer);
+
+ /* choose new backoff times */
+ peer = pshared->peers_list;
+ while (peer != NULL) {
+ struct http_client *client = peer->client;
+
+ if (client->set.connect_backoff_time_msecs <
+ pshared->backoff_initial_time_msecs) {
+ pshared->backoff_initial_time_msecs =
+ client->set.connect_backoff_time_msecs;
+ }
+ if (client->set.connect_backoff_max_time_msecs >
+ pshared->backoff_max_time_msecs) {
+ pshared->backoff_max_time_msecs =
+ client->set.connect_backoff_max_time_msecs;
+ }
+ peer = peer->shared_next;
+ }
+
+ http_client_peer_pool_unref(&ppool);
+ http_client_peer_shared_unref(&pshared);
return FALSE;
}
(void)http_client_peer_unref(_peer);
}
-static void http_client_peer_drop(struct http_client_peer **_peer)
+static void
+http_client_peer_drop(struct http_client_peer **_peer)
{
struct http_client_peer *peer = *_peer;
+ struct http_client_peer_shared *pshared = peer->shared;
unsigned int conns_active =
http_client_peer_active_connections(peer);
return;
}
- if (peer->to_backoff != NULL)
+ if (pshared->to_backoff != NULL)
return;
- if (http_client_peer_start_backoff_timer(peer)) {
+ if (http_client_peer_shared_start_backoff_timer(pshared)) {
http_client_peer_debug(peer,
"Dropping peer (waiting for backof timeout)");
const struct http_client_peer_addr *addr)
{
struct http_client_peer *peer;
+ struct http_client_peer_shared *pshared;
+
+ pshared = http_client_peer_shared_get(client->cctx, addr);
+
+ peer = pshared->peers_list;
+ while (peer != NULL) {
+ if (peer->client == client)
+ break;
+ peer = peer->shared_next;
+ }
- peer = hash_table_lookup(client->peers, addr);
if (peer == NULL)
- peer = http_client_peer_create(client, addr);
+ peer = http_client_peer_create(client, pshared);
+ http_client_peer_shared_unref(&pshared);
return peer;
}
idle_conns = array_get(&ppool->idle_conns, &idle_count);
for (i = 0; i < count && i < idle_count; i++) {
- http_client_connection_use_idle(idle_conns[i], peer);
+ http_client_connection_claim_idle(idle_conns[i], peer);
http_client_peer_debug(peer,
- "Using idle connection (connections=%u)",
+ "Claimed idle connection (connections=%u)",
array_count(&peer->conns));
}
}
}
+static void
+http_client_peer_connect_backoff(struct http_client_peer *peer)
+{
+ if (peer->connect_backoff &&
+ array_count(&peer->queues) == 0) {
+ http_client_peer_close(&peer);
+ return;
+ }
+
+ http_client_peer_do_connect(peer, 1);
+ peer->connect_backoff = FALSE;
+}
+
static void
http_client_peer_connect(struct http_client_peer *peer, unsigned int count)
{
- if (http_client_peer_start_backoff_timer(peer))
+ peer->connecting = TRUE;
+
+ if (http_client_peer_shared_start_backoff_timer(peer->shared)) {
+ peer->connect_backoff = TRUE;
return;
+ }
http_client_peer_do_connect(peer, count);
}
http_client_peer_debug(peer, "Peer cancel");
+ peer->connecting = FALSE;
+
/* make a copy of the connection array; freed connections modify it */
t_array_init(&conns, array_count(&peer->conns));
array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
struct http_client_connection *const *conn_idx;
ARRAY(struct _conn_available) conns_avail;
struct _conn_available *conn_avail_idx;
+ struct http_client_peer_shared *pshared = peer->shared;
unsigned int connecting, closing, idle;
unsigned int num_pending, num_urgent, new_connections, working_conn_count;
struct http_client_peer *tmp_peer;
/* disconnect pending connections if we're not linked to any queue
anymore */
if (array_count(&peer->queues) == 0) {
- if (array_count(&peer->conns) == 0 && peer->to_backoff == NULL) {
+ if (array_count(&peer->conns) == 0 && pshared->to_backoff == NULL) {
/* peer is completely unused and inactive; drop it immediately */
http_client_peer_drop(&peer);
return;
i_assert(idle == 0);
/* determine how many new connections we can set up */
- if (peer->last_failure.tv_sec > 0 && working_conn_count > 0 &&
+ if (pshared->last_failure.tv_sec > 0 && working_conn_count > 0 &&
working_conn_count == connecting) {
/* don't create new connections until the existing ones have
finished connecting successfully. */
peer->client->set.max_parallel_connections) {
unsigned int pipeline_level = 0, total_handled = 0, handled;
- if (!peer->allows_pipelining) {
+ if (!pshared->allows_pipelining) {
http_client_peer_debug(peer,
"Will not pipeline until peer has shown support");
return;
array_foreach(&peer->queues, queue_idx) {
if ((req=http_client_queue_claim_request
- (*queue_idx, &peer->addr, no_urgent)) != NULL) {
+ (*queue_idx, &peer->shared->addr, no_urgent)) != NULL) {
req->peer = peer;
return req;
}
void http_client_peer_connection_success(struct http_client_peer *peer)
{
+ struct http_client_peer_pool *ppool = peer->ppool;
struct http_client_queue *const *queue;
+ peer->connecting = FALSE;
+
+ http_client_peer_pool_connection_success(ppool);
+
http_client_peer_debug(peer,
"Successfully connected (connections=%u)",
array_count(&peer->conns));
- peer->last_failure.tv_sec = peer->last_failure.tv_usec = 0;
- http_client_peer_reset_backoff_timer(peer);
-
- array_foreach(&peer->queues, queue) {
- http_client_queue_connection_success(*queue, &peer->addr);
- }
+ array_foreach(&peer->queues, queue)
+ http_client_queue_connection_success(*queue, peer);
http_client_peer_trigger_request_handler(peer);
}
-void http_client_peer_connection_failure(struct http_client_peer *peer,
+static void
+http_client_peer_connection_failed_any(struct http_client_peer *peer,
const char *reason)
{
struct http_client_queue *const *queue;
- unsigned int pending;
-
- peer->last_failure = ioloop_timeval;
- /* count number of pending connections */
- pending = http_client_peer_pending_connections(peer);
- i_assert(pending > 0);
+ if (!peer->connecting)
+ return;
+ peer->connecting = FALSE;
http_client_peer_debug(peer,
- "Failed to make connection "
- "(connections=%u, connecting=%u)",
- array_count(&peer->conns), pending);
+ "Connection failed: %s", reason);
- /* manage backoff timer only when this was the only attempt */
- if (pending == 1)
- http_client_peer_increase_backoff_timer(peer);
+ /* failed to make any connection. a second connect will probably also
+ fail, so just try another IP for the hosts(s) or abort all requests
+ if this was the only/last option. */
+ array_foreach(&peer->queues, queue)
+ http_client_queue_connection_failure(*queue, peer, reason);
+}
- if (pending > 1) {
- /* if there are other connections attempting to connect, wait
- for them before failing the requests. remember that we had
- trouble with connecting so in future we don't try to create
- more than one connection until connects work again. */
- } else {
- /* this was the only/last connection and connecting to it
- failed. a second connect will probably also fail, so just
- try another IP for the hosts(s) or abort all requests if this
- was the only/last option. */
- array_foreach(&peer->queues, queue) {
- http_client_queue_connection_failure(*queue, &peer->addr, reason);
- }
- }
+void http_client_peer_connection_failure(struct http_client_peer *peer,
+ const char *reason)
+{
+ struct http_client_peer_pool *ppool = peer->ppool;
+
+ http_client_peer_pool_connection_failure(ppool, reason);
}
void http_client_peer_connection_lost(struct http_client_peer *peer,
unsigned int num_pending, num_urgent;
/* we get here when an already connected connection fails. if the
- connect itself fails, http_client_peer_connection_failure() is
+ connect itself fails, http_client_peer_shared_connection_failure() is
called instead. */
if (peer->disconnected)
return;
+ http_client_peer_shared_connection_lost(peer->shared, premature);
+
num_pending = http_client_peer_requests_pending(peer, &num_urgent);
http_client_peer_debug(peer,
array_count(&peer->queues), array_count(&peer->conns),
num_pending, num_urgent);
- /* update backoff timer if the connection was lost prematurely.
- this prevents reconnecting immediately to a server that is
- misbehaving by disconnecting before sending a response.
- */
- if (premature) {
- peer->last_failure = ioloop_timeval;
- http_client_peer_increase_backoff_timer(peer);
- }
-
if (peer->handling_requests) {
/* we got here from the request handler loop */
http_client_peer_debug(peer,
http_client_peer_trigger_request_handler(peer);
}
-unsigned int
-http_client_peer_idle_connections(struct http_client_peer *peer)
-{
- struct http_client_connection *const *conn_idx;
- unsigned int idle = 0;
-
- /* find idle connections */
- array_foreach(&peer->conns, conn_idx) {
- if (http_client_connection_is_idle(*conn_idx))
- idle++;
- }
-
- return idle;
-}
-
unsigned int
http_client_peer_active_connections(struct http_client_peer *peer)
{
peer->to_req_handling =
io_loop_move_timeout(&peer->to_req_handling);
}
- if (peer->to_backoff != NULL) {
- peer->to_backoff =
- io_loop_move_timeout(&peer->to_backoff);
- }
-}
+ http_client_peer_shared_switch_ioloop(peer->shared);
+}
*/
struct http_client_connection;
+struct http_client_peer_pool;
+struct http_client_peer_shared;
struct http_client_peer;
struct http_client_queue;
struct http_client_host_shared;
struct http_client_connection *);
ARRAY_DEFINE_TYPE(http_client_peer,
struct http_client_peer *);
+ARRAY_DEFINE_TYPE(http_client_peer_shared,
+ struct http_client_peer_shared *);
+ARRAY_DEFINE_TYPE(http_client_peer_pool,
+ struct http_client_peer_pool *);
ARRAY_DEFINE_TYPE(http_client_queue,
struct http_client_queue *);
ARRAY_DEFINE_TYPE(http_client_host,
- struct http_client_host *);
+ struct http_client_host_shared *);
-HASH_TABLE_DEFINE_TYPE(http_client_peer,
+HASH_TABLE_DEFINE_TYPE(http_client_peer_shared,
const struct http_client_peer_addr *,
- struct http_client_peer *);
+ struct http_client_peer_shared *);
HASH_TABLE_DEFINE_TYPE(http_client_host_shared,
const char *, struct http_client_host_shared *);
struct http_client_peer_pool *ppool;
struct http_client_peer *peer;
- struct http_client *client;
char *label;
unsigned int id; // DEBUG: identify parallel connections
bool output_locked:1; /* output is locked; no pipelining */
bool output_broken:1; /* output is broken; no more requests */
bool in_req_callback:1; /* performing request callback (busy) */
+ bool debug:1;
+};
+
+struct http_client_peer_shared {
+ unsigned int refcount;
+ struct http_client_peer_addr addr;
+ char *addr_name;
+
+ char *label;
+
+ struct http_client_context *cctx;
+ struct http_client_peer_shared *prev, *next;
+
+ struct http_client_peer_pool *pools_list;
+
+ struct http_client_peer *peers_list;
+ unsigned int peers_count;
+
+ /* connection retry */
+ struct timeval last_failure;
+ struct timeout *to_backoff;
+ unsigned int backoff_initial_time_msecs;
+ unsigned int backoff_current_time_msecs;
+ unsigned int backoff_max_time_msecs;
+
+ bool no_payload_sync:1; /* expect: 100-continue failed before */
+ bool seen_100_response:1;/* expect: 100-continue succeeded before */
+ bool allows_pipelining:1;/* peer is known to allow persistent
+ connections */
};
struct http_client_peer_pool {
unsigned int refcount;
- struct http_client_peer *peer;
+ struct http_client_peer_shared *peer;
struct http_client_peer_pool *prev, *next;
/* all connections to this peer */
ARRAY_TYPE(http_client_connection) conns;
+ /* pending connections (not ready connecting) */
+ ARRAY_TYPE(http_client_connection) pending_conns;
+
/* available connections to this peer */
ARRAY_TYPE(http_client_connection) idle_conns;
+ /* distinguishing settings for these connections */
+ struct ssl_iostream_context *ssl_ctx;
+ char *rawlog_dir;
+ struct pcap_output *pcap_output;
+
bool destroyed:1; /* peer pool is being destroyed */
};
struct http_client_peer {
unsigned int refcount;
- struct http_client_peer_addr addr;
- char *addr_name;
-
- char *label;
+ struct http_client_peer_shared *shared;
+ struct http_client_peer *shared_prev, *shared_next;
struct http_client *client;
- struct http_client_peer *prev, *next;
+ struct http_client_peer *client_prev, *client_next;
struct http_client_peer_pool *ppool;
/* zero time-out for consolidating request handling */
struct timeout *to_req_handling;
- /* connection retry */
- struct timeval last_failure;
- struct timeout *to_backoff;
- unsigned int backoff_time_msecs;
-
+ bool connecting:1; /* peer is waiting to be connected */
+ bool connect_backoff:1; /* peer is waiting for backoff timout*/
bool disconnected:1; /* peer is already disconnected */
- bool no_payload_sync:1; /* expect: 100-continue failed before */
- bool seen_100_response:1;/* expect: 100-continue succeeded before */
- bool allows_pipelining:1;/* peer is known to allow persistent
- connections */
bool handling_requests:1;/* currently running request handler */
};
struct http_client_queue {
struct http_client *client;
+ struct http_client_queue *prev, *next;
+
struct http_client_host *host;
char *name;
struct timeout *to_failing_requests;
struct http_client_host *hosts_list;
- HASH_TABLE_TYPE(http_client_peer) peers;
struct http_client_peer *peers_list;
+
struct http_client_request *requests_list;
unsigned int requests_count;
};
struct connection_list *conn_list;
+ HASH_TABLE_TYPE(http_client_peer_shared) peers;
+ struct http_client_peer_shared *peers_list;
HASH_TABLE_TYPE(http_client_host_shared) hosts;
struct http_client_host_shared *unix_host;
struct http_client_host_shared *hosts_list;
void http_client_connection_switch_ioloop(struct http_client_connection *conn);
void http_client_connection_start_tunnel(struct http_client_connection **_conn,
struct http_client_tunnel *tunnel);
-void http_client_connection_use_idle(struct http_client_connection *conn,
+void http_client_connection_lost_peer(struct http_client_connection *conn);
+void http_client_connection_claim_idle(struct http_client_connection *conn,
struct http_client_peer *peer);
/*
void http_client_peer_pool_close(struct http_client_peer_pool **_ppool);
-/* peer */
+/* peer (shared) */
const char *
-http_client_peer_label(struct http_client_peer *peer);
+http_client_peer_shared_label(struct http_client_peer_shared *pshared);
+
+void http_client_peer_shared_ref(struct http_client_peer_shared *pshared);
+void http_client_peer_shared_unref(struct http_client_peer_shared **_pshared);
+void http_client_peer_shared_close(struct http_client_peer_shared **_pshared);
+
+void http_client_peer_shared_switch_ioloop(struct http_client_peer_shared *pshared);
+
+unsigned int
+http_client_peer_shared_max_connections(struct http_client_peer_shared *pshared);
+
+/* peer */
struct http_client_peer *
http_client_peer_get(struct http_client *client,
unsigned int
http_client_queue_requests_active(struct http_client_queue *queue);
void http_client_queue_connection_success(struct http_client_queue *queue,
- const struct http_client_peer_addr *addr);
+ struct http_client_peer *peer);
void http_client_queue_connection_failure(struct http_client_queue *queue,
- const struct http_client_peer_addr *addr, const char *reason);
+ struct http_client_peer *peer, const char *reason);
void http_client_queue_peer_disconnected(struct http_client_queue *queue,
struct http_client_peer *peer);
void http_client_queue_switch_ioloop(struct http_client_queue *queue);
if (queue->client->set.debug) {
+ // FIXME: find some other method of distinguishing clients
va_start(args, format);
i_debug("http-client: queue %s: %s",
queue->name, t_strdup_vprintf(format, args));
http_client_queue_debug(queue, "Destroy");
+ /* currently only called when peer is freed, so there is no need to
+ unlink from the peer */
+
/* unlink all peers */
if (queue->cur_peer != NULL) {
struct http_client_peer *peer = queue->cur_peer;
}
if (http_client_host_get_ip_idx(host,
- &queue->cur_peer->addr.a.tcp.ip, &ip_idx)) {
+ &queue->cur_peer->shared->addr.a.tcp.ip, &ip_idx)) {
/* continue with current peer */
queue->ips_connect_idx = queue->ips_connect_start_idx = ip_idx;
} else {
static struct http_client_peer *
http_client_queue_connection_attempt(struct http_client_queue *queue)
{
+ struct http_client *client = queue->client;
struct http_client_host *host = queue->host;
struct http_client_peer *peer;
struct http_client_peer_addr *addr = &queue->addr;
i_assert(array_count(&queue->pending_peers) == 0);
/* is it still the one we want? */
- if (http_client_peer_addr_cmp(addr, &queue->cur_peer->addr) == 0) {
+ if (http_client_peer_addr_cmp
+ (addr, &queue->cur_peer->shared->addr) == 0) {
/* is it still connected? */
if (http_client_peer_is_connected(queue->cur_peer)) {
/* yes */
struct http_client_peer *const *peer_idx;
array_foreach(&queue->pending_peers, peer_idx) {
- i_assert(http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) != 0);
+ i_assert(http_client_peer_addr_cmp
+ (&(*peer_idx)->shared->addr, addr) != 0);
http_client_peer_unlink_queue(*peer_idx, queue);
}
array_clear(&queue->pending_peers);
}
queue->cur_peer = peer;
+ http_client_peer_trigger_request_handler(queue->cur_peer);
+
} else {
struct http_client_peer *const *peer_idx;
unsigned int msecs;
/* we may be waiting for this peer already */
array_foreach(&queue->pending_peers, peer_idx) {
- if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
+ if (http_client_peer_addr_cmp
+ (&(*peer_idx)->shared->addr, addr) == 0) {
i_assert(*peer_idx == peer);
new_peer = FALSE;
break;
/* start soft connect time-out (but only if we have another IP left) */
if (queue->addr.type != HTTP_CLIENT_PEER_ADDR_UNIX) {
- msecs = host->client->set.soft_connect_timeout_msecs;
+ msecs = client->set.soft_connect_timeout_msecs;
if (!http_client_queue_is_last_connect_ip(queue) && msecs > 0 &&
queue->to_connect == NULL) {
queue->to_connect =
void
http_client_queue_connection_success(struct http_client_queue *queue,
- const struct http_client_peer_addr *addr)
+ struct http_client_peer *peer)
{
+ const struct http_client_peer_addr *addr = &peer->shared->addr;
struct http_client_host *host = queue->host;
if (http_client_host_ready(host) &&
struct http_client_peer *const *peer_idx;
array_foreach(&queue->pending_peers, peer_idx) {
- if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
+ if (*peer_idx == peer) {
/* don't drop any connections to the successfully
connected peer, even if some of the connections
are pending. they may be intended for urgent
*/
http_client_peer_unlink_queue(*peer_idx, queue);
}
+
array_clear(&queue->pending_peers);
i_assert(queue->cur_peer != NULL);
}
void
http_client_queue_connection_failure(struct http_client_queue *queue,
- const struct http_client_peer_addr *addr, const char *reason)
+ struct http_client_peer *peer, const char *reason)
{
const struct http_client_settings *set =
&queue->client->set;
+ const struct http_client_peer_addr *addr = &peer->shared->addr;
const char *https_name = http_client_peer_addr_get_https_name(addr);
struct http_client_host *host = queue->host;
unsigned int ips_count = http_client_host_get_ips_count(host);
- struct http_client_peer *failed_peer;
struct http_client_peer *const *peer_idx;
http_client_queue_debug(queue,
reason, array_count(&queue->pending_peers),
array_count(&queue->requests));
- if (queue->cur_peer != NULL) {
- if (http_client_peer_is_connected(queue->cur_peer)) {
- /* The peer still has some working connections, which means that
- pending requests wait until they're picked up by those connections
- or the remaining connections fail as well. In the latter case,
- connecting to different peer can resolve the situation, but only
- if there is more than one IP. In any other case, the requests will
- eventually fail. In the future we could start connections to the next
- IP at this point already, but that is no small change. */
- i_assert(array_count(&queue->pending_peers) == 0);
- return;
- }
+ if (array_count(&queue->pending_peers) == 0) {
+ i_assert(queue->cur_peer == peer);
+ } else {
+ bool found = FALSE;
- failed_peer = queue->cur_peer;
- http_client_peer_unlink_queue(queue->cur_peer, queue);
- queue->cur_peer = NULL;
+ i_assert(queue->cur_peer == NULL);
- } else {
/* we're still doing the initial connections to this hport. if
we're also doing parallel connections with soft timeouts
(pending_peer_count>1), wait for them to finish
first. */
- failed_peer = NULL;
array_foreach(&queue->pending_peers, peer_idx) {
- if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) {
- failed_peer = *peer_idx;
+ if (*peer_idx == peer) {
array_delete(&queue->pending_peers,
array_foreach_idx(&queue->pending_peers, peer_idx), 1);
+ found = TRUE;
break;
}
}
- i_assert(failed_peer != NULL);
+ i_assert(found);
if (array_count(&queue->pending_peers) > 0) {
http_client_queue_debug(queue,
"Waiting for remaining pending peers.");
- http_client_peer_unlink_queue(failed_peer, queue);
+ http_client_peer_unlink_queue(peer, queue);
return;
}
- }
- /* one of the connections failed. if we're not using soft timeouts,
- we need to try to connect to the next IP. if we are using soft
- timeouts, we've already tried all of the IPs by now. */
- timeout_remove(&queue->to_connect);
+ /* one of the connections failed. if we're not using soft timeouts,
+ we need to try to connect to the next IP. if we are using soft
+ timeouts, we've already tried all of the IPs by now. */
+ timeout_remove(&queue->to_connect);
- if (queue->addr.type == HTTP_CLIENT_PEER_ADDR_UNIX) {
- http_client_queue_fail(queue,
- HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
- return;
+ if (queue->addr.type == HTTP_CLIENT_PEER_ADDR_UNIX) {
+ http_client_peer_unlink_queue(peer, queue);
+ http_client_queue_fail(queue,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
+ return;
+ }
}
if (http_client_queue_is_last_connect_ip(queue)) {
queue->ips_connect_idx = queue->ips_connect_start_idx =
(queue->ips_connect_idx + 1) % ips_count;
- if (set->max_connect_attempts == 0 ||
- queue->connect_attempts >= set->max_connect_attempts) {
+ if (queue->cur_peer == NULL && (set->max_connect_attempts == 0 ||
+ queue->connect_attempts >= set->max_connect_attempts)) {
http_client_queue_debug(queue,
"Failed to set up any connection; failing all queued requests");
if (queue->connect_attempts > 1) {
total_msecs/1000, total_msecs%1000);
}
queue->connect_attempts = 0;
- http_client_peer_unlink_queue(failed_peer, queue);
+ http_client_peer_unlink_queue(peer, queue);
http_client_queue_fail(queue,
HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);
return;
queue->ips_connect_idx = (queue->ips_connect_idx + 1) % ips_count;
}
- if (http_client_queue_connection_attempt(queue) != failed_peer)
- http_client_peer_unlink_queue(failed_peer, queue);
+ if (http_client_queue_connection_attempt(queue) != peer) {
+ http_client_peer_unlink_queue(peer, queue);
+ queue->cur_peer = NULL;
+ }
return;
}
req->submit_time = ioloop_timeval;
http_client_request_do_submit(req);
- http_client_request_debug(req, "Submitted");
req->submitted = TRUE;
http_client_request_add(req);
+
+ http_client_request_debug(req, "Submitted (requests left=%d)",
+ req->client->requests_count);
}
void
p_strdup(pool, set->proxy_url->password);
}
- if (set->max_idle_time_msecs > 0)
- client->set.max_idle_time_msecs = set->max_idle_time_msecs;
if (set->max_parallel_connections > 0)
client->set.max_parallel_connections = set->max_parallel_connections;
if (set->max_pipelined_requests > 0)
client->set.no_ssl_tunnel || set->no_ssl_tunnel;
if (set->max_redirects > 0)
client->set.max_redirects = set->max_redirects;
- client->set.response_hdr_limits = set->response_hdr_limits;
if (set->request_absolute_timeout_msecs > 0) {
client->set.request_absolute_timeout_msecs =
set->request_absolute_timeout_msecs;
client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
if (set->max_auto_retry_delay > 0)
client->set.max_auto_retry_delay = set->max_auto_retry_delay;
- client->set.socket_send_buffer_size = set->socket_send_buffer_size;
- client->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
client->set.debug = client->set.debug || set->debug;
}
i_array_init(&client->delayed_failing_requests, 1);
- hash_table_create(&client->peers, default_pool, 0,
- http_client_peer_addr_hash, http_client_peer_addr_cmp);
-
return client;
}
peer = client->peers_list;
http_client_peer_close(&peer);
}
- hash_table_destroy(&client->peers);
/* free hosts */
while (client->hosts_list != NULL) {
void http_client_switch_ioloop(struct http_client *client)
{
- struct http_client_host *host;
struct http_client_peer *peer;
+ struct http_client_host *host;
/* move peers */
- for (peer = client->peers_list; peer != NULL; peer = peer->next)
+ for (peer = client->peers_list; peer != NULL;
+ peer = peer->client_next)
http_client_peer_switch_ioloop(peer);
/* move hosts/queues */
hash_table_create(&cctx->hosts, default_pool, 0, str_hash, strcmp);
+ hash_table_create(&cctx->peers, default_pool, 0,
+ http_client_peer_addr_hash, http_client_peer_addr_cmp);
+
return cctx;
}
void http_client_context_unref(struct http_client_context **_cctx)
{
struct http_client_context *cctx = *_cctx;
+ struct http_client_peer_shared *peer;
struct http_client_host_shared *hshared;
*_cctx = NULL;
}
hash_table_destroy(&cctx->hosts);
+ /* close all idle connections */
+ while (cctx->peers_list != NULL) {
+ peer = cctx->peers_list;
+ http_client_peer_shared_close(&peer);
+ i_assert(peer == NULL);
+ }
+ hash_table_destroy(&cctx->peers);
+
connection_list_deinit(&cctx->conn_list);
pool_unref(&cctx->pool);
projects / thirdparty / dovecot / core.git / commitdiff
