libhttp_la_SOURCES = \
http-date.c \
- http-url.c
+ http-url.c \
+ http-parser.c \
+ http-header-parser.c \
+ http-transfer-chunked.c \
+ http-response-parser.c \
+ http-client-request.c \
+ http-client-connection.c \
+ http-client-peer.c \
+ http-client-host.c \
+ http-client.c
headers = \
http-date.h \
- http-url.h
+ http-url.h \
+ http-parser.h \
+ http-header-parser.h \
+ http-transfer.h \
+ http-response-parser.h \
+ http-client-private.h \
+ http-client.h
pkginc_libdir=$(pkgincludedir)
pkginc_lib_HEADERS = $(headers)
test_programs = \
test-http-date \
- test-http-url
+ test-http-url \
+ test-http-header-parser \
+ test-http-transfer \
+ test-http-response-parser \
+ test-http-client
-noinst_PROGRAMS = $(test_programs)
+noinst_PROGRAMS = $(test_programs) test-http-responses
test_libs = \
../lib-test/libtest.la \
- ../lib/liblib.la
+ ../lib/liblib.la \
+ ../lib/safe-memset.lo \
+ $(MODULE_LIBS)
-test_deps = $(noinst_LTLIBRARIES) $(test_libs)
+test_deps = \
+ $(noinst_LTLIBRARIES)
+ ../lib-test/libtest.la \
+ ../lib/liblib.la \
+ ../lib/safe-memset.lo
test_http_url_SOURCES = test-http-url.c
test_http_url_LDADD = http-url.lo $(test_libs)
test_http_date_LDADD = http-date.lo $(test_libs)
test_http_date_DEPENDENCIES = $(test_deps)
+test_http_header_parser_SOURCES = test-http-header-parser.c
+test_http_header_parser_LDADD = http-parser.lo http-header-parser.lo $(test_libs)
+test_http_header_parser_DEPENDENCIES = $(test_deps)
+
+test_http_transfer_SOURCES = test-http-transfer.c
+test_http_transfer_LDADD = \
+ http-parser.lo \
+ http-header-parser.lo \
+ http-transfer-chunked.lo \
+ $(test_libs)
+test_http_transfer_DEPENDENCIES = $(test_deps)
+
+test_http_response_parser_SOURCES = test-http-response-parser.c
+test_http_response_parser_LDADD = \
+ http-date.lo \
+ http-parser.lo \
+ http-header-parser.lo \
+ http-transfer-chunked.lo \
+ http-response-parser.lo \
+ $(test_libs)
+test_http_response_parser_DEPENDENCIES = $(test_deps)
+
+test_http_client_SOURCES = test-http-client.c
+test_http_client_LDFLAGS = -export-dynamic
+test_http_client_LDADD = \
+ libhttp.la \
+ ../lib-dns/libdns.la \
+ ../lib-ssl-iostream/libssl_iostream.la \
+ $(test_libs)
+test_http_client_DEPENDENCIES = \
+ libhttp.la \
+ ../lib-dns/libdns.la \
+ ../lib-ssl-iostream/libssl_iostream.la \
+ $(test_deps)
+
+test_http_responses_SOURCES = test-http-responses.c
+test_http_responses_LDADD = \
+ http-date.lo \
+ http-parser.lo \
+ http-header-parser.lo \
+ http-transfer-chunked.lo \
+ http-response-parser.lo \
+ ../lib/liblib.la \
+ ../lib/safe-memset.lo \
+ $(MODULE_LIBS)
+test_http_responses_DEPENDENCIES = \
+ http-date.lo \
+ http-parser.lo \
+ http-header-parser.lo \
+ http-transfer-chunked.lo \
+ http-response-parser.lo \
+ $(noinst_LTLIBRARIES) \
+ ../lib/liblib.la \
+ ../lib/safe-memset.lo
+
check: check-am check-test
check-test: all-am
for bin in $(test_programs); do \
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-rawlog.h"
+#include "iostream-ssl.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_connection_debug(struct http_client_connection *conn,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+static inline void
+http_client_connection_error(struct http_client_connection *conn,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_connection_debug(struct http_client_connection *conn,
+ const char *format, ...)
+{
+ va_list args;
+
+ if (conn->client->set.debug) {
+
+ va_start(args, format);
+ i_debug("http-client: conn %s: %s",
+ http_client_connection_label(conn), t_strdup_vprintf(format, args));
+ va_end(args);
+ }
+}
+
+static inline void
+http_client_connection_error(struct http_client_connection *conn,
+ const char *format, ...)
+{
+ va_list args;
+
+ va_start(args, format);
+ i_error("http-client: conn %s: %s",
+ http_client_connection_label(conn), t_strdup_vprintf(format, args));
+ va_end(args);
+}
+
+
+/*
+ * Connection
+ */
+
+static void http_client_connection_input(struct connection *_conn);
+
+bool http_client_connection_is_ready(struct http_client_connection *conn)
+{
+ return (conn->connected && !conn->output_locked &&
+ array_count(&conn->request_wait_list) <
+ conn->client->set.max_pipelined_requests);
+}
+
+static void
+http_client_connection_retry_requests(struct http_client_connection *conn,
+ unsigned int status, const char *error)
+{
+ struct http_client_request **req;
+
+ array_foreach_modifiable(&conn->request_wait_list, req) {
+ http_client_request_retry(*req, status, error);
+ }
+ array_clear(&conn->request_wait_list);
+}
+
+static void
+http_client_connection_server_close(struct http_client_connection **_conn)
+{
+ struct http_client_connection *conn = *_conn;
+ struct http_client_request **req;
+
+ conn->connected = FALSE;
+ conn->closing = TRUE;
+
+ http_client_connection_debug(conn,
+ "Server explicitly closed connection");
+
+ array_foreach_modifiable(&conn->request_wait_list, req) {
+ http_client_request_resubmit(*req);
+ }
+ array_clear(&conn->request_wait_list);
+
+ if (conn->client->ioloop != NULL)
+ io_loop_stop(conn->client->ioloop);
+
+ http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_abort_temp_error(struct http_client_connection **_conn,
+ unsigned int status, const char *error)
+{
+ struct http_client_connection *conn = *_conn;
+
+ conn->connected = FALSE;
+ conn->closing = TRUE;
+
+ http_client_connection_retry_requests(conn, status, error);
+ http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_abort_error(struct http_client_connection **_conn,
+ unsigned int status, const char *error)
+{
+ struct http_client_connection *conn = *_conn;
+ struct http_client_request **req;
+
+ conn->connected = FALSE;
+ conn->closing = TRUE;
+
+ array_foreach_modifiable(&conn->request_wait_list, req) {
+ http_client_request_error(*req, status, error);
+ }
+ array_clear(&conn->request_wait_list);
+ http_client_connection_free(_conn);
+}
+
+static void
+http_client_connection_idle_timeout(struct http_client_connection *conn)
+{
+ http_client_connection_debug(conn, "Idle connection timed out");
+
+ http_client_connection_free(&conn);
+}
+
+static void
+http_client_connection_check_idle(struct http_client_connection *conn)
+{
+ unsigned int timeout, count;
+
+ if (array_count(&conn->request_wait_list) == 0 &&
+ conn->incoming_payload == NULL &&
+ conn->client->set.max_idle_time_msecs > 0) {
+
+ if (conn->to_idle != NULL) {
+ /* timeout already set */
+ return;
+ }
+
+ http_client_connection_debug(conn,
+ "No more requests queued; going idle");
+
+ if (conn->client->ioloop != NULL)
+ io_loop_stop(conn->client->ioloop);
+
+ count = array_count(&conn->peer->conns);
+ i_assert(count > 0);
+
+ /* set timeout for this connection */
+ if (count > conn->client->set.max_parallel_connections) {
+ /* instant death for (urgent) connections above limit */
+ timeout = 0;
+ } else {
+ /* kill duplicate connections quicker;
+ linearly based on the number of connections */
+ timeout = (conn->client->set.max_parallel_connections - count) *
+ (conn->client->set.max_idle_time_msecs /
+ conn->client->set.max_parallel_connections);
+ }
+
+ conn->to_idle =
+ timeout_add(timeout, http_client_connection_idle_timeout, conn);
+
+ } else {
+ /* there should be no idle timeout */
+ i_assert(conn->to_idle == NULL);
+ }
+}
+
+static void
+http_client_connection_continue_timeout(struct http_client_connection *conn)
+{
+ struct http_client_request *const *req_idx;
+ struct http_client_request *req;
+
+ if (conn->to_response != NULL)
+ timeout_remove(&conn->to_response);
+ conn->peer->no_payload_sync = TRUE;
+
+ http_client_connection_debug(conn,
+ "Expected 100-continue response timed out; sending payload anyway");
+
+ req_idx = array_idx(&conn->request_wait_list, 0);
+ req = req_idx[0];
+
+ // FIXME: we should mark this in the peer object for next requests
+ conn->payload_continue = TRUE;
+ if (http_client_request_send_more(req) < 0) {
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Failed to send request");
+ }
+}
+
+bool http_client_connection_next_request(struct http_client_connection *conn)
+{
+ struct http_client_request *req = NULL;
+
+ if (!http_client_connection_is_ready(conn)) {
+
+ http_client_connection_debug(conn, "Not ready for next request");
+ return FALSE;
+ }
+
+ /* claim request, but no urgent request can be second in line */
+ req = http_client_peer_claim_request(conn->peer,
+ array_count(&conn->request_wait_list) > 0);
+ if (req == NULL) {
+ http_client_connection_check_idle(conn);
+ return FALSE;
+ }
+
+ if (conn->to_idle != NULL)
+ timeout_remove(&conn->to_idle);
+
+ req->conn = conn;
+ conn->payload_continue = FALSE;
+ if (conn->peer->no_payload_sync)
+ req->payload_sync = FALSE;
+ array_append(&conn->request_wait_list, &req, 1);
+ http_client_request_ref(req);
+
+ if (http_client_request_send(req) < 0) {
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+ "Failed to send request");
+ return FALSE;
+ }
+
+ /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21;
+ Section 6.1.2.1:
+
+ Because of the presence of older implementations, the protocol allows
+ ambiguous situations in which a client might send "Expect: 100-continue"
+ without receiving either a 417 (Expectation Failed) or a 100 (Continue)
+ status code. Therefore, when a client sends this header field to an
+ origin server (possibly via a proxy) from which it has never seen a 100
+ (Continue) status code, the client SHOULD NOT wait for an indefinite
+ period before sending the payload body.
+ */
+ if (req->payload_sync) {
+ i_assert(req->input_size > 0);
+ i_assert(conn->to_response == NULL);
+ conn->to_response = timeout_add(HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS,
+ http_client_connection_continue_timeout, conn);
+ }
+
+ return TRUE;
+}
+
+static void http_client_connection_destroy(struct connection *_conn)
+{
+ struct http_client_connection *conn =
+ (struct http_client_connection *)_conn;
+
+ conn->closing = TRUE;
+ conn->connected = FALSE;
+
+ switch (_conn->disconnect_reason) {
+ case CONNECTION_DISCONNECT_CONNECT_TIMEOUT:
+ http_client_peer_connection_failure(conn->peer);
+ break;
+ case CONNECTION_DISCONNECT_CONN_CLOSED:
+ /* retry pending requests if possible */
+ http_client_connection_retry_requests(conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+ default:
+ break;
+ }
+
+ http_client_connection_free(&conn);
+}
+
+static void http_client_payload_finished(struct http_client_connection *conn)
+{
+ timeout_remove(&conn->to_input);
+ conn->conn.io = io_add(conn->conn.fd_in, IO_READ,
+ http_client_connection_input, &conn->conn);
+}
+
+static void http_client_payload_destroyed(struct http_client_connection *conn)
+{
+ i_assert(conn->incoming_payload != NULL);
+ i_assert(conn->pending_request != NULL);
+ i_assert(conn->conn.io == NULL);
+
+ http_client_connection_debug(conn, "Response payload stream destroyed");
+
+ /* caller is allowed to change the socket fd to blocking while reading
+ the payload. make sure here that it's switched back. */
+ net_set_nonblock(conn->conn.fd_in, TRUE);
+
+ conn->incoming_payload = NULL;
+
+ http_client_request_finish(&conn->pending_request);
+ conn->pending_request = NULL;
+
+ if (conn->close_indicated) {
+ http_client_connection_server_close(&conn);
+ return;
+ }
+
+ /* input stream may have pending input. make sure input handler
+ gets called (but don't do it directly, since we get get here
+ somewhere from the API user's code, which we can't really know what
+ state it is in). this call also triggers sending a new request if
+ necessary. */
+ conn->to_input =
+ timeout_add_short(0, http_client_connection_input, &conn->conn);
+}
+
+static bool
+http_client_connection_return_response(struct http_client_connection *conn,
+ struct http_client_request *req, struct http_response *response)
+{
+ struct istream *payload;
+
+ i_assert(conn->incoming_payload == NULL);
+ i_assert(conn->pending_request == NULL);
+
+ req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
+
+ if (response->payload != NULL) {
+ /* wrap the stream to capture the destroy event without destroying the
+ actual payload stream. */
+ conn->incoming_payload = response->payload =
+ i_stream_create_limit(response->payload, (uoff_t)-1);
+ i_stream_set_destroy_callback(response->payload,
+ http_client_payload_destroyed,
+ conn);
+ /* the callback may add its own I/O, so we need to remove
+ our one before calling it */
+ io_remove(&conn->conn.io);
+ }
+
+ http_client_request_callback(req, response);
+
+ // FIXME: conn may be freed at this point..
+
+ if (response->payload != NULL) {
+ req->state = HTTP_REQUEST_STATE_PAYLOAD_IN;
+ payload = response->payload;
+ response->payload = NULL;
+ conn->pending_request = req;
+ i_stream_unref(&payload);
+ if (conn->to_input != NULL) {
+ /* already finished reading the payload */
+ http_client_payload_finished(conn);
+ }
+ } else {
+ http_client_request_finish(&req);
+ }
+
+ if (conn->incoming_payload == NULL) {
+ i_assert(conn->conn.io != NULL);
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static void http_client_connection_input(struct connection *_conn)
+{
+ struct http_client_connection *conn =
+ (struct http_client_connection *)_conn;
+ struct http_response *response;
+ struct http_client_request *const *req_idx;
+ struct http_client_request *req = NULL;
+ int finished = 0, ret;
+ const char *error;
+ bool no_payload = FALSE;
+
+ i_assert(conn->incoming_payload == NULL);
+
+ if (conn->to_input != NULL) {
+ /* We came here from a timeout added by
+ http_client_payload_destroyed(). The IO couldn't be added
+ back immediately in there, because the HTTP API user may
+ still have had its own IO pointed to the same fd. It should
+ be removed by now, so we can add it back. */
+ http_client_payload_finished(conn);
+ finished++;
+ }
+
+ /* get first waiting request */
+ if (array_count(&conn->request_wait_list) > 0) {
+ req_idx = array_idx(&conn->request_wait_list, 0);
+ req = req_idx[0];
+
+ /* https://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+ Section 3.3.2:
+
+ A server MAY send a Content-Length header field in a response to a
+ HEAD request [...]
+ */
+ no_payload = (strcmp(req->method, "HEAD") == 0);
+ }
+
+ // FIXME: handle somehow if server replies before request->input is at EOF
+ while ((ret=http_response_parse_next
+ (conn->http_parser, no_payload, &response, &error)) > 0) {
+ bool aborted;
+
+ if (req == NULL) {
+ /* server sent response without any requests in the wait list */
+ http_client_connection_error(conn, "Got unexpected input from server");
+ http_client_connection_free(&conn);
+ return;
+ }
+
+ /* https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-21
+ Section 7.2:
+
+ A client MUST be prepared to accept one or more 1xx status responses
+ prior to a regular response, even if the client does not expect a 100
+ (Continue) status message. Unexpected 1xx status responses MAY be
+ ignored by a user agent.
+ */
+ if (req->payload_sync && response->status == 100) {
+ conn->payload_continue = TRUE;
+ if (conn->to_response != NULL)
+ timeout_remove(&conn->to_response);
+ http_client_connection_debug(conn,
+ "Got expected 100-continue response");
+ if (http_client_request_send_more(req) < 0) {
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+ "Failed to send request");
+ }
+ return;
+ } else if (response->status / 100 == 1) {
+ /* ignore them for now */
+ http_client_connection_debug(conn,
+ "Got unexpected %u response; ignoring", response->status);
+ continue;
+ } // FIXME: handle 417 Expectation Failed
+
+ http_client_connection_debug(conn,
+ "Got %u response for request %s",
+ response->status, http_client_request_label(req));
+
+ /* remove request from queue */
+ array_delete(&conn->request_wait_list, 0, 1);
+ aborted = (req->state == HTTP_REQUEST_STATE_ABORTED);
+ http_client_request_unref(&req);
+
+ conn->close_indicated = response->connection_close;
+
+ if (!aborted) {
+ if (response->status / 100 == 3) {
+ /* redirect */
+ http_client_request_redirect(req, response->location);
+ } else {
+ /* response for application */
+ if (!http_client_connection_return_response(conn, req, response))
+ return;
+ }
+
+ finished++;
+ }
+
+ /* server closing connection? */
+ if (conn->close_indicated) {
+ http_client_connection_server_close(&conn);
+ return;
+ }
+
+ /* get next waiting request */
+ if (array_count(&conn->request_wait_list) > 0) {
+ req_idx = array_idx(&conn->request_wait_list, 0);
+ req = req_idx[0];
+ no_payload = (strcmp(req->method, "HEAD") == 0);
+ } else {
+ req = NULL;
+ no_payload = FALSE;
+ }
+ }
+
+ if (ret <= 0 &&
+ (conn->conn.input->eof || conn->conn.input->stream_errno != 0)) {
+ int stream_errno = conn->conn.input->stream_errno;
+ http_client_connection_debug(conn,
+ "Lost connection to server (error=%s)",
+ stream_errno != 0 ? strerror(stream_errno) : "EOF");
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+ return;
+ }
+
+ if (ret < 0) {
+ http_client_connection_abort_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, error);
+ return;
+ }
+
+ if (finished > 0) {
+ /* room for new requests */
+ http_client_peer_handle_requests(conn->peer);
+ http_client_connection_check_idle(conn);
+ }
+}
+
+static int http_client_connection_output(struct http_client_connection *conn)
+{
+ struct http_client_request *const *req_idx, *req;
+ struct ostream *output = conn->conn.output;
+ int ret;
+
+ if ((ret = o_stream_flush(output)) <= 0) {
+ if (ret < 0) {
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+ }
+ return ret;
+ }
+
+ if (array_count(&conn->request_wait_list) > 0 && conn->output_locked) {
+ req_idx = array_idx(&conn->request_wait_list, 0);
+ req = req_idx[0];
+
+ if (!req->payload_sync || conn->payload_continue) {
+ if (http_client_request_send_more(req) < 0) {
+ http_client_connection_abort_temp_error(&conn,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST, "Connection lost");
+ return -1;
+ }
+ if (!conn->output_locked) {
+ /* room for new requests */
+ http_client_peer_handle_requests(conn->peer);
+ http_client_connection_check_idle(conn);
+ }
+ }
+ }
+ return 1;
+}
+
+static void
+http_client_connection_ready(struct http_client_connection *conn)
+{
+ struct stat st;
+
+ conn->connected = TRUE;
+
+ if (*conn->client->set.rawlog_dir != '\0' &&
+ stat(conn->client->set.rawlog_dir, &st) == 0) {
+ iostream_rawlog_create(conn->client->set.rawlog_dir,
+ &conn->conn.input, &conn->conn.output);
+ }
+
+ conn->http_parser = http_response_parser_init(conn->conn.input);
+ o_stream_set_flush_callback(conn->conn.output,
+ http_client_connection_output, conn);
+
+ /* we never pipeline before the first response */
+ (void)http_client_connection_next_request(conn);
+}
+
+#ifdef HTTP_BUILD_SSL
+static int http_client_connection_ssl_handshaked(void *context)
+{
+ struct http_client_connection *conn = context;
+
+ if (!conn->client->set.ssl_verify) {
+ /* skip certificate checks */
+ http_client_connection_debug(conn, "SSL handshake successful");
+ return 0;
+ } else if (!ssl_iostream_has_valid_client_cert(conn->ssl_iostream)) {
+ if (!ssl_iostream_has_broken_client_cert(conn->ssl_iostream)) {
+ http_client_connection_error(conn, "SSL certificate not received");
+ } else {
+ http_client_connection_error(conn, "Received invalid SSL certificate");
+ }
+ } else {
+ const char *host = http_client_peer_get_hostname(conn->peer);
+
+ i_assert(host != NULL);
+
+ if (ssl_iostream_cert_match_name(conn->ssl_iostream, host) < 0) {
+ http_client_connection_error(conn,
+ "SSL certificate doesn't match host name");
+ } else {
+ http_client_connection_debug(conn, "SSL handshake successful");
+ return 0;
+ }
+ }
+ i_stream_close(conn->conn.input);
+ return -1;
+}
+
+static int
+http_client_connection_ssl_init(struct http_client_connection *conn)
+{
+ struct ssl_iostream_settings ssl_set;
+ const char *source;
+
+ if (conn->peer->ssl_ctx == NULL) {
+ http_client_connection_error(conn, "No SSL context");
+ return -1;
+ }
+
+ memset(&ssl_set, 0, sizeof(ssl_set));
+ if (conn->client->set.ssl_verify) {
+ ssl_set.verbose_invalid_cert = TRUE;
+ ssl_set.verify_remote_cert = TRUE;
+ ssl_set.require_valid_cert = TRUE;
+ }
+
+ if (conn->client->set.debug)
+ http_client_connection_debug(conn, "Starting SSL handshake");
+
+ source = t_strdup_printf
+ ("connection %s: ", http_client_connection_label(conn));
+ if (io_stream_create_ssl(conn->peer->ssl_ctx, source, &ssl_set,
+ &conn->conn.input, &conn->conn.output, &conn->ssl_iostream) < 0) {
+ http_client_connection_error(conn, "Couldn't initialize SSL client");
+ return -1;
+ }
+ ssl_iostream_set_handshake_callback(conn->ssl_iostream,
+ http_client_connection_ssl_handshaked, conn);
+ if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
+ http_client_connection_error(conn, "SSL handshake failed: %s",
+ ssl_iostream_get_last_error(conn->ssl_iostream));
+ return -1;
+ }
+
+ http_client_connection_ready(conn);
+ return 0;
+}
+#endif
+
+static void
+http_client_connection_connected(struct connection *_conn, bool success)
+{
+ struct http_client_connection *conn =
+ (struct http_client_connection *)_conn;
+
+ if (!success) {
+ http_client_connection_error(conn, "Connect failed: %m");
+ http_client_peer_connection_failure(conn->peer);
+
+ } else {
+ http_client_connection_debug(conn, "Connected");
+#ifdef HTTP_BUILD_SSL
+ if (conn->peer->addr.ssl) {
+ if (http_client_connection_ssl_init(conn) < 0)
+ http_client_peer_connection_failure(conn->peer);
+ return;
+ }
+#endif
+ http_client_connection_ready(conn);
+ }
+}
+
+static const struct connection_settings http_client_connection_set = {
+ .input_max_size = (size_t)-1,
+ .output_max_size = (size_t)-1,
+ .client = TRUE
+};
+
+static const struct connection_vfuncs http_client_connection_vfuncs = {
+ .destroy = http_client_connection_destroy,
+ .input = http_client_connection_input,
+ .client_connected = http_client_connection_connected
+};
+
+struct connection_list *
+http_client_connection_list_init(void)
+{
+ return connection_list_init
+ (&http_client_connection_set, &http_client_connection_vfuncs);
+}
+
+static int http_client_connection_connect(struct http_client_connection *conn)
+{
+ if (conn->conn.fd_in == -1) {
+ if (connection_client_connect(&conn->conn) < 0) {
+ http_client_connection_error(conn, "Could not connect");
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+struct http_client_connection *
+http_client_connection_create(struct http_client_peer *peer)
+{
+ struct http_client_connection *conn;
+ static unsigned int id = 0;
+
+ conn = i_new(struct http_client_connection, 1);
+ conn->client = peer->client;
+ conn->peer = peer;
+ i_array_init(&conn->request_wait_list, 16);
+
+ connection_init_client_ip
+ (peer->client->conn_list, &conn->conn, &peer->addr.ip, peer->addr.port);
+
+ if (http_client_connection_connect(conn) < 0) {
+ http_client_connection_free(&conn);
+ return NULL;
+ }
+
+ conn->id = id++;
+ array_append(&peer->conns, &conn, 1);
+
+ http_client_connection_debug(conn,
+ "Connection created (%d parallel connections exist)",
+ array_count(&peer->conns));
+ return conn;
+}
+
+void http_client_connection_free(struct http_client_connection **_conn)
+{
+ struct http_client_connection *conn = *_conn;
+ struct http_client_connection *const *conn_idx;
+ struct http_client_peer *peer = conn->peer;
+ struct http_client_request **req;
+
+ http_client_connection_debug(conn, "Connection destroy");
+
+ conn->connected = FALSE;
+
+#ifdef HTTP_BUILD_SSL
+ if (conn->ssl_iostream != NULL)
+ ssl_iostream_unref(&conn->ssl_iostream);
+#endif
+
+ connection_deinit(&conn->conn);
+
+ /* abort all pending requests */
+ array_foreach_modifiable(&conn->request_wait_list, req) {
+ http_client_request_error(*req, HTTP_CLIENT_REQUEST_ERROR_ABORTED,
+ "Aborting");
+ http_client_request_unref(req);
+ }
+ array_free(&conn->request_wait_list);
+
+ if (conn->to_input != NULL)
+ timeout_remove(&conn->to_input);
+ if (conn->to_idle != NULL)
+ timeout_remove(&conn->to_idle);
+ if (conn->to_response != NULL)
+ timeout_remove(&conn->to_response);
+
+ /* remove this connection from the list */
+ array_foreach(&conn->peer->conns, conn_idx) {
+ if (*conn_idx == conn) {
+ array_delete(&conn->peer->conns,
+ array_foreach_idx(&conn->peer->conns, conn_idx), 1);
+ break;
+ }
+ }
+
+ if (conn->http_parser != NULL)
+ http_response_parser_deinit(&conn->http_parser);
+
+ i_free(conn);
+ *_conn = NULL;
+
+ http_client_peer_connection_lost(peer);
+}
+
+void http_client_connection_switch_ioloop(struct http_client_connection *conn)
+{
+ if (conn->to_input != NULL)
+ conn->to_input = io_loop_move_timeout(&conn->to_input);
+ if (conn->to_idle != NULL)
+ conn->to_idle = io_loop_move_timeout(&conn->to_idle);
+ if (conn->to_response != NULL)
+ conn->to_response = io_loop_move_timeout(&conn->to_response);
+ connection_switch_ioloop(&conn->conn);
+}
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "dns-lookup.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_host_debug(struct http_client_host *host,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_host_debug(struct http_client_host *host,
+ const char *format, ...)
+{
+ va_list args;
+
+ if (host->client->set.debug) {
+
+ va_start(args, format);
+ i_debug("http-client: host %s: %s",
+ host->name, t_strdup_vprintf(format, args));
+ va_end(args);
+ }
+}
+
+/*
+ * Host:port
+ */
+
+static struct http_client_host_port *
+http_client_host_port_find(struct http_client_host *host,
+ unsigned int port, bool ssl)
+{
+ struct http_client_host_port *hport;
+
+ array_foreach_modifiable(&host->ports, hport) {
+ if (hport->port == port && hport->ssl == ssl)
+ return hport;
+ }
+
+ return NULL;
+}
+
+static struct http_client_host_port *
+http_client_host_port_init(struct http_client_host *host,
+ unsigned int port, bool ssl)
+{
+ struct http_client_host_port *hport;
+
+ hport = http_client_host_port_find(host, port, ssl);
+ if (hport == NULL) {
+ hport = array_append_space(&host->ports);
+ hport->port = port;
+ hport->ssl = ssl;
+ hport->ips_connect_idx = 0;
+ i_array_init(&hport->request_queue, 16);
+ i_array_init(&hport->urgent_request_queue, 4);
+ }
+
+ return hport;
+}
+
+static void http_client_host_port_error(struct http_client_host_port *hport,
+ unsigned int status, const char *error)
+{
+ struct http_client_request **req;
+
+ /* abort all pending requests */
+ array_foreach_modifiable(&hport->request_queue, req) {
+ http_client_request_error(*req, status, error);
+ }
+ array_foreach_modifiable(&hport->urgent_request_queue, req) {
+ http_client_request_error(*req, status, error);
+ }
+ array_clear(&hport->request_queue);
+ array_clear(&hport->urgent_request_queue);
+}
+
+static void http_client_host_port_deinit(struct http_client_host_port *hport)
+{
+ http_client_host_port_error
+ (hport, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborted");
+ array_free(&hport->request_queue);
+ array_free(&hport->urgent_request_queue);
+}
+
+static void
+http_client_host_port_drop_request(struct http_client_host_port *hport,
+ struct http_client_request *req)
+{
+ struct http_client_request **req_idx;
+ ARRAY_TYPE(http_client_request) *queue =
+ (req->urgent ? &hport->urgent_request_queue : &hport->request_queue);
+ unsigned int idx;
+
+ array_foreach_modifiable(queue, req_idx) {
+ if (*req_idx == req) {
+ idx = array_foreach_idx(queue, req_idx);
+ break;
+ }
+ }
+
+ array_delete(queue, idx, 1);
+}
+
+/*
+ * Host
+ */
+
+static void
+http_client_host_connection_setup(struct http_client_host *host,
+ struct http_client_host_port *hport)
+{
+ struct http_client_peer *peer = NULL;
+ struct http_client_peer_addr addr;
+
+ while (hport->ips_connect_idx < host->ips_count) {
+ addr.ip = host->ips[hport->ips_connect_idx];
+ addr.port = hport->port;
+ addr.ssl = hport->ssl;
+
+ http_client_host_debug(host, "Setting up connection to %s:%u (ssl=%s)",
+ net_ip2addr(&addr.ip), addr.port, (addr.ssl ? "yes" : "no"));
+
+ if ((peer=http_client_peer_get(host->client, &addr)) != NULL)
+ break;
+
+ hport->ips_connect_idx++;
+ }
+
+ if (peer == NULL) {
+ http_client_host_port_error
+ (hport, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, "Connection failed");
+ if (host->client->ioloop != NULL)
+ io_loop_stop(host->client->ioloop);
+ return;
+ }
+
+ http_client_peer_add_host(peer, host);
+}
+
+void http_client_host_connection_failure(struct http_client_host *host,
+ const struct http_client_peer_addr *addr)
+{
+ struct http_client_host_port *hport;
+
+ http_client_host_debug(host, "Failed to connect to %s:%u",
+ net_ip2addr(&addr->ip), addr->port);
+
+ hport = http_client_host_port_find(host, addr->port, addr->ssl);
+ if (hport == NULL)
+ return;
+
+ hport->ips_connect_idx++;
+ http_client_host_connection_setup(host, hport);
+}
+
+static void
+http_client_host_lookup_failure(struct http_client_host *host)
+{
+ struct http_client_host_port *hport;
+
+ array_foreach_modifiable(&host->ports, hport) {
+ http_client_host_port_error(hport,
+ HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED, "Failed to lookup host");
+ }
+}
+
+static void
+http_client_host_dns_callback(const struct dns_lookup_result *result,
+ struct http_client_host *host)
+{
+ struct http_client_host_port *hport;
+ unsigned int requests = 0;
+
+ host->dns_lookup = NULL;
+
+ if (result->ret != 0) {
+ i_error("http-client: dns_lookup(%s) failed: %s",
+ host->name, result->error);
+ http_client_host_lookup_failure(host);
+ return;
+ }
+
+ http_client_host_debug(host,
+ "DNS lookup successful; got %d IPs", result->ips_count);
+
+ i_assert(result->ips_count > 0);
+ host->ips_count = result->ips_count;
+ host->ips = i_new(struct ip_addr, host->ips_count);
+ memcpy(host->ips, result->ips, sizeof(*host->ips) * host->ips_count);
+
+ // FIXME: make DNS result expire
+
+ /* make connections to requested ports */
+ array_foreach_modifiable(&host->ports, hport) {
+ unsigned int count = array_count(&hport->request_queue);
+ hport->ips_connect_idx = 0;
+ if (count > 0)
+ http_client_host_connection_setup(host, hport);
+ requests += count;
+ }
+
+ if (requests == 0 && host->client->ioloop != NULL)
+ io_loop_stop(host->client->ioloop);
+}
+
+static void http_client_host_lookup
+(struct http_client_host *host)
+{
+ struct http_client *client = host->client;
+ struct dns_lookup_settings dns_set;
+ struct ip_addr ip, *ips;
+ unsigned int ips_count;
+ int ret;
+
+ memset(&dns_set, 0, sizeof(dns_set));
+ dns_set.dns_client_socket_path =
+ client->set.dns_client_socket_path;
+ dns_set.timeout_msecs = HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS;
+
+ if (host->ips_count == 0 &&
+ net_addr2ip(host->name, &ip) == 0) { // FIXME: remove this?
+ host->ips_count = 1;
+ host->ips = i_new(struct ip_addr, host->ips_count);
+ host->ips[0] = ip;
+ } else if (*dns_set.dns_client_socket_path == '\0') {
+ ret = net_gethostbyname(host->name, &ips, &ips_count);
+ if (ret != 0) {
+ i_error("http-client: net_gethostbyname(%s) failed: %s",
+ host->name, net_gethosterror(ret));
+ http_client_host_lookup_failure(host);
+ return;
+ }
+
+ http_client_host_debug(host,
+ "DNS lookup successful; got %d IPs", ips_count);
+
+ host->ips_count = ips_count;
+ host->ips = i_new(struct ip_addr, ips_count);
+ memcpy(host->ips, ips, ips_count * sizeof(*ips));
+ }
+
+ if (host->ips_count == 0) {
+ http_client_host_debug(host,
+ "Performing asynchronous DNS lookup");
+ (void)dns_lookup(host->name, &dns_set,
+ http_client_host_dns_callback, host, &host->dns_lookup);
+ }
+}
+
+struct http_client_host *http_client_host_get
+(struct http_client *client, const char *hostname)
+{
+ struct http_client_host *host;
+
+ host = hash_table_lookup(client->hosts, hostname);
+ if (host == NULL) {
+ // FIXME: limit the maximum number of inactive cached hosts
+ host = i_new(struct http_client_host, 1);
+ host->client = client;
+ host->name = i_strdup(hostname);
+ i_array_init(&host->ports, 4);
+
+ hash_table_insert(client->hosts, hostname, host);
+
+ http_client_host_debug(host, "Host created");
+ }
+ return host;
+}
+
+void http_client_host_submit_request(struct http_client_host *host,
+ struct http_client_request *req)
+{
+ struct http_client_host_port *hport;
+
+ req->host = host;
+
+ if (host->ips_count == 0 && host->dns_lookup == NULL)
+ http_client_host_lookup(host);
+ hport = http_client_host_port_init(host, req->port, req->ssl);
+ if (req->urgent)
+ array_append(&hport->urgent_request_queue, &req, 1);
+ else
+ array_append(&hport->request_queue, &req, 1);
+
+ if (host->ips_count == 0)
+ return;
+
+ http_client_host_connection_setup(host, hport);
+}
+
+struct http_client_request *
+http_client_host_claim_request(struct http_client_host *host,
+ const struct http_client_peer_addr *addr, bool no_urgent)
+{
+ struct http_client_host_port *hport;
+ struct http_client_request *const *req_idx;
+ struct http_client_request *req;
+
+ hport = http_client_host_port_find(host, addr->port, addr->ssl);
+ if (hport == NULL)
+ return NULL;
+ if (!no_urgent && array_count(&hport->urgent_request_queue) > 0) {
+ req_idx = array_idx(&hport->urgent_request_queue, 0);
+ req = *req_idx;
+ array_delete(&hport->urgent_request_queue, 0, 1);
+ } else if (array_count(&hport->request_queue) > 0) {
+ req_idx = array_idx(&hport->request_queue, 0);
+ req = *req_idx;
+ array_delete(&hport->request_queue, 0, 1);
+ } else {
+ return NULL;
+ }
+
+ http_client_host_debug(host,
+ "Connection %s:%u claimed request %s %s",
+ net_ip2addr(&addr->ip), addr->port, http_client_request_label(req),
+ (req->urgent ? "(urgent)" : ""));
+
+ return req;
+}
+
+bool http_client_host_have_requests(struct http_client_host *host,
+ const struct http_client_peer_addr *addr, bool urgent)
+{
+ struct http_client_host_port *hport;
+
+ hport = http_client_host_port_find(host, addr->port, addr->ssl);
+ if (hport == NULL)
+ return FALSE;
+
+ if (urgent)
+ return (array_count(&hport->urgent_request_queue) > 0);
+ return (array_count(&hport->request_queue) > 0);
+}
+
+void http_client_host_drop_request(struct http_client_host *host,
+ struct http_client_request *req)
+{
+ struct http_client_host_port *hport;
+
+ hport = http_client_host_port_find(host, req->port, req->ssl);
+ if (hport == NULL)
+ return;
+
+ http_client_host_port_drop_request(hport, req);
+}
+
+void http_client_host_free(struct http_client_host **_host)
+{
+ struct http_client_host *host = *_host;
+ struct http_client_host_port *hport;
+
+ http_client_host_debug(host, "Host destroy");
+
+ if (host->dns_lookup != NULL)
+ dns_lookup_abort(&host->dns_lookup);
+
+ /* drop request queues */
+ array_foreach_modifiable(&host->ports, hport) {
+ http_client_host_port_deinit(hport);
+ }
+ array_free(&host->ports);
+
+ i_free(host->ips);
+ i_free(host->name);
+ i_free(host);
+}
+
+void http_client_host_switch_ioloop(struct http_client_host *host)
+{
+ if (host->dns_lookup != NULL)
+ dns_lookup_switch_ioloop(host->dns_lookup);
+}
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-ssl.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_peer_debug(struct http_client_peer *peer,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_peer_debug(struct http_client_peer *peer,
+ const char *format, ...)
+{
+ va_list args;
+
+ if (peer->client->set.debug) {
+ va_start(args, format);
+ i_debug("http-client: peer %s:%u: %s",
+ net_ip2addr(&peer->addr.ip), peer->addr.port,
+ t_strdup_vprintf(format, args));
+ va_end(args);
+ }
+}
+
+static inline void ATTR_FORMAT(2, 3)
+http_client_peer_error(struct http_client_peer *peer,
+ const char *format, ...)
+{
+ va_list args;
+ va_start(args, format);
+ i_error("http-client: peer %s:%u: %s",
+ net_ip2addr(&peer->addr.ip), peer->addr.port,
+ t_strdup_vprintf(format, args));
+ va_end(args);
+}
+
+/*
+ * Peer address
+ */
+
+unsigned int http_client_peer_addr_hash
+(const struct http_client_peer_addr *peer)
+{
+ return net_ip_hash(&peer->ip) + peer->port + peer->ssl;
+}
+
+int http_client_peer_addr_cmp
+(const struct http_client_peer_addr *peer1,
+ const struct http_client_peer_addr *peer2)
+{
+ int ret;
+
+ if ((ret=net_ip_cmp(&peer1->ip, &peer2->ip)) != 0)
+ return ret;
+ if (peer1->port != peer2->port)
+ return (peer1->port > peer2->port ? 1 : -1);
+ if (peer1->ssl != peer2->ssl)
+ return (peer1->ssl > peer2->ssl ? 1 : -1);
+ return 0;
+}
+
+/*
+ * Peer
+ */
+
+const char *
+http_client_peer_get_hostname(struct http_client_peer *peer)
+{
+ struct http_client_host *const *host;
+
+ if (array_count(&peer->hosts) == 0)
+ return NULL;
+
+ /* just return name of initial host */
+ host = array_idx(&peer->hosts, 1);
+ return (*host)->name;
+}
+
+static int
+http_client_peer_connect(struct http_client_peer *peer)
+{
+ struct http_client_connection *conn;
+
+ conn = http_client_connection_create(peer);
+ if (conn == NULL) {
+ http_client_peer_debug(peer, "Failed to make new connection");
+ return -1;
+ }
+
+ return 0;
+}
+
+static bool
+http_client_peer_requests_pending(struct http_client_peer *peer, bool urgent)
+{
+ struct http_client_host *const *host;
+
+ array_foreach(&peer->hosts, host) {
+ if (http_client_host_have_requests(*host, &peer->addr, urgent))
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static bool
+http_client_peer_next_request(struct http_client_peer *peer,
+ bool urgent)
+{
+ struct http_client_connection *const *conn_idx;
+ struct http_client_connection *conn = NULL;
+ unsigned int closing = 0, min_waiting = (unsigned int)-1;
+
+ /* at this point we already know that a request for this peer is pending
+ */
+
+ /* find the least busy connection */
+ array_foreach(&peer->conns, conn_idx) {
+ if (http_client_connection_is_ready(*conn_idx)) {
+ unsigned int waiting = array_count(&(*conn_idx)->request_wait_list);
+ if (waiting < min_waiting) {
+ min_waiting = waiting;
+ conn = *conn_idx;
+ if (min_waiting == 0)
+ break;
+ }
+ }
+ /* count the number of closing connections */
+ if ((*conn_idx)->closing)
+ closing++;
+ }
+
+ /* do we have an idle connection? */
+ if (conn != NULL && min_waiting == 0) {
+ /* yes */
+ return http_client_connection_next_request(conn);
+ }
+
+ /* no, but can we create a new connection? */
+ if (!urgent && (array_count(&peer->conns) - closing) >=
+ peer->client->set.max_parallel_connections) {
+ /* no */
+ if (conn == NULL)
+ return FALSE;
+ /* pipeline it */
+ return http_client_connection_next_request(conn);
+ }
+
+ /* yes */
+ if (http_client_peer_connect(peer) < 0) {
+ /* connection failed */
+ if (conn == NULL)
+ return FALSE;
+ /* pipeline it on the least busy connection we found */
+ return http_client_connection_next_request(conn);
+ }
+
+ /* now we wait until it is connected */
+ return FALSE;
+}
+
+void http_client_peer_handle_requests(struct http_client_peer *peer)
+{
+ /* check urgent requests first */
+ while (http_client_peer_requests_pending(peer, TRUE)) {
+ if (!http_client_peer_next_request(peer, TRUE))
+ break;
+ }
+
+ /* check normal requests once we're done */
+ while (http_client_peer_requests_pending(peer, FALSE)) {
+ if (!http_client_peer_next_request(peer, FALSE))
+ break;
+ }
+}
+
+static struct http_client_peer *
+http_client_peer_create(struct http_client *client,
+ const struct http_client_peer_addr *addr)
+{
+ struct http_client_peer *peer;
+#ifdef HTTP_BUILD_SSL
+ struct ssl_iostream_settings ssl_set;
+#endif
+
+ peer = i_new(struct http_client_peer, 1);
+ peer->client = client;
+ peer->addr = *addr;
+ i_array_init(&peer->hosts, 16);
+ i_array_init(&peer->conns, 16);
+
+ hash_table_insert
+ (client->peers, (const struct http_client_peer_addr *)&peer->addr, peer);
+
+ http_client_peer_debug(peer, "Peer created");
+
+#ifdef HTTP_BUILD_SSL
+ if (peer->addr.ssl && peer->ssl_ctx == NULL) {
+ const char *source;
+ memset(&ssl_set, 0, sizeof(ssl_set));
+ ssl_set.ca_dir = peer->client->set.ssl_ca_dir;
+ ssl_set.verify_remote_cert = TRUE;
+ ssl_set.crypto_device = peer->client->set.ssl_crypto_device;
+
+ source = t_strdup_printf("http-client: peer %s:%u",
+ net_ip2addr(&peer->addr.ip), peer->addr.port);
+ if (ssl_iostream_context_init_client
+ (source, &ssl_set, &peer->ssl_ctx) < 0) {
+ http_client_peer_error(peer, "Couldn't initialize SSL context");
+ http_client_peer_free(&peer);
+ return NULL;
+ }
+ }
+#else
+ if (peer->addr.ssl) {
+ http_client_peer_error(peer, "HTTPS is not supported");
+ http_client_peer_free(&peer);
+ return NULL;
+ }
+#endif
+
+ if (http_client_peer_connect(peer) < 0) {
+ http_client_peer_free(&peer);
+ return NULL;
+ }
+
+ return peer;
+}
+
+void http_client_peer_free(struct http_client_peer **_peer)
+{
+ struct http_client_peer *peer = *_peer;
+ struct http_client_connection **conn;
+ ARRAY_TYPE(http_client_connection) conns;
+
+ if (peer->destroyed)
+ return;
+ peer->destroyed = TRUE;
+
+ http_client_peer_debug(peer, "Peer destroy");
+
+ /* make a copy of the connection array; freed connections modify it */
+ t_array_init(&conns, array_count(&peer->conns));
+ array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
+
+ array_foreach_modifiable(&conns, conn) {
+ http_client_connection_free(conn);
+ }
+
+ i_assert(array_count(&peer->conns) == 0);
+ array_free(&peer->conns);
+ array_free(&peer->hosts);
+
+#ifdef HTTP_BUILD_SSL
+ if (peer->ssl_ctx != NULL)
+ ssl_iostream_context_deinit(&peer->ssl_ctx);
+#endif
+
+ hash_table_remove
+ (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
+
+ i_free(peer);
+ *_peer = NULL;
+}
+
+struct http_client_peer *
+http_client_peer_get(struct http_client *client,
+ const struct http_client_peer_addr *addr)
+{
+ struct http_client_peer *peer;
+
+ peer = hash_table_lookup(client->peers, addr);
+ if (peer == NULL)
+ peer = http_client_peer_create(client, addr);
+
+ return peer;
+}
+
+void http_client_peer_add_host(struct http_client_peer *peer,
+ struct http_client_host *host)
+{
+ struct http_client_host *const *host_idx;
+ bool exists = FALSE;
+
+ array_foreach(&peer->hosts, host_idx) {
+ if (*host_idx == host) {
+ exists = TRUE;
+ break;
+ }
+ }
+
+ if (!exists)
+ array_append(&peer->hosts, &host, 1);
+ if (exists || array_count(&peer->hosts) > 1)
+ (void)http_client_peer_next_request(peer, FALSE);
+}
+
+struct http_client_request *
+http_client_peer_claim_request(struct http_client_peer *peer, bool no_urgent)
+{
+ struct http_client_host *const *host_idx;
+ struct http_client_request *req;
+
+ array_foreach(&peer->hosts, host_idx) {
+ if ((req=http_client_host_claim_request
+ (*host_idx, &peer->addr, no_urgent)) != NULL) {
+ req->peer = peer;
+ return req;
+ }
+ }
+
+ return NULL;
+}
+
+void http_client_peer_connection_failure(struct http_client_peer *peer)
+{
+ struct http_client_host *const *host;
+
+ http_client_peer_debug(peer, "Failed to make connection");
+
+ if (array_count(&peer->conns) == 1) {
+ array_foreach(&peer->hosts, host) {
+ http_client_host_connection_failure(*host, &peer->addr);
+ }
+ }
+}
+
+void http_client_peer_connection_lost(struct http_client_peer *peer)
+{
+ if (peer->destroyed)
+ return;
+
+ http_client_peer_debug(peer, "Lost connection (%d connections left)",
+ array_count(&peer->conns));
+
+ if (array_count(&peer->conns) == 0) {
+ if (http_client_peer_requests_pending(peer, TRUE))
+ (void)http_client_peer_next_request(peer, TRUE);
+ else if (http_client_peer_requests_pending(peer, FALSE))
+ (void)http_client_peer_next_request(peer, FALSE);
+ else
+ http_client_peer_free(&peer);
+ }
+}
+
--- /dev/null
+#ifndef HTTP_CLIENT_PRIVATE_H
+#define HTTP_CLIENT_PRIVATE_H
+
+#include "connection.h"
+
+#include "http-client.h"
+
+#define HTTP_BUILD_SSL
+
+#define HTTP_DEFAULT_PORT 80
+#define HTTPS_DEFAULT_PORT 443
+
+#define HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
+#define HTTP_CLIENT_CONNECT_TIMEOUT_MSECS (1000*30)
+#define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5)
+#define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2)
+
+struct http_client_host;
+struct http_client_host_port;
+struct http_client_peer;
+struct http_client_connection;
+
+ARRAY_DEFINE_TYPE(http_client_host, struct http_client_host *);
+ARRAY_DEFINE_TYPE(http_client_host_port, struct http_client_host_port);
+ARRAY_DEFINE_TYPE(http_client_connection, struct http_client_connection *);
+ARRAY_DEFINE_TYPE(http_client_request, struct http_client_request *);
+
+HASH_TABLE_DEFINE_TYPE(http_client_host, const char *,
+ struct http_client_host *);
+HASH_TABLE_DEFINE_TYPE(http_client_peer, const struct http_client_peer_addr *,
+ struct http_client_peer *);
+
+enum http_request_state {
+ HTTP_REQUEST_STATE_NEW = 0,
+ HTTP_REQUEST_STATE_QUEUED,
+ HTTP_REQUEST_STATE_PAYLOAD_OUT,
+ HTTP_REQUEST_STATE_WAITING,
+ HTTP_REQUEST_STATE_GOT_RESPONSE,
+ HTTP_REQUEST_STATE_PAYLOAD_IN,
+ HTTP_REQUEST_STATE_FINISHED,
+ HTTP_REQUEST_STATE_ABORTED
+};
+
+struct http_client_request {
+ pool_t pool;
+ unsigned int refcount;
+
+ struct http_client_request *prev, *next;
+
+ const char *method, *hostname, *target;
+ unsigned int port;
+
+ struct http_client *client;
+ struct http_client_host *host;
+ struct http_client_peer *peer;
+ struct http_client_connection *conn;
+
+ string_t *headers;
+ struct istream *input;
+ uoff_t input_size;
+
+ unsigned int attempts;
+ unsigned int redirects;
+
+ http_client_request_callback_t *callback;
+ void *context;
+
+ enum http_request_state state;
+
+ unsigned int payload_sync:1;
+ unsigned int ssl:1;
+ unsigned int urgent:1;
+};
+
+struct http_client_host_port {
+ unsigned int port;
+
+ /* current index in host->ips */
+ unsigned int ips_connect_idx;
+
+ /* requests pending in queue to be picked up by connections */
+ ARRAY_TYPE(http_client_request) request_queue;
+
+ /* urgent request queue */
+ ARRAY_TYPE(http_client_request) urgent_request_queue;
+
+ unsigned int ssl:1;
+};
+
+struct http_client_host {
+ struct http_client *client;
+ char *name;
+
+ /* the ip addresses DNS returned for this host */
+ unsigned int ips_count;
+ struct ip_addr *ips;
+
+ /* requests are managed on a per-port basis */
+ ARRAY_TYPE(http_client_host_port) ports;
+
+ /* active DNS lookup */
+ struct dns_lookup *dns_lookup;
+};
+
+struct http_client_peer_addr {
+ struct ip_addr ip;
+ unsigned int port;
+ unsigned int ssl:1; /* https */
+};
+
+struct http_client_peer {
+ struct http_client_peer_addr addr;
+ struct http_client *client;
+
+ /* hosts served through this peer */
+ ARRAY_TYPE(http_client_host) hosts;
+
+ /* active connections to this peer */
+ ARRAY_TYPE(http_client_connection) conns;
+
+ /* ssl */
+#ifdef HTTP_BUILD_SSL
+ struct ssl_iostream_context *ssl_ctx;
+#endif
+
+ unsigned int destroyed:1; /* peer is being destroyed */
+ unsigned int no_payload_sync:1; /* expect: 100-continue failed before */
+};
+
+struct http_client_connection {
+ struct connection conn;
+ struct http_client_peer *peer;
+ struct http_client *client;
+
+ const char *label;
+
+ unsigned int id; // DEBUG: identify parallel connections
+
+ /* ssl */
+#ifdef HTTP_BUILD_SSL
+ struct ssl_iostream *ssl_iostream;
+#endif
+
+ struct http_response_parser *http_parser;
+ struct timeout *to_input, *to_idle, *to_response;
+
+ struct http_client_request *pending_request;
+ struct istream *incoming_payload;
+
+ /* requests that have been sent, waiting for response */
+ ARRAY_TYPE(http_client_request) request_wait_list;
+
+ unsigned int connected:1; /* connection is connected */
+ unsigned int closing:1;
+ unsigned int close_indicated:1;
+ unsigned int output_locked:1; /* output is locked; no pipelining */
+ unsigned int payload_continue:1; /* received 100-continue for current
+ request */
+};
+
+struct http_client {
+ pool_t pool;
+
+ struct http_client_settings set;
+
+ struct ioloop *ioloop;
+
+ struct connection_list *conn_list;
+
+ HASH_TABLE_TYPE(http_client_host) hosts;
+ HASH_TABLE_TYPE(http_client_peer) peers;
+ unsigned int pending_requests;
+};
+
+static inline const char *
+http_client_request_label(struct http_client_request *req)
+{
+ return t_strdup_printf("[%s http%s://%s:%d%s]",
+ req->method, req->ssl ? "s" : "", req->hostname, req->port, req->target);
+}
+
+static inline const char *
+http_client_connection_label(struct http_client_connection *conn)
+{
+ return t_strdup_printf("%s:%u [%d]",
+ net_ip2addr(&conn->conn.ip), conn->conn.port, conn->id);
+}
+
+void http_client_request_ref(struct http_client_request *req);
+void http_client_request_unref(struct http_client_request **_req);
+int http_client_request_send(struct http_client_request *req);
+int http_client_request_send_more(struct http_client_request *req);
+void http_client_request_callback(struct http_client_request *req,
+ struct http_response *response);
+void http_client_request_resubmit(struct http_client_request *req);
+void http_client_request_retry(struct http_client_request *req,
+ unsigned int status, const char *error);
+void http_client_request_error(struct http_client_request *req,
+ unsigned int status, const char *error);
+void http_client_request_redirect(struct http_client_request *req,
+ const char *location);
+void http_client_request_finish(struct http_client_request **_req);
+
+struct connection_list *http_client_connection_list_init(void);
+
+struct http_client_connection *
+ http_client_connection_create(struct http_client_peer *peer);
+void http_client_connection_free(struct http_client_connection **_conn);
+bool http_client_connection_is_ready(struct http_client_connection *conn);
+bool http_client_connection_next_request(struct http_client_connection *conn);
+void http_client_connection_switch_ioloop(struct http_client_connection *conn);
+
+unsigned int http_client_peer_addr_hash
+ (const struct http_client_peer_addr *peer) ATTR_PURE;
+int http_client_peer_addr_cmp
+ (const struct http_client_peer_addr *peer1,
+ const struct http_client_peer_addr *peer2) ATTR_PURE;
+
+const char *
+ http_client_peer_get_hostname(struct http_client_peer *peer);
+struct http_client_peer *
+ http_client_peer_get(struct http_client *client,
+ const struct http_client_peer_addr *addr);
+void http_client_peer_free(struct http_client_peer **_peer);
+void http_client_peer_add_host(struct http_client_peer *peer,
+ struct http_client_host *host);
+struct http_client_request *
+ http_client_peer_claim_request(struct http_client_peer *peer,
+ bool no_urgent);
+void http_client_peer_handle_requests(struct http_client_peer *peer);
+void http_client_peer_connection_failure(struct http_client_peer *peer);
+void http_client_peer_connection_lost(struct http_client_peer *peer);
+
+struct http_client_host *
+ http_client_host_get(struct http_client *client, const char *hostname);
+void http_client_host_free(struct http_client_host **_host);
+void http_client_host_submit_request(struct http_client_host *host,
+ struct http_client_request *req);
+struct http_client_request *
+http_client_host_claim_request(struct http_client_host *host,
+ const struct http_client_peer_addr *addr, bool no_urgent);
+void http_client_host_connection_failure(struct http_client_host *host,
+ const struct http_client_peer_addr *addr);
+bool http_client_host_have_requests(struct http_client_host *host,
+ const struct http_client_peer_addr *addr, bool urgent);
+void http_client_host_drop_request(struct http_client_host *host,
+ struct http_client_request *req);
+void http_client_host_switch_ioloop(struct http_client_host *host);
+
+#endif
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "http-url.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_request_debug(struct http_client_request *req,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_request_debug(struct http_client_request *req,
+ const char *format, ...)
+{
+ va_list args;
+
+ if (req->client->set.debug) {
+ va_start(args, format);
+ i_debug("http-client: request %s: %s",
+ http_client_request_label(req), t_strdup_vprintf(format, args));
+ va_end(args);
+ }
+}
+
+/*
+ * Request
+ */
+
+#undef http_client_request
+struct http_client_request *
+http_client_request(struct http_client *client,
+ const char *method, const char *host, const char *target,
+ http_client_request_callback_t *callback, void *context)
+{
+ pool_t pool;
+ struct http_client_request *req;
+
+ pool = pool_alloconly_create("http client request", 2048);
+ req = p_new(pool, struct http_client_request, 1);
+ req->pool = pool;
+ req->refcount = 1;
+ req->client = client;
+ req->method = p_strdup(pool, method);
+ req->hostname = p_strdup(pool, host);
+ req->port = HTTP_DEFAULT_PORT;
+ req->target = p_strdup(pool, target);
+ req->callback = callback;
+ req->context = context;
+ req->headers = str_new(default_pool, 256);
+
+ req->state = HTTP_REQUEST_STATE_NEW;
+ return req;
+}
+
+void http_client_request_ref(struct http_client_request *req)
+{
+ req->refcount++;
+}
+
+void http_client_request_unref(struct http_client_request **_req)
+{
+ struct http_client_request *req = *_req;
+ struct http_client *client = req->client;
+
+ i_assert(req->refcount > 0);
+
+ if (--req->refcount > 0)
+ return;
+
+ /* only decrease pending request counter if this request was submitted */
+ if (req->state > HTTP_REQUEST_STATE_NEW)
+ req->client->pending_requests--;
+
+ http_client_request_debug(req, "Destroy (requests left=%d)",
+ client->pending_requests);
+
+ if (req->input != NULL)
+ i_stream_unref(&req->input);
+ str_free(&req->headers);
+ pool_unref(&req->pool);
+ *_req = NULL;
+}
+
+void http_client_request_set_port(struct http_client_request *req,
+ unsigned int port)
+{
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ req->port = port;
+}
+
+void http_client_request_set_ssl(struct http_client_request *req,
+ bool ssl)
+{
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ if (ssl) {
+ if (!req->ssl && req->port == HTTP_DEFAULT_PORT)
+ req->port = HTTPS_DEFAULT_PORT;
+ } else {
+ if (req->ssl && req->port == HTTPS_DEFAULT_PORT)
+ req->port = HTTP_DEFAULT_PORT;
+ }
+ req->ssl = ssl;
+}
+
+void http_client_request_set_urgent(struct http_client_request *req)
+{
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ req->urgent = TRUE;
+}
+
+void http_client_request_add_header(struct http_client_request *req,
+ const char *key, const char *value)
+{
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ str_printfa(req->headers, "%s: %s\r\n", key, value);
+}
+
+void http_client_request_set_payload(struct http_client_request *req,
+ struct istream *input, bool sync)
+{
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ i_assert(req->input == NULL);
+
+ i_stream_ref(input);
+ req->input = input;
+ if (i_stream_get_size(input, TRUE, &req->input_size) <= 0)
+ i_unreached(); //FIXME
+
+ /* prepare request payload sync using 100 Continue response from server */
+ if (req->input_size > 0 && sync) {
+ req->payload_sync = TRUE;
+ }
+}
+
+void http_client_request_submit(struct http_client_request *req)
+{
+ struct http_client_host *host;
+
+ i_assert(req->state == HTTP_REQUEST_STATE_NEW);
+ http_client_request_debug(req, "Submitted");
+
+ host = http_client_host_get(req->client, req->hostname);
+ http_client_host_submit_request(host, req);
+ req->state = HTTP_REQUEST_STATE_QUEUED;
+ req->client->pending_requests++;
+}
+
+int http_client_request_send_more(struct http_client_request *req)
+{
+ struct http_client_connection *conn = req->conn;
+ struct ostream *output = conn->conn.output;
+ int ret = 0;
+
+ i_assert(req->input != NULL);
+
+ o_stream_set_max_buffer_size(output, 0);
+ if (o_stream_send_istream(output, req->input) < 0)
+ ret = -1;
+ o_stream_set_max_buffer_size(output, (size_t)-1);
+
+ if (req->input->eof) {
+ if (req->input->v_offset != req->input_size) {
+ i_error("stream input size changed"); //FIXME
+ return -1;
+ }
+ i_stream_unref(&req->input);
+ req->state = HTTP_REQUEST_STATE_WAITING;
+ conn->output_locked = FALSE;
+ http_client_request_debug(req, "Sent all payload");
+
+ } else {
+ conn->output_locked = TRUE;
+ o_stream_set_flush_pending(output, TRUE);
+ http_client_request_debug(req, "Partially sent payload");
+ }
+ return ret;
+}
+
+int http_client_request_send(struct http_client_request *req)
+{
+ struct http_client_connection *conn = req->conn;
+ struct ostream *output = conn->conn.output;
+ string_t *rtext = t_str_new(256);
+ struct const_iovec iov[3];
+ int ret = 0;
+
+ i_assert(!req->conn->output_locked);
+
+ str_append(rtext, req->method);
+ str_append(rtext, " ");
+ str_append(rtext, req->target);
+ str_append(rtext, " HTTP/1.1\r\n");
+ str_append(rtext, "Host: ");
+ str_append(rtext, req->hostname);
+ if ((!req->ssl &&req->port != HTTP_DEFAULT_PORT) ||
+ (req->ssl && req->port != HTTPS_DEFAULT_PORT)) {
+ str_printfa(rtext, ":%u", req->port);
+ }
+ str_append(rtext, "\r\n");
+ if (req->payload_sync) {
+ str_append(rtext, "Expect: 100-continue\r\n");
+ }
+ if (req->input_size != 0) {
+ str_printfa(rtext, "Content-Length: %"PRIuUOFF_T"\r\n",
+ req->input_size);
+ }
+
+ iov[0].iov_base = str_data(rtext);
+ iov[0].iov_len = str_len(rtext);
+ iov[1].iov_base = str_data(req->headers);
+ iov[1].iov_len = str_len(req->headers);
+ iov[2].iov_base = "\r\n";
+ iov[2].iov_len = 2;
+
+ req->state = HTTP_REQUEST_STATE_PAYLOAD_OUT;
+ o_stream_cork(output);
+ if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0)
+ ret = -1;
+
+ http_client_request_debug(req, "Sent");
+
+ if (ret >= 0 && req->input_size != 0) {
+ if (!req->payload_sync) {
+ if (http_client_request_send_more(req) < 0)
+ ret = -1;
+ } else {
+ http_client_request_debug(req, "Waiting for 100-continue");
+ }
+ } else {
+ req->state = HTTP_REQUEST_STATE_WAITING;
+ conn->output_locked = FALSE;
+ }
+ o_stream_uncork(output);
+ return ret;
+}
+
+void http_client_request_callback(struct http_client_request *req,
+ struct http_response *response)
+{
+ http_client_request_callback_t *callback = req->callback;
+
+ req->state = HTTP_REQUEST_STATE_GOT_RESPONSE;
+
+ req->callback = NULL;
+ if (callback != NULL)
+ callback(response, req->context);
+}
+
+static void
+http_client_request_send_error(struct http_client_request *req,
+ unsigned int status, const char *error)
+{
+ http_client_request_callback_t *callback;
+
+ req->state = HTTP_REQUEST_STATE_ABORTED;
+
+ callback = req->callback;
+ req->callback = NULL;
+ if (callback != NULL) {
+ struct http_response response;
+
+ memset(&response, 0, sizeof(response));
+ response.status = status;
+ response.reason = error;
+ (void)callback(&response, req->context);
+ }
+}
+
+void http_client_request_error(struct http_client_request *req,
+ unsigned int status, const char *error)
+{
+ http_client_request_send_error(req, status, error);
+ http_client_request_unref(&req);
+}
+
+void http_client_request_abort(struct http_client_request **_req)
+{
+ struct http_client_request *req = *_req;
+
+ if (req->state >= HTTP_REQUEST_STATE_FINISHED)
+ return;
+ req->callback = NULL;
+ req->state = HTTP_REQUEST_STATE_ABORTED;
+ if (req->host != NULL)
+ http_client_host_drop_request(req->host, req);
+ http_client_request_unref(_req);
+}
+
+void http_client_request_finish(struct http_client_request **_req)
+{
+ struct http_client_request *req = *_req;
+
+ if (req->state >= HTTP_REQUEST_STATE_FINISHED)
+ return;
+
+ http_client_request_debug(req, "Finished");
+
+ req->callback = NULL;
+ req->state = HTTP_REQUEST_STATE_FINISHED;
+ http_client_request_unref(_req);
+}
+
+void http_client_request_redirect(struct http_client_request *req,
+ const char *location)
+{
+ struct http_url *url;
+ const char *error;
+ unsigned int newport;
+
+ /* parse URL */
+ if (http_url_parse(location, NULL, 0, &url, &error) < 0) {
+ http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+ t_strdup_printf("Invalid redirect location: %s", error));
+ return;
+ }
+
+ if (++req->redirects > req->client->set.max_redirects) {
+ if (req->client->set.max_redirects > 0) {
+ http_client_request_error(req,
+ HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+ t_strdup_printf("Redirected more than %d times",
+ req->client->set.max_redirects));
+ } else {
+ http_client_request_error(req,
+ HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+ "Redirect refused");
+ }
+ return;
+ }
+
+ newport = (url->have_port ? url->port : (url->have_ssl ? 443 : 80));
+
+ http_client_request_debug(req, "Redirecting to http://%s:%u%s",
+ url->host_name, newport, url->path);
+
+ // FIXME: handle literal IP specially (avoid duplicate parsing)
+ req->host = NULL;
+ req->conn = NULL;
+ req->hostname = p_strdup(req->pool, url->host_name);
+ req->port = newport;
+ req->target = p_strdup(req->pool, url->path);
+ req->ssl = url->have_ssl;
+
+ /* resubmit */
+ req->client->pending_requests--;
+ req->state = HTTP_REQUEST_STATE_NEW;
+ http_client_request_submit(req);
+}
+
+void http_client_request_resubmit(struct http_client_request *req)
+{
+ http_client_request_debug(req, "Resubmitting request");
+
+ req->conn = NULL;
+ req->peer = NULL;
+ req->state = HTTP_REQUEST_STATE_QUEUED;
+ http_client_host_submit_request(req->host, req);
+}
+
+void http_client_request_retry(struct http_client_request *req,
+ unsigned int status, const char *error)
+{
+ /* limit the number of attempts for each request */
+ if (++req->attempts >= req->client->set.max_attempts) {
+ /* return error */
+ http_client_request_error(req, status, error);
+ return;
+ }
+
+ http_client_request_debug(req, "Retrying (attempts=%d)", req->attempts);
+
+ /* resubmit */
+ http_client_request_resubmit(req);
+}
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "hash.h"
+#include "array.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "connection.h"
+#include "dns-lookup.h"
+#include "iostream-rawlog.h"
+#include "iostream-ssl.h"
+#include "http-url.h"
+#include "http-response-parser.h"
+
+#include "http-client-private.h"
+
+#define HTTP_DEFAULT_PORT 80
+#define HTTPS_DEFAULT_PORT 443
+
+/* FIXME: This implementation not yet finished. The essence works: it is
+ possible to submit requests through the client. Responses are dumped to
+ stdout
+
+ Structure so far:
+ Client - Acts much like a browser; it is not dedicated to a single host.
+ Client can accept requests to different hosts, which can be served
+ at different IPs. Redirects can be handled in the background by
+ making a new connection. Connections to new hosts are created once
+ needed for servicing a request.
+ Requests - Semantics are similar to imapc commands. Create a request,
+ optionally modify some aspects of it and finally submit it.
+ Hosts - We maintain a 'cache' of hosts for which we have looked up IPs.
+ Requests are first queued in the host struct on a per-port basis.
+ Peers - Group connections to the same ip/port (== peer_addr).
+ Connections - Actual connections to a server. Once a connection is ready to
+ handle requests, it claims a request from a host object. One
+ connection hand service multiple hosts and one host can have
+ multiple associated connections, possibly to different ips and
+ ports.
+
+ TODO: lots of cleanup, authentication, ssl, timeouts, rawlog etc.
+ */
+
+/*
+ * Logging
+ */
+
+static inline void
+http_client_debug(struct http_client *client,
+ const char *format, ...) ATTR_FORMAT(2, 3);
+
+static inline void
+http_client_debug(struct http_client *client,
+ const char *format, ...)
+{
+ va_list args;
+
+ va_start(args, format);
+ if (client->set.debug)
+ i_debug("http-client: %s", t_strdup_vprintf(format, args));
+ va_end(args);
+}
+
+/*
+ * Client
+ */
+
+struct http_client *http_client_init(const struct http_client_settings *set)
+{
+ struct http_client *client;
+ pool_t pool;
+
+ pool = pool_alloconly_create("http client", 1024);
+ client = p_new(pool, struct http_client, 1);
+ client->pool = pool;
+ client->set.dns_client_socket_path =
+ p_strdup(pool, set->dns_client_socket_path);
+ client->set.rawlog_dir = p_strdup(pool, set->rawlog_dir);
+ client->set.ssl_ca_dir = p_strdup(pool, set->ssl_ca_dir);
+ client->set.max_idle_time_msecs = set->max_idle_time_msecs;
+ client->set.max_parallel_connections =
+ (set->max_parallel_connections > 0 ? set->max_parallel_connections : 1);
+ client->set.max_pipelined_requests =
+ (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
+ client->set.max_attempts = set->max_attempts;
+ client->set.max_redirects = set->max_redirects;
+ client->set.debug = set->debug;
+
+ client->conn_list = http_client_connection_list_init();
+
+ hash_table_create(&client->hosts, default_pool, 0, str_hash, strcmp);
+ hash_table_create(&client->peers, default_pool, 0,
+ http_client_peer_addr_hash, http_client_peer_addr_cmp);
+
+ return client;
+}
+
+void http_client_deinit(struct http_client **_client)
+{
+ struct http_client *client = *_client;
+ struct hash_iterate_context *iter;
+ const char *hostname;
+ struct http_client_host *host;
+ const struct http_client_peer_addr *addr;
+ struct http_client_peer *peer;
+
+ /* free peers */
+ iter = hash_table_iterate_init(client->peers);
+ while (hash_table_iterate(iter, client->peers, &addr, &peer)) {
+ http_client_peer_free(&peer);
+ }
+ hash_table_iterate_deinit(&iter);
+ hash_table_destroy(&client->peers);
+
+ /* free hosts */
+ iter = hash_table_iterate_init(client->hosts);
+ while (hash_table_iterate(iter, client->hosts, &hostname, &host)) {
+ http_client_host_free(&host);
+ }
+ hash_table_iterate_deinit(&iter);
+ hash_table_destroy(&client->hosts);
+
+ connection_list_deinit(&client->conn_list);
+ pool_unref(&client->pool);
+ *_client = NULL;
+}
+
+void http_client_switch_ioloop(struct http_client *client)
+{
+ struct connection *_conn = client->conn_list->connections;
+
+ /* move connections */
+ /* FIXME: we wouldn't necessarily need to switch all of them
+ immediately, only those that have requests now. but also connections
+ that get new requests before ioloop is switched again.. */
+ for (; _conn != NULL; _conn = _conn->next) {
+ struct http_client_connection *conn =
+ (struct http_client_connection *)_conn;
+
+ http_client_connection_switch_ioloop(conn);
+ }
+
+ /* move dns lookups */
+ if (client->set.dns_client_socket_path != '\0') {
+ struct hash_iterate_context *iter;
+ struct http_client_host *host;
+ const char *hostname;
+
+ iter = hash_table_iterate_init(client->hosts);
+ while (hash_table_iterate(iter, client->hosts, &hostname, &host)) {
+ http_client_host_switch_ioloop(host);
+ }
+ hash_table_iterate_deinit(&iter);
+ }
+}
+
+void http_client_wait(struct http_client *client)
+{
+ struct ioloop *prev_ioloop = current_ioloop;
+
+ i_assert(client->ioloop == NULL);
+
+ client->ioloop = io_loop_create();
+ http_client_switch_ioloop(client);
+
+ do {
+ http_client_debug(client,
+ "Waiting for %d requests to finish", client->pending_requests);
+ io_loop_run(client->ioloop);
+ } while (client->pending_requests > 0);
+
+ http_client_debug(client, "All requests finished");
+
+ current_ioloop = prev_ioloop;
+ http_client_switch_ioloop(client);
+ current_ioloop = client->ioloop;
+ io_loop_destroy(&client->ioloop);
+}
+
--- /dev/null
+#ifndef HTTP_CLIENT_H
+#define HTTP_CLIENT_H
+
+struct http_response;
+
+struct http_client;
+struct http_client_request;
+
+enum http_client_request_error {
+ HTTP_CLIENT_REQUEST_ERROR_ABORTED = 9000,
+ HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED,
+ HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+ HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST,
+ HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
+ HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT,
+};
+
+struct http_client_settings {
+ const char *dns_client_socket_path;
+
+ const char *ssl_ca_dir;
+ const char *ssl_crypto_device;
+ bool ssl_allow_invalid_cert;
+ bool ssl_verify;
+
+ const char *rawlog_dir;
+
+ unsigned int max_idle_time_msecs;
+
+ /* maximum number of parallel connections per peer (default = 1) */
+ unsigned int max_parallel_connections;
+
+ /* maximum number of pipelined requests per connection (default = 1) */
+ unsigned int max_pipelined_requests;
+
+ /* maximum number of redirects for a request
+ (default = 0; redirects refused)
+ */
+ unsigned int max_redirects;
+
+ /* maximum number of attempts for a request */
+ unsigned int max_attempts;
+
+ bool debug;
+};
+
+typedef void
+http_client_request_callback_t(const struct http_response *response,
+ void *context);
+
+struct http_client *http_client_init(const struct http_client_settings *set);
+void http_client_deinit(struct http_client **_client);
+
+struct http_client_request *
+http_client_request(struct http_client *client,
+ const char *method, const char *host, const char *target,
+ http_client_request_callback_t *callback, void *context);
+#define http_client_request(client, method, host, target, callback, context) \
+ http_client_request(client, method, host, target + \
+ CALLBACK_TYPECHECK(callback, void (*)( \
+ const struct http_response *response, typeof(context))), \
+ (http_client_request_callback_t *)callback, context)
+
+void http_client_request_set_port(struct http_client_request *req,
+ unsigned int port);
+void http_client_request_set_ssl(struct http_client_request *req,
+ bool ssl);
+void http_client_request_set_urgent(struct http_client_request *req);
+
+void http_client_request_add_header(struct http_client_request *req,
+ const char *key, const char *value);
+void http_client_request_set_payload(struct http_client_request *req,
+ struct istream *input, bool sync);
+void http_client_request_submit(struct http_client_request *req);
+void http_client_request_abort(struct http_client_request **req);
+
+void http_client_switch_ioloop(struct http_client *client);
+
+/* blocks until all currently submitted requests are handled */
+void http_client_wait(struct http_client *client);
+
+#endif
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "istream.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "http-parser.h"
+
+#include "http-header-parser.h"
+
+enum http_header_parse_state {
+ HTTP_HEADER_PARSE_STATE_INIT = 0,
+ HTTP_HEADER_PARSE_STATE_NAME,
+ HTTP_HEADER_PARSE_STATE_COLON,
+ HTTP_HEADER_PARSE_STATE_OWS,
+ HTTP_HEADER_PARSE_STATE_CONTENT,
+ HTTP_HEADER_PARSE_STATE_CR,
+ HTTP_HEADER_PARSE_STATE_LF,
+ HTTP_HEADER_PARSE_STATE_NEW_LINE,
+ HTTP_HEADER_PARSE_STATE_LAST_LINE,
+ HTTP_HEADER_PARSE_STATE_EOH
+};
+
+struct http_header_parser {
+ struct istream *input;
+
+ const unsigned char *begin, *cur, *end;
+ const char *error;
+
+ string_t *name;
+ buffer_t *value_buf;
+
+ enum http_header_parse_state state;
+};
+
+// FIXME(Stephan): Add support for limiting maximum header size.
+
+struct http_header_parser *http_header_parser_init(struct istream *input)
+{
+ struct http_header_parser *parser;
+
+ parser = i_new(struct http_header_parser, 1);
+ parser->input = input;
+ parser->name = str_new(default_pool, 128);
+ parser->value_buf = buffer_create_dynamic(default_pool, 4096);
+
+ return parser;
+}
+
+void http_header_parser_deinit(struct http_header_parser **_parser)
+{
+ struct http_header_parser *parser = *_parser;
+
+ //i_stream_skip(ctx->input, ctx->skip);
+ buffer_free(&parser->value_buf);
+ str_free(&parser->name);
+ i_free(parser);
+ *_parser = NULL;
+}
+
+void http_header_parser_reset(struct http_header_parser *parser)
+{
+ parser->state = HTTP_HEADER_PARSE_STATE_INIT;
+}
+
+static int http_header_parse_name(struct http_header_parser *parser)
+{
+ const unsigned char *first = parser->cur;
+
+ /* field-name = token
+ token = 1*tchar
+ */
+ while (parser->cur < parser->end && http_char_is_token(*parser->cur))
+ parser->cur++;
+
+ str_append_n(parser->name, first, parser->cur-first);
+
+ if (parser->cur == parser->end)
+ return 0;
+ if (str_len(parser->name) == 0) {
+ parser->error = "Empty header field name";
+ return -1;
+ }
+ return 1;
+}
+
+static int http_header_parse_ows(struct http_header_parser *parser)
+{
+ /* OWS = *( SP / HTAB )
+ ; "optional" whitespace
+ */
+ while (parser->cur < parser->end &&
+ (*parser->cur == ' ' || *parser->cur == '\t'))
+ parser->cur++;
+ return (parser->cur == parser->end ? 0 : 1);
+}
+
+static int http_header_parse_content(struct http_header_parser *parser)
+{
+ const unsigned char *first = parser->cur;
+
+ /* field-content = *( HTAB / SP / VCHAR / obs-text )
+ */
+ while (parser->cur < parser->end && http_char_is_text(*parser->cur))
+ parser->cur++;
+
+ buffer_append(parser->value_buf, first, parser->cur-first);
+
+ if (parser->cur == parser->end)
+ return 0;
+ return 1;
+}
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+ if (c >= 0x20 && c < 0x7F)
+ return t_strdup_printf("'%c'", c);
+ return t_strdup_printf("0x%02x", c);
+}
+
+static int http_header_parse(struct http_header_parser *parser)
+{
+ int ret;
+
+ /* 'header' = *( header-field CRLF ) CRLF
+ header-field = field-name ":" OWS field-value BWS
+ field-name = token
+ field-value = *( field-content / obs-fold )
+ field-content = *( HTAB / SP / VCHAR / obs-text )
+ obs-fold = CRLF ( SP / HTAB )
+ ; obsolete line folding
+ ; see Section 3.2.2
+ */
+
+ for (;;) {
+ switch (parser->state) {
+ case HTTP_HEADER_PARSE_STATE_INIT:
+ buffer_set_used_size(parser->value_buf, 0);
+ str_truncate(parser->name, 0);
+ parser->state = HTTP_HEADER_PARSE_STATE_NAME;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_NAME:
+ if (http_char_is_token(*parser->cur)) {
+ if ((ret=http_header_parse_name(parser)) <= 0)
+ return ret;
+ } else if (str_len(parser->name) == 0) {
+ parser->state = HTTP_HEADER_PARSE_STATE_LAST_LINE;
+ break;
+ }
+ parser->state = HTTP_HEADER_PARSE_STATE_COLON;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_COLON:
+ if (*parser->cur != ':') {
+ parser->error = t_strdup_printf
+ ("Expected ':' after header field name '%s', but found %s",
+ str_sanitize(str_c(parser->name),64),
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ parser->cur++;
+ if (str_len(parser->name) == 0) {
+ parser->error = "Empty header field name";
+ return -1;
+ }
+ parser->state = HTTP_HEADER_PARSE_STATE_OWS;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_OWS:
+ if ((ret=http_header_parse_ows(parser)) <= 0)
+ return ret;
+ parser->state = HTTP_HEADER_PARSE_STATE_CONTENT;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_CONTENT:
+ if ((ret=http_header_parse_content(parser)) <= 0)
+ return ret;
+ parser->state = HTTP_HEADER_PARSE_STATE_CR;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_CR:
+ if (*parser->cur == '\r') {
+ parser->cur++;
+ }
+ parser->state = HTTP_HEADER_PARSE_STATE_LF;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_LF:
+ if (*parser->cur != '\n') {
+ parser->error = t_strdup_printf
+ ("Expected line end after header field '%s', but found %s",
+ str_sanitize(str_c(parser->name),64),
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ parser->cur++;
+ parser->state = HTTP_HEADER_PARSE_STATE_NEW_LINE;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_HEADER_PARSE_STATE_NEW_LINE:
+ if (*parser->cur == ' ' || *parser->cur == '\t') {
+ /* obs-fold */
+ buffer_append_c(parser->value_buf, ' ');
+ parser->state = HTTP_HEADER_PARSE_STATE_OWS;
+ break;
+ }
+ parser->state = HTTP_HEADER_PARSE_STATE_NAME;
+ return 1;
+ case HTTP_HEADER_PARSE_STATE_LAST_LINE:
+ if (*parser->cur == '\r') {
+ /* last CRLF */
+ parser->cur++;
+ parser->state = HTTP_HEADER_PARSE_STATE_EOH;
+ if (parser->cur == parser->end)
+ return 0;
+ break;
+ } else if (*parser->cur == '\n') {
+ /* header fully parsed */
+ parser->cur++;
+ parser->state = HTTP_HEADER_PARSE_STATE_EOH;
+ return 1;
+ }
+ parser->error = t_strdup_printf
+ ("Expected CRLF or header field name, but found %s",
+ _chr_sanitize(*parser->cur));
+ return -1;
+ case HTTP_HEADER_PARSE_STATE_EOH:
+ if (*parser->cur != '\n') {
+ parser->error = t_strdup_printf
+ ("Expected LF after CR at end of header, but found %s",
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ /* header fully parsed */
+ parser->cur++;
+ return 1;
+
+ default:
+ i_unreached();
+ }
+ }
+
+ i_unreached();
+ return -1;
+}
+
+int http_header_parse_next_field(struct http_header_parser *parser,
+ const char **name_r, const unsigned char **data_r, size_t *size_r,
+ const char **error_r)
+{
+ size_t size;
+ int ret;
+
+ while ((ret=i_stream_read_data
+ (parser->input, &parser->begin, &size, 0)) > 0) {
+ parser->cur = parser->begin;
+ parser->end = parser->cur + size;
+
+ if ((ret=http_header_parse(parser)) < 0) {
+ *error_r = parser->error;
+ return -1;
+ }
+
+ i_stream_skip(parser->input, parser->cur - parser->begin);
+
+ if (ret == 1) {
+ if (parser->state != HTTP_HEADER_PARSE_STATE_EOH) {
+ *name_r = str_c(parser->name);
+ *data_r = buffer_get_data(parser->value_buf, size_r);
+ parser->state = HTTP_HEADER_PARSE_STATE_INIT;
+ } else {
+ *name_r = NULL;
+ *data_r = NULL;
+ }
+ return 1;
+ }
+ }
+
+ i_assert(ret != -2);
+ if (ret < 0)
+ *error_r = "Stream error";
+ return ret;
+}
--- /dev/null
+#ifndef HTTP_HEADER_PARSER_H
+#define HTTP_HEADER_PARSER_H
+
+struct http_header_parser;
+
+struct http_header_parser *http_header_parser_init(struct istream *input);
+void http_header_parser_deinit(struct http_header_parser **_parser);
+
+void http_header_parser_reset(struct http_header_parser *parser);
+
+int http_header_parse_next_field(struct http_header_parser *parser,
+ const char **name_r, const unsigned char **data_r, size_t *size_r,
+ const char **error_r);
+
+#endif
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "strescape.h"
+#include "http-url.h"
+
+#include "http-parser.h"
+
+/*
+ Character definitions:
+
+ tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+ / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+ / DIGIT / ALPHA
+ ; any VCHAR, except special
+ special = "(" / ")" / "<" / ">" / "@" / ","
+ / ";" / ":" / "\" / DQUOTE / "/" / "["
+ / "]" / "?" / "=" / "{" / "}"
+ qdtext = OWS / %x21 / %x23-5B / %x5D-7E / obs-text
+ qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text
+ ctext = OWS / %x21-27 / %x2A-5B / %x5D-7E / obs-text
+ obs-text = %x80-FF
+ OWS = *( SP / HTAB )
+
+ Mapping
+ (1<<0) => tchar
+ (1<<1) => special
+ (1<<2) => %x21 / %x2A-5B / %x5D-7E
+ (1<<3) => %x23-29
+ (1<<4) => %x22-27
+ (1<<5) => HTAB / SP / obs-text
+ */
+
+const unsigned char _http_token_char_mask = (1<<0);
+const unsigned char _http_value_char_mask = (1<<0)|(1<<1);
+const unsigned char _http_text_char_mask = (1<<0)|(1<<1)|(1<<5);
+const unsigned char _http_qdtext_char_mask = (1<<2)|(1<<3)|(1<<5);
+const unsigned char _http_ctext_char_mask = (1<<2)|(1<<4)|(1<<5);
+
+const unsigned char _http_char_lookup[256] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, // 00
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 10
+ 32, 21, 18, 25, 25, 25, 25, 25, 10, 10, 5, 5, 6, 5, 5, 6, // 20
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, // 30
+ 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, // 40
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 2, 6, 5, 5, // 50
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, // 60
+ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 5, 6, 5, 0, // 70
+
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // 80
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // 90
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // A0
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // B0
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // C0
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // D0
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // E0
+ 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, // F0
+};
+
+/*
+ * HTTP value parsing
+ */
+
+void http_parser_init(struct http_parser *parser,
+ const unsigned char *data, size_t size)
+{
+ memset(parser, 0, sizeof(*parser));
+ parser->begin = data;
+ parser->cur = data;
+ parser->end = data + size;
+}
+
+void http_parse_ows(struct http_parser *parser)
+{
+ /* OWS = *( SP / HTAB ) */
+ if (parser->cur >= parser->end)
+ return;
+ while (parser->cur < parser->end &&
+ (parser->cur[0] == ' ' || parser->cur[0] == '\t')) {
+ parser->cur++;
+ }
+}
+
+int http_parse_token(struct http_parser *parser, const char **token_r)
+{
+ const unsigned char *first;
+
+ /* token = 1*tchar */
+
+ if (parser->cur >= parser->end || !http_char_is_token(*parser->cur))
+ return 0;
+
+ first = parser->cur++;
+ while (parser->cur < parser->end && http_char_is_token(*parser->cur))
+ parser->cur++;
+
+ *token_r = t_strndup(first, parser->cur - first);
+ return 1;
+}
+
+int http_parse_token_list_next(struct http_parser *parser,
+ const char **token_r)
+{
+ /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21;
+ Appendix B:
+
+ For compatibility with legacy list rules, recipients SHOULD accept
+ empty list elements. In other words, consumers would follow the list
+ productions:
+
+ #element => [ ( "," / element ) *( OWS "," [ OWS element ] ) ]
+ 1#element => *( "," OWS ) element *( OWS "," [ OWS element ] )
+ */
+
+ for (;;) {
+ if (http_parse_token(parser, token_r) > 0)
+ break;
+ http_parse_ows(parser);
+ if (parser->cur >= parser->end || parser->cur[0] != ',')
+ return 0;
+ parser->cur++;
+ http_parse_ows(parser);
+ }
+
+ return 1;
+}
+
+
+
+
--- /dev/null
+#ifndef HTTP_PARSER_H
+#define HTTP_PARSER_H
+
+/*
+ * Character definitions
+ */
+
+extern const unsigned char _http_token_char_mask;
+extern const unsigned char _http_value_char_mask;
+extern const unsigned char _http_text_char_mask;
+extern const unsigned char _http_qdtext_char_mask;
+extern const unsigned char _http_ctext_char_mask;
+
+extern const unsigned char _http_char_lookup[256];
+
+static inline bool http_char_is_token(unsigned char ch) {
+ return (_http_char_lookup[ch] & _http_token_char_mask) != 0;
+}
+
+static inline bool http_char_is_value(unsigned char ch) {
+ return (_http_char_lookup[ch] & _http_value_char_mask) != 0;
+}
+
+static inline bool http_char_is_text(unsigned char ch) {
+ return (_http_char_lookup[ch] & _http_text_char_mask) != 0;
+}
+
+static inline bool http_char_is_qdtext(unsigned char ch) {
+ return (_http_char_lookup[ch] & _http_qdtext_char_mask) != 0;
+}
+
+static inline bool http_char_is_ctext(unsigned char ch) {
+ return (_http_char_lookup[ch] & _http_ctext_char_mask) != 0;
+}
+
+/*
+ * HTTP value parsing
+ */
+
+struct http_parser {
+ const unsigned char *begin, *cur, *end;
+};
+
+void http_parser_init(struct http_parser *parser,
+ const unsigned char *data, size_t size);
+
+void http_parse_ows(struct http_parser *parser);
+
+int http_parse_token(struct http_parser *parser, const char **token_r);
+int http_parse_token_list_next(struct http_parser *parser,
+ const char **token_r);
+
+#endif
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strfuncs.h"
+#include "istream.h"
+#include "http-parser.h"
+#include "http-date.h"
+#include "http-header-parser.h"
+#include "http-transfer.h"
+
+#include "http-response-parser.h"
+
+#include <ctype.h>
+
+enum http_response_parser_state {
+ HTTP_RESPONSE_PARSE_STATE_INIT = 0,
+ HTTP_RESPONSE_PARSE_STATE_VERSION,
+ HTTP_RESPONSE_PARSE_STATE_SP1,
+ HTTP_RESPONSE_PARSE_STATE_STATUS,
+ HTTP_RESPONSE_PARSE_STATE_SP2,
+ HTTP_RESPONSE_PARSE_STATE_REASON,
+ HTTP_RESPONSE_PARSE_STATE_CR,
+ HTTP_RESPONSE_PARSE_STATE_LF,
+ HTTP_RESPONSE_PARSE_STATE_HEADER
+};
+
+struct http_response_parser {
+ struct istream *input;
+
+ const unsigned char *begin, *cur, *end;
+ const char *error;
+
+ string_t *strbuf;
+
+ enum http_response_parser_state state;
+ struct http_header_parser *header_parser;
+
+ uoff_t content_length;
+ const char *transfer_encoding;
+ struct istream *payload;
+
+ struct http_response *response;
+ pool_t response_pool;
+};
+
+struct http_response_parser *http_response_parser_init(struct istream *input)
+{
+ struct http_response_parser *parser;
+
+ parser = i_new(struct http_response_parser, 1);
+ parser->input = input;
+ parser->strbuf = str_new(default_pool, 128);
+ return parser;
+}
+
+void http_response_parser_deinit(struct http_response_parser **_parser)
+{
+ struct http_response_parser *parser = *_parser;
+
+ str_free(&parser->strbuf);
+ if (parser->header_parser != NULL)
+ http_header_parser_deinit(&parser->header_parser);
+ if (parser->response_pool != NULL)
+ pool_unref(&parser->response_pool);
+ if (parser->payload != NULL)
+ i_stream_unref(&parser->payload);
+ i_free(parser);
+}
+
+static void
+http_response_parser_restart(struct http_response_parser *parser)
+{
+ i_assert(parser->payload == NULL);
+ parser->content_length = 0;
+ parser->transfer_encoding = NULL;
+ str_truncate(parser->strbuf, 0);
+ if (parser->response_pool != NULL)
+ pool_unref(&parser->response_pool);
+ parser->response_pool = pool_alloconly_create("http_response", 2048);
+ parser->response = p_new(parser->response_pool, struct http_response, 1);
+ parser->response->date = (time_t)-1;
+ p_array_init(&parser->response->headers, parser->response_pool, 32);
+}
+
+static int http_response_parse_version(struct http_response_parser *parser)
+{
+ const unsigned char *first = parser->cur;
+ const char *p;
+
+ /* HTTP-version = HTTP-name "/" DIGIT "." DIGIT
+ HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive
+ */
+ while (parser->cur < parser->end && http_char_is_value(*parser->cur))
+ parser->cur++;
+
+ if (str_len(parser->strbuf) + (parser->cur-first) > 8)
+ return -1;
+
+ str_append_n(parser->strbuf, first, parser->cur-first);
+ if (parser->cur == parser->end)
+ return 0;
+
+ if (str_len(parser->strbuf) != 8)
+ return -1;
+ if (strncmp(str_c(parser->strbuf), "HTTP/",5) != 0)
+ return -1;
+ p = str_c(parser->strbuf) + 5;
+ if (!i_isdigit(*p))
+ return -1;
+ parser->response->version_major = *p - '0';
+ p++;
+ if (*(p++) != '.')
+ return -1;
+ if (!i_isdigit(*p))
+ return -1;
+ parser->response->version_minor = *p - '0';
+ return 1;
+}
+
+static int http_response_parse_status(struct http_response_parser *parser)
+{
+ const unsigned char *first = parser->cur;
+ const char *p;
+
+ /* status-code = 3DIGIT
+ */
+ while (parser->cur < parser->end && i_isdigit(*parser->cur))
+ parser->cur++;
+
+ if (str_len(parser->strbuf) + (parser->cur-first) > 3)
+ return -1;
+
+ str_append_n(parser->strbuf, first, parser->cur-first);
+ if (parser->cur == parser->end)
+ return 0;
+ if (str_len(parser->strbuf) != 3)
+ return -1;
+ p = str_c(parser->strbuf);
+ parser->response->status =
+ (p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0');
+ return 1;
+}
+
+static int http_response_parse_reason(struct http_response_parser *parser)
+{
+ const unsigned char *first = parser->cur;
+
+ /* reason-phrase = *( HTAB / SP / VCHAR / obs-text )
+ */
+ while (parser->cur < parser->end && http_char_is_text(*parser->cur))
+ parser->cur++;
+
+ str_append_n(parser->strbuf, first, parser->cur-first);
+
+ if (parser->cur == parser->end)
+ return 0;
+ parser->response->reason =
+ p_strdup(parser->response_pool, str_c(parser->strbuf));
+ return 1;
+}
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+ if (c >= 0x20 && c < 0x7F)
+ return t_strdup_printf("'%c'", c);
+ return t_strdup_printf("0x%02x", c);
+}
+
+static int http_response_parse(struct http_response_parser *parser)
+{
+ int ret;
+
+ /* status-line = HTTP-version SP status-code SP reason-phrase CRLF
+ status-code = 3DIGIT
+ reason-phrase = *( HTAB / SP / VCHAR / obs-text )
+ */
+
+ for (;;) {
+ switch (parser->state) {
+ case HTTP_RESPONSE_PARSE_STATE_INIT:
+ http_response_parser_restart(parser);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_VERSION;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_VERSION:
+ if ((ret=http_response_parse_version(parser)) <= 0) {
+ if (ret < 0)
+ parser->error = "Invalid HTTP version in response";
+ return ret;
+ }
+ str_truncate(parser->strbuf, 0);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_SP1;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_SP1:
+ if (*parser->cur != ' ') {
+ parser->error = t_strdup_printf
+ ("Expected ' ' after response version, but found %s",
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ parser->cur++;
+ str_truncate(parser->strbuf, 0);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_STATUS;
+ if (parser->cur >= parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_STATUS:
+ if ((ret=http_response_parse_status(parser)) <= 0) {
+ if (ret < 0)
+ parser->error = "Invalid HTTP status code in response";
+ return ret;
+ }
+ str_truncate(parser->strbuf, 0);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_SP2;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_SP2:
+ if (*parser->cur != ' ') {
+ parser->error = t_strdup_printf
+ ("Expected ' ' after response status code, but found %s",
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ parser->cur++;
+ str_truncate(parser->strbuf, 0);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_REASON;
+ if (parser->cur >= parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_REASON:
+ if ((ret=http_response_parse_reason(parser)) <= 0)
+ return ret;
+ str_truncate(parser->strbuf, 0);
+ parser->state = HTTP_RESPONSE_PARSE_STATE_CR;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_CR:
+ if (*parser->cur == '\r')
+ parser->cur++;
+ parser->state = HTTP_RESPONSE_PARSE_STATE_LF;
+ if (parser->cur == parser->end)
+ return 0;
+ /* fall through */
+ case HTTP_RESPONSE_PARSE_STATE_LF:
+ if (*parser->cur != '\n') {
+ parser->error = t_strdup_printf
+ ("Expected line end after response, but found %s",
+ _chr_sanitize(*parser->cur));
+ return -1;
+ }
+ parser->cur++;
+ parser->state = HTTP_RESPONSE_PARSE_STATE_HEADER;
+ return 1;
+ case HTTP_RESPONSE_PARSE_STATE_HEADER:
+ default:
+ i_unreached();
+ }
+ }
+
+ i_unreached();
+ return -1;
+}
+
+static int http_response_parse_status_line(struct http_response_parser *parser)
+{
+ size_t size;
+ int ret;
+
+ while ((ret = i_stream_read_data(parser->input,
+ &parser->begin, &size, 0)) > 0) {
+ parser->cur = parser->begin;
+ parser->end = parser->cur + size;
+
+ if ((ret = http_response_parse(parser)) < 0)
+ return -1;
+
+ i_stream_skip(parser->input, parser->cur - parser->begin);
+ if (ret > 0)
+ return 1;
+ }
+
+ i_assert(ret != -2);
+ if (ret < 0) {
+ if (parser->input->eof && parser->state == HTTP_RESPONSE_PARSE_STATE_INIT)
+ return 0;
+ parser->error = "Stream error";
+ return -1;
+ }
+ return 0;
+}
+
+static int
+http_response_parse_header(struct http_response_parser *parser,
+ const char *name, const unsigned char *data, size_t size)
+{
+ struct http_response_header *hdr;
+ struct http_parser hparser;
+
+ hdr = array_append_space(&parser->response->headers);
+ hdr->key = p_strdup(parser->response_pool, name);
+ hdr->value = p_strndup(parser->response_pool, data, size);
+ hdr->size = size;
+
+ switch (name[0]) {
+ case 'C': case 'c':
+ if (strcasecmp(name, "Connection") == 0) {
+ const char *option;
+
+ /* Connection = 1#connection-option
+ connection-option = token
+ */
+ http_parser_init(&hparser, data, size);
+ for (;;) {
+ if (http_parse_token_list_next(&hparser, &option) <= 0)
+ break;
+ if (strcasecmp(option, "close") == 0) {
+ parser->response->connection_close = TRUE;
+ break; // not interested in any other options
+ }
+ }
+ return 1;
+ }
+ if (strcasecmp(name, "Content-Length") == 0) {
+ /* Content-Length = 1*DIGIT */
+ if (str_to_uoff(hdr->value, &parser->content_length) < 0) {
+ parser->error = "Invalid Content-Length header";
+ return -1;
+ }
+ return 1;
+ }
+ break;
+ case 'D': case 'd':
+ if (strcasecmp(name, "Date") == 0) {
+ /* Date = HTTP-date */
+ (void)http_date_parse(data, size, &parser->response->date);
+ return 1;
+ }
+ break;
+ case 'L': case 'l':
+ if (strcasecmp(name, "Location") == 0) {
+ /* Location = URI-reference (not parsed here) */
+ parser->response->location =
+ p_strndup(parser->response_pool, data, size);
+ return 1;
+ }
+ break;
+ case 'T': case 't':
+ if (strcasecmp(name, "Transfer-Encoding") == 0) {
+ /* Transfer-Encoding = 1#transfer-coding */
+ parser->transfer_encoding = hdr->value;
+ return 1;
+ }
+ break;
+ default:
+ break;
+ }
+ return 1;
+}
+
+int http_response_parse_next(struct http_response_parser *parser,
+ bool no_payload, struct http_response **response_r,
+ const char **error_r)
+{
+ struct http_parser hparser;
+ const char *field_name, *error;
+ const unsigned char *field_data;
+ size_t field_size;
+ int ret;
+
+ /* make sure we finished streaming payload from previous response
+ before we continue. */
+ if (parser->payload != NULL) {
+ struct istream *payload = parser->payload;
+
+ i_assert(parser->state == HTTP_RESPONSE_PARSE_STATE_INIT);
+
+ if (!payload->eof) {
+ while ((ret=i_stream_read(payload)) > 0) {
+ i_stream_skip(payload, i_stream_get_data_size(payload));
+ }
+ if (ret == 0)
+ return 0;
+ if (ret < 0 && !payload->eof) {
+ *error_r = "Stream error while skipping payload";
+ return -1;
+ }
+ }
+
+ if (payload->eof) {
+ i_stream_unref(&parser->payload);
+ parser->payload = NULL;
+ }
+ }
+
+ /* HTTP-message = start-line
+ *( header-field CRLF )
+ CRLF
+ [ message-body ]
+ */
+
+ /* start-line */
+ if (parser->state != HTTP_RESPONSE_PARSE_STATE_HEADER) {
+ if ((ret=http_response_parse_status_line(parser)) <= 0) {
+ *error_r = parser->error;
+ return ret;
+ }
+ }
+
+ /* *( header-field CRLF ) CRLF */
+ if (parser->header_parser == NULL)
+ parser->header_parser = http_header_parser_init(parser->input);
+ else
+ http_header_parser_reset(parser->header_parser);
+
+ while ((ret=http_header_parse_next_field
+ (parser->header_parser, &field_name, &field_data, &field_size, &error)) > 0) {
+ if (field_name == NULL) break;
+ if ((ret=http_response_parse_header
+ (parser, field_name, field_data, field_size)) < 0) {
+ *error_r = parser->error;
+ return -1;
+ }
+ }
+
+ if (ret <= 0) {
+ if (ret < 0)
+ *error_r = t_strdup_printf("Failed to parse response header: %s", error);
+ return ret;
+ }
+
+ /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+ Section 3.3.2:
+
+ A server MUST NOT send a Content-Length header field in any response
+ with a status code of 1xx (Informational) or 204 (No Content). [...]
+ */
+ if ((parser->response->status / 100 == 1 ||
+ parser->response->status == 204) && parser->content_length > 0) {
+ *error_r = t_strdup_printf(
+ "Unexpected Content-Length header field for %u response "
+ "(length=%"PRIuUOFF_T")", parser->response->status,
+ parser->content_length);
+ return -1;
+ }
+
+ /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21
+ Section 3.3.3:
+
+ Any response to a HEAD request and any response with a 1xx
+ (Informational), 204 (No Content), or 304 (Not Modified) status
+ code is always terminated by the first empty line after the
+ header fields, regardless of the header fields present in the
+ message, and thus cannot contain a message body.
+ */
+ if (parser->response->status / 100 == 1 || parser->response->status == 204
+ || parser->response->status == 304) { // HEAD is handled in caller
+ no_payload = TRUE;
+ }
+
+ if (!no_payload) {
+ /* [ message-body ] */
+ if (parser->content_length > 0) {
+ /* Got explicit message size from Content-Length: header */
+ parser->payload = parser->response->payload =
+ i_stream_create_limit(parser->input, parser->content_length);
+ } else if (parser->transfer_encoding != NULL) {
+ const char *tenc;
+
+ /* Transfer-Encoding = 1#transfer-coding
+ transfer-coding = "chunked" / "compress" / "deflate" / "gzip"
+ / transfer-extension ; [FIXME]
+ transfer-extension = token *( OWS ";" OWS transfer-parameter )
+ */
+ // FIXME: parse this directly when the header is parsed
+ http_parser_init(&hparser,
+ (const unsigned char *)parser->transfer_encoding,
+ strlen(parser->transfer_encoding));
+ for (;;) {
+ if (http_parse_token_list_next(&hparser, &tenc) <= 0)
+ break;
+ if (strcasecmp(tenc, "chunked") == 0) {
+ parser->payload = parser->response->payload =
+ http_transfer_chunked_istream_create(parser->input);
+ break; // FIXME
+ } else {
+ *error_r = t_strdup_printf(
+ "Unkown Transfer-Encoding `%s' for %u response",
+ tenc, parser->response->status);
+ return -1;
+ }
+ }
+ }
+ }
+
+ parser->state = HTTP_RESPONSE_PARSE_STATE_INIT;
+ *response_r = parser->response;
+ return 1;
+}
--- /dev/null
+#ifndef HTTP_RESPONSE_PARSER_H
+#define HTTP_RESPONSE_PARSER_H
+
+struct http_response_header {
+ const char *key;
+ const char *value;
+ size_t size;
+};
+
+struct http_response {
+ unsigned char version_major;
+ unsigned char version_minor;
+
+ unsigned int status;
+
+ const char *reason;
+ const char *location;
+
+ time_t date;
+ struct istream *payload;
+
+ ARRAY(struct http_response_header) headers;
+
+ unsigned int connection_close:1;
+};
+
+struct http_response_parser;
+
+struct http_response_parser *
+http_response_parser_init(struct istream *input);
+void http_response_parser_deinit(struct http_response_parser **_parser);
+
+int http_response_parse_next(struct http_response_parser *parser,
+ bool no_payload, struct http_response **response_r,
+ const char **error_r);
+
+#endif
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream-private.h"
+#include "ostream-private.h"
+#include "http-parser.h"
+#include "http-header-parser.h"
+
+#include "http-transfer.h"
+
+/*
+ * Chunked input stream
+ */
+
+enum http_transfer_chunked_parse_state {
+ HTTP_CHUNKED_PARSE_STATE_INIT,
+ HTTP_CHUNKED_PARSE_STATE_SIZE,
+ HTTP_CHUNKED_PARSE_STATE_EXT,
+ HTTP_CHUNKED_PARSE_STATE_EXT_NAME,
+ HTTP_CHUNKED_PARSE_STATE_EXT_EQ,
+ HTTP_CHUNKED_PARSE_STATE_EXT_VALUE,
+ HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING,
+ HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE,
+ HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN,
+ HTTP_CHUNKED_PARSE_STATE_CR,
+ HTTP_CHUNKED_PARSE_STATE_LF,
+ HTTP_CHUNKED_PARSE_STATE_DATA,
+ HTTP_CHUNKED_PARSE_STATE_DATA_READY,
+ HTTP_CHUNKED_PARSE_STATE_DATA_CR,
+ HTTP_CHUNKED_PARSE_STATE_DATA_LF,
+ HTTP_CHUNKED_PARSE_STATE_TRAILER,
+ HTTP_CHUNKED_PARSE_STATE_FINISHED,
+};
+
+struct http_transfer_chunked_istream {
+ struct istream_private istream;
+ struct stat statbuf;
+
+ const unsigned char *begin, *cur, *end;
+ enum http_transfer_chunked_parse_state state;
+ unsigned int parsed_chars;
+
+ uoff_t chunk_size, chunk_v_offset, chunk_pos;
+ const char *error;
+
+ struct http_header_parser *header_parser;
+
+ unsigned int finished:1;
+};
+
+/* Chunk parser */
+
+static inline const char *_chr_sanitize(unsigned char c)
+{
+ if (c >= 0x20 && c < 0x7F)
+ return t_strdup_printf("'%c'", c);
+ return t_strdup_printf("0x%02x", c);
+}
+
+static int http_transfer_chunked_parse_size
+(struct http_transfer_chunked_istream *tcstream)
+{
+ uoff_t size = 0, prev;
+
+ /* chunk-size = 1*HEXDIG */
+
+ while (tcstream->cur < tcstream->end) {
+ prev = tcstream->chunk_size;
+
+ if (*tcstream->cur >= '0' && *tcstream->cur <= '9')
+ size = *tcstream->cur-'0';
+ else if (*tcstream->cur >= 'A' && *tcstream->cur <= 'F')
+ size = *tcstream->cur-'A' + 10;
+ else if (*tcstream->cur >= 'a' && *tcstream->cur <= 'f')
+ size = *tcstream->cur-'a' + 10;
+ else {
+ if (tcstream->parsed_chars == 0) {
+ tcstream->error = t_strdup_printf(
+ "Expected chunk size digit, but found %s",
+ _chr_sanitize(*tcstream->cur));
+ return -1;
+ }
+ tcstream->parsed_chars = 0;
+ return 1;
+ }
+ tcstream->chunk_size <<= 4;
+ tcstream->chunk_size += size;
+ if (tcstream->chunk_size < prev) {
+ tcstream->error = "Chunk size exceeds integer limit";
+ return -1;
+ }
+ tcstream->parsed_chars++;
+ tcstream->cur++;
+ }
+
+ return 0;
+}
+
+static int http_transfer_chunked_skip_token
+(struct http_transfer_chunked_istream *tcstream)
+{
+ const unsigned char *first = tcstream->cur;
+
+ /* token = 1*tchar */
+ while (tcstream->cur < tcstream->end && http_char_is_token(*tcstream->cur))
+ tcstream->cur++;
+
+ tcstream->parsed_chars += (tcstream->cur-first);
+ if (tcstream->cur == tcstream->end)
+ return 0;
+ if (tcstream->parsed_chars == 0)
+ return -1;
+ return 1;
+}
+
+static int http_transfer_chunked_skip_qdtext
+(struct http_transfer_chunked_istream *tcstream)
+{
+ /* qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text */
+ while (tcstream->cur < tcstream->end && http_char_is_qdtext(*tcstream->cur))
+ tcstream->cur++;
+ if (tcstream->cur == tcstream->end)
+ return 0;
+ return 1;
+}
+
+static int
+http_transfer_chunked_parse(struct http_transfer_chunked_istream *tcstream)
+{
+ int ret;
+
+ /* http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-21;
+ Section 4.1:
+
+ chunked-body = *chunk
+ last-chunk
+ trailer-part
+ CRLF
+
+ chunk = chunk-size [ chunk-ext ] CRLF
+ chunk-data CRLF
+ chunk-size = 1*HEXDIG
+ last-chunk = 1*("0") [ chunk-ext ] CRLF
+
+ chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] )
+ chunk-ext-name = token
+ chunk-ext-val = token / quoted-str-nf
+ chunk-data = 1*OCTET ; a sequence of chunk-size octets
+ trailer-part = *( header-field CRLF )
+
+ quoted-str-nf = DQUOTE *( qdtext-nf / quoted-pair ) DQUOTE
+ ; like quoted-string, but disallowing line folding
+ qdtext-nf = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text
+ quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text )
+ */
+
+ for (;;) {
+ switch (tcstream->state) {
+ case HTTP_CHUNKED_PARSE_STATE_INIT:
+ tcstream->chunk_size = 0;
+ tcstream->chunk_pos = 0;
+ tcstream->parsed_chars = 0;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_SIZE;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_SIZE:
+ if ((ret=http_transfer_chunked_parse_size(tcstream)) <= 0)
+ return ret;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_EXT:
+ if (*tcstream->cur != ';') {
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_CR;
+ break;
+ }
+ /* chunk-ext */
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_NAME;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_EXT_NAME:
+ /* chunk-ext-name = token */
+ if ((ret=http_transfer_chunked_skip_token(tcstream)) <= 0) {
+ if (ret < 0)
+ tcstream->error = "Invalid chunked extension name";
+ return ret;
+ }
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_EQ;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_EXT_EQ:
+ if (*tcstream->cur != '=') {
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+ break;
+ }
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE:
+ /* chunk-ext-val = token / quoted-str-nf */
+ if (*tcstream->cur != '"') {
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN;
+ break;
+ }
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING:
+ for (;;) {
+ if (*tcstream->cur == '"') {
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ break;
+ } else if ((ret=http_transfer_chunked_skip_qdtext(tcstream)) <= 0) {
+ if (ret < 0)
+ tcstream->error = "Invalid chunked extension value";
+ return ret;
+ } else if (*tcstream->cur == '\\') {
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ break;
+ } else {
+ tcstream->error = t_strdup_printf(
+ "Invalid character %s in chunked extension value string",
+ _chr_sanitize(*tcstream->cur));
+ return -1;
+ }
+ }
+ break;
+ case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_ESCAPE:
+ /* ( HTAB / SP / VCHAR / obs-text ) */
+ if (!http_char_is_text(*tcstream->cur)) {
+ tcstream->error = t_strdup_printf(
+ "Escaped invalid character %s in chunked extension value string",
+ _chr_sanitize(*tcstream->cur));
+ return -1;
+ }
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_STRING;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ break;
+ case HTTP_CHUNKED_PARSE_STATE_EXT_VALUE_TOKEN:
+ if ((ret=http_transfer_chunked_skip_token(tcstream)) <= 0) {
+ if (ret < 0)
+ tcstream->error = "Invalid chunked extension value";
+ return ret;
+ }
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_EXT;
+ break;
+ case HTTP_CHUNKED_PARSE_STATE_CR:
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_LF;
+ if (*tcstream->cur == '\r') {
+ tcstream->cur++;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ }
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_LF:
+ if (*tcstream->cur != '\n') {
+ tcstream->error = t_strdup_printf(
+ "Expected new line after chunk size, but found %s",
+ _chr_sanitize(*tcstream->cur));
+ return -1;
+ }
+ tcstream->cur++;
+ if (tcstream->chunk_size > 0)
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA;
+ else
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_TRAILER;
+ return 1;
+ case HTTP_CHUNKED_PARSE_STATE_DATA_READY:
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_DATA_CR:
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_LF;
+ if (*tcstream->cur == '\r') {
+ tcstream->cur++;
+ if (tcstream->cur >= tcstream->end)
+ return 0;
+ }
+ /* fall through */
+ case HTTP_CHUNKED_PARSE_STATE_DATA_LF:
+ if (*tcstream->cur != '\n') {
+ tcstream->error = t_strdup_printf(
+ "Expected new line after chunk data, but found %s",
+ _chr_sanitize(*tcstream->cur));
+ return -1;
+ }
+ tcstream->cur++;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_INIT;
+ break;
+ default:
+ i_unreached();
+ }
+ }
+
+ i_unreached();
+ return -1;
+}
+
+static int http_transfer_chunked_parse_next(
+ struct http_transfer_chunked_istream *tcstream)
+{
+ struct istream *input = tcstream->istream.parent;
+ size_t size;
+ int ret;
+
+ while ((ret=i_stream_read_data
+ (input, &tcstream->begin, &size, 0)) > 0) {
+ tcstream->cur = tcstream->begin;
+ tcstream->end = tcstream->cur + size;
+
+ if ((ret=http_transfer_chunked_parse(tcstream)) < 0)
+ return -1;
+
+ i_stream_skip(input, tcstream->cur - tcstream->begin);
+
+ if (ret > 0) {
+ if (tcstream->state == HTTP_CHUNKED_PARSE_STATE_DATA)
+ tcstream->chunk_v_offset = input->v_offset;
+ return ret;
+ }
+ }
+
+ i_assert(ret != -2);
+ return ret;
+}
+
+static ssize_t
+http_transfer_chunked_istream_read_data(
+ struct http_transfer_chunked_istream *tcstream)
+{
+ struct istream_private *stream = &tcstream->istream;
+ const unsigned char *data;
+ size_t size, avail;
+ ssize_t ret = 0;
+
+ if (tcstream->chunk_pos >= tcstream->chunk_size) {
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY;
+ return 0;
+ }
+
+ // FIXME: is this even necessary?
+ i_stream_seek(stream->parent, tcstream->chunk_v_offset + tcstream->chunk_pos);
+
+ /* read from parent if necessary */
+ data = i_stream_get_data(stream->parent, &size);
+ if (size == 0) {
+ ret = i_stream_read(stream->parent);
+ if (ret <= 0 && (ret != -2 || stream->skip == 0)) {
+ if ( stream->parent->eof && stream->parent->stream_errno == 0 ) {
+ /* unexpected EOF */
+ tcstream->error = "Unexpected end of payload";
+ stream->istream.stream_errno = EIO;
+ } else {
+ /* parent stream error */
+ tcstream->error = "Stream error";
+ stream->istream.stream_errno = stream->parent->stream_errno;
+ }
+ return ret;
+ }
+ data = i_stream_get_data(stream->parent, &size);
+ i_assert(size != 0);
+ }
+
+ size = size > (tcstream->chunk_size - tcstream->chunk_pos) ?
+ (tcstream->chunk_size - tcstream->chunk_pos) : size;
+
+ /* Allocate buffer space */
+ if (!i_stream_try_alloc(stream, size, &avail))
+ return -2;
+
+ /* Copy payload */
+ size = size > avail ? avail : size;
+ memcpy(&stream->w_buffer[stream->pos], data, size);
+
+ i_stream_skip(stream->parent, size);
+
+ tcstream->chunk_pos += size;
+ if (tcstream->chunk_pos >= tcstream->chunk_size)
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_DATA_READY;
+
+ if ( ret < 0 ) {
+ stream->pos = stream->pos+size;
+ return ret;
+ }
+
+ ret = size;
+ stream->pos = stream->pos+size;
+ return ret;
+}
+
+static int http_transfer_chunked_parse_trailer(
+ struct http_transfer_chunked_istream *tcstream)
+{
+ const char *field_name, *error;
+ const unsigned char *field_data;
+ size_t field_size;
+ int ret;
+
+ if (tcstream->header_parser == NULL) {
+ tcstream->header_parser = http_header_parser_init(tcstream->istream.parent);
+ }
+
+ while ((ret=http_header_parse_next_field(tcstream->header_parser,
+ &field_name, &field_data, &field_size, &error)) > 0) {
+ if (field_name == NULL) break;
+ }
+
+ if (ret <= 0) {
+ if (ret < 0) {
+ tcstream->error = t_strdup_printf
+ ("Failed to parse chunked trailer: %s", error);
+ }
+ return ret;
+ }
+ return 1;
+}
+
+static ssize_t
+http_transfer_chunked_istream_read(struct istream_private *stream)
+{
+ struct http_transfer_chunked_istream *tcstream =
+ (struct http_transfer_chunked_istream *)stream;
+ ssize_t ret = 0;
+
+ for (;;) {
+ switch (tcstream->state) {
+ case HTTP_CHUNKED_PARSE_STATE_FINISHED:
+ tcstream->istream.istream.eof = TRUE;
+ return -1;
+ case HTTP_CHUNKED_PARSE_STATE_DATA:
+ if ((ret=http_transfer_chunked_istream_read_data(tcstream)) != 0)
+ return ret;
+ if (tcstream->state != HTTP_CHUNKED_PARSE_STATE_DATA_READY)
+ return 0;
+ break;
+ case HTTP_CHUNKED_PARSE_STATE_TRAILER:
+ if ((ret=http_transfer_chunked_parse_trailer(tcstream)) <= 0)
+ return ret;
+ tcstream->state = HTTP_CHUNKED_PARSE_STATE_FINISHED;
+ tcstream->istream.istream.eof = TRUE;
+ return -1;
+ default:
+ if ((ret=http_transfer_chunked_parse_next(tcstream)) <= 0)
+ return ret;
+ }
+ }
+
+ return -1;
+}
+
+struct istream *
+http_transfer_chunked_istream_create(struct istream *input)
+{
+ struct http_transfer_chunked_istream *tcstream;
+
+ tcstream = i_new(struct http_transfer_chunked_istream, 1);
+
+ tcstream->istream.max_buffer_size =
+ input->real_stream->max_buffer_size;
+
+ tcstream->istream.read = http_transfer_chunked_istream_read;
+
+ tcstream->istream.istream.readable_fd = FALSE;
+ tcstream->istream.istream.blocking = input->blocking;
+ tcstream->istream.istream.seekable = FALSE;
+ return i_stream_create(&tcstream->istream, input, i_stream_get_fd(input));
+}
+
+
+
+
--- /dev/null
+#ifndef HTTP_TRANSFER_H
+#define HTTP_TRANSFER_H
+
+struct istream *
+ http_transfer_chunked_istream_create(struct istream *input);
+
+#endif
+
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "write-full.h"
+#include "http-response-parser.h"
+#include "http-client.h"
+
+struct http_test_request {
+ struct io *io;
+ struct istream *payload;
+};
+
+static void payload_input(struct http_test_request *req)
+{
+ const unsigned char *data;
+ size_t size;
+ int ret;
+
+ /* read payload */
+ while ((ret=i_stream_read_data(req->payload, &data, &size, 0)) > 0) {
+ i_info("DEBUG: got data (size=%d)", (int)size);
+ //write_full(1, data, size);
+ i_stream_skip(req->payload, size);
+ }
+
+ if (ret == 0) {
+ i_info("DEBUG: REQUEST: NEED MORE DATA");
+ /* we will be called again for this request */
+ } else {
+ if (req->payload->stream_errno != 0)
+ i_error("REQUEST PAYLOAD READ ERROR: %m");
+ else
+ i_info("DEBUG: REQUEST: Finished");
+ io_remove(&req->io);
+ i_stream_unref(&req->payload);
+ i_free(req);
+ }
+}
+
+static void
+got_request_response(const struct http_response *response,
+ struct http_test_request *req)
+{
+ if (response == NULL) {
+ /* request failed */
+ i_free(req);
+ i_error("HTTP REQUEST FAILED");
+ return;
+ }
+
+ if (response->status / 100 != 2) {
+ i_error("HTTP Request failed: %s", response->reason);
+ i_free(req);
+ /* payload (if any) is skipped implicitly */
+ return;
+ }
+
+ i_info("DEBUG: REQUEST SUCCEEDED: %s", response->reason);
+
+ if (response->payload == NULL) {
+ i_free(req);
+ return;
+ }
+
+ i_info("DEBUG: REQUEST: Got payload");
+ i_stream_ref(response->payload);
+ req->payload = response->payload;
+ req->io = io_add(i_stream_get_fd(response->payload), IO_READ,
+ payload_input, req);
+ payload_input(req);
+}
+
+static const char *test_query1 = "action=fullsearch&context=180&value=sieve&titlesearch=Titles";
+static const char *test_query2 = "action=fullsearch&context=180&value=pigeonhole&titlesearch=Titles";
+
+int main(void)
+{
+ struct http_client_settings http_set;
+ struct http_client *http_client;
+ struct http_client_request *http_req;
+ struct ioloop *ioloop;
+ struct http_test_request *test_req;
+ struct istream *post_payload;
+
+ memset(&http_set, 0, sizeof(http_set));
+ http_set.dns_client_socket_path = "/var/run/dovecot/dns-client";
+ http_set.debug = TRUE;
+ http_set.ssl_ca_dir = "/etc/ssl/certs";
+ http_set.max_idle_time_msecs = 5*1000;
+ http_set.max_parallel_connections = 4;
+ http_set.max_pipelined_requests = 4;
+ http_set.max_redirects = 1;
+ http_set.max_attempts = 1;
+ http_set.debug = TRUE;
+ http_set.rawlog_dir = "/tmp/http-test";
+
+ lib_init();
+
+ ioloop = io_loop_create();
+ io_loop_set_running(ioloop);
+
+ // JigSAW is useful for testing: http://jigsaw.w3.org/HTTP/
+ // POST test server is useful for POST testing: http://posttestserver.com
+
+ http_client = http_client_init(&http_set);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/download.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "jigsaw.w3.org", "/HTTP/300/301.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/frop.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "jigsaw.w3.org", "/HTTP/300/307.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/documentation.html",
+ got_request_response, test_req);
+ http_client_request_set_urgent(http_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "jigsaw.w3.org", "/HTTP/300/302.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "POST", "posttestserver.com", "/post.php",
+ got_request_response, test_req);
+ post_payload = i_stream_create_from_data
+ ((unsigned char *)test_query1, strlen(test_query1));
+ http_client_request_set_payload(http_req, post_payload, TRUE);
+ i_stream_unref(&post_payload);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "POST", "posttestserver.com", "/post.php",
+ got_request_response, test_req);
+ post_payload = i_stream_create_from_data
+ ((unsigned char *)test_query2, strlen(test_query2));
+ http_client_request_set_payload(http_req, post_payload, TRUE);
+ i_stream_unref(&post_payload);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/",
+ got_request_response, test_req);
+ http_client_request_set_port(http_req, 81);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "HEAD", "pigeonhole.dovecot.org", "/download.html",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/",
+ got_request_response, test_req);
+ http_client_request_set_ssl(http_req, TRUE);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/download.html",
+ got_request_response, test_req);
+ http_client_request_set_ssl(http_req, TRUE);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "pigeonhole.dovecot.org", "/documentation.html",
+ got_request_response, test_req);
+ http_client_request_set_ssl(http_req, TRUE);
+ http_client_request_submit(http_req);
+ http_client_request_abort(&http_req);
+ i_free(test_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "wiki2.dovecot.org", "/Pigeonhole",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ test_req = i_new(struct http_test_request, 1);
+ http_req = http_client_request(http_client,
+ "GET", "jigsaw.w3.org", "/HTTP/ChunkedScript",
+ got_request_response, test_req);
+ http_client_request_submit(http_req);
+
+ http_client_wait(http_client);
+
+ http_client_deinit(&http_client);
+
+ io_loop_destroy(&ioloop);
+ lib_deinit();
+}
{ NULL, NULL }
};
+static struct http_header_parse_result valid_header_parse_result4[] = {
+ { NULL, NULL }
+};
+
static const struct http_header_parse_test valid_header_parse_tests[] = {
{ .header =
"Date: Sat, 06 Oct 2012 16:01:44 GMT\r\n"
"Content-Type: text/html; charset=iso-8859-1\r\n"
"\r\n",
.fields = valid_header_parse_result3
+ },{
+ .header =
+ "\r\n",
+ .fields = valid_header_parse_result4
}
};
for (i = 0; i < valid_header_parse_test_count; i++) T_BEGIN {
struct istream *input;
struct http_header_parser *parser;
- const char *header, *field_name;
+ const char *header, *field_name, *error;
const unsigned char *field_data;
size_t field_size;
int ret;
j = 0;
while ((ret=http_header_parse_next_field
- (parser, &field_name, &field_data, &field_size)) > 0) {
+ (parser, &field_name, &field_data, &field_size, &error)) > 0) {
const struct http_header_parse_result *result;
const char *field_value;
for (i = 0; i < invalid_header_parse_test_count; i++) T_BEGIN {
struct istream *input;
struct http_header_parser *parser;
- const char *header, *field_name;
+ const char *header, *field_name, *error;
const unsigned char *field_data;
size_t field_size;
int ret;
-
+
header = invalid_header_parse_tests[i];
input = i_stream_create_from_data(header, strlen(header));
parser = http_header_parser_init(input);
test_begin(t_strdup_printf("http header invalid [%d]", i));
while ((ret=http_header_parse_next_field
- (parser, &field_name, &field_data, &field_size)) > 0) {
+ (parser, &field_name, &field_data, &field_size, &error)) > 0) {
if (field_name == NULL) break;
}
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "istream.h"
+#include "ostream.h"
+#include "test-common.h"
+#include "http-response-parser.h"
+
+#include <time.h>
+
+struct http_response_parse_test {
+ const char *response;
+ unsigned char version_major;
+ unsigned char version_minor;
+ unsigned int status;
+ uoff_t content_length;
+ const char *payload;
+};
+
+/* Valid header tests */
+
+static const struct http_response_parse_test
+valid_response_parse_tests[] = {
+ { .response =
+ "HTTP/1.1 200 OK\r\n"
+ "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n"
+ "Server: Apache/2.2.16 (Debian)\r\n"
+ "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n"
+ "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n"
+ "Accept-Ranges: bytes\r\n"
+ "Content-Length: 33\r\n"
+ "Keep-Alive: timeout=15, max=100\r\n"
+ "Connection: Keep-Alive\r\n"
+ "Content-Type: text/plain\r\n"
+ "\r\n"
+ "This is a piece of stupid text.\r\n",
+ .status = 200,
+ .payload = "This is a piece of stupid text.\r\n"
+ },{
+ .response =
+ "HTTP/1.1 200 OK\r\n"
+ "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n"
+ "Server: Apache/2.2.16 (Debian)\r\n"
+ "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n"
+ "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n"
+ "Accept-Ranges: bytes\r\n"
+ "Content-Length: 33\r\n"
+ "Keep-Alive: timeout=15, max=100\r\n"
+ "Connection: Keep-Alive\r\n"
+ "Content-Type: text/plain\r\n"
+ "\r\n"
+ "This is a piece of stupid text.\r\n"
+ "HTTP/1.1 200 OK\r\n"
+ "Date: Sun, 07 Oct 2012 13:02:27 GMT\r\n"
+ "Server: Apache/2.2.16 (Debian)\r\n"
+ "Last-Modified: Tue, 18 Sep 2012 19:31:41 GMT\r\n"
+ "Etag: \"2a8400c-10751f-4c9fef0858140\"\r\n"
+ "Accept-Ranges: bytes\r\n"
+ "Content-Length: 43\r\n"
+ "Keep-Alive: timeout=15, max=100\r\n"
+ "Connection: Keep-Alive\r\n"
+ "Content-Type: text/plain\r\n"
+ "\r\n"
+ "This is a piece of even more stupid text.\r\n",
+ .status = 200,
+ .payload = "This is a piece of even more stupid text.\r\n"
+ },{
+ .response =
+ "HTTP/1.1 401 Authorization Required\r\n"
+ "Date: Sun, 07 Oct 2012 19:52:03 GMT\r\n"
+ "Server: Apache/2.2.16 (Debian) PHP/5.3.3-7+squeeze14\r\n"
+ "WWW-Authenticate: Basic realm=\"Munin\"\r\n"
+ "Vary: Accept-Encoding\r\n"
+ "Content-Encoding: gzip\r\n"
+ "Content-Length: 5\r\n"
+ "Keep-Alive: timeout=15, max=99\r\n"
+ "Connection: Keep-Alive\r\n"
+ "Content-Type: text/html; charset=iso-8859-1\r\n"
+ "\r\n"
+ "Frop!",
+ .status = 401,
+ .payload = "Frop!"
+ }
+};
+
+unsigned int valid_response_parse_test_count =
+ N_ELEMENTS(valid_response_parse_tests);
+
+static void test_http_response_parse_valid(void)
+{
+ unsigned int i;
+ buffer_t *payload_buffer = buffer_create_dynamic(default_pool, 1024);
+
+ for (i = 0; i < valid_response_parse_test_count; i++) T_BEGIN {
+ struct istream *input;
+ struct ostream *output;
+ const struct http_response_parse_test *test;
+ struct http_response_parser *parser;
+ struct http_response *response;
+ const char *response_text, *payload, *error;
+ int ret;
+
+ test = &valid_response_parse_tests[i];
+ response_text = test->response;
+ input = i_stream_create_from_data(response_text, strlen(response_text));
+ parser = http_response_parser_init(input);
+
+ test_begin(t_strdup_printf("http response valid [%d]", i));
+
+ payload = NULL;
+ while ((ret=http_response_parse_next(parser, FALSE, &response, &error)) > 0) {
+ if (response->payload != NULL) {
+ buffer_set_used_size(payload_buffer, 0);
+ output = o_stream_create_buffer(payload_buffer);
+ test_out("payload receive",
+ o_stream_send_istream(output, response->payload));
+ o_stream_destroy(&output);
+ payload = str_c(payload_buffer);
+ } else {
+ payload = NULL;
+ }
+ }
+
+ test_out("parse success", ret == 0);
+
+ if (ret == 0) {
+ /* verify last response only */
+ test_out(t_strdup_printf("response->status = %d",test->status),
+ response->status == test->status);
+ if (payload == NULL || test->payload == NULL) {
+ test_out(t_strdup_printf("response->payload = %s",
+ str_sanitize(payload, 80)),
+ payload == test->payload);
+ } else {
+ test_out(t_strdup_printf("response->payload = %s",
+ str_sanitize(payload, 80)),
+ strcmp(payload, test->payload) == 0);
+ }
+ }
+ test_end();
+ http_response_parser_deinit(&parser);
+ } T_END;
+
+ buffer_free(&payload_buffer);
+}
+
+static const char *invalid_response_parse_tests[] = {
+ "XMPP/1.0 302 Found\r\n"
+ "Location: http://www.example.nl/\r\n"
+ "Cache-Control: private\r\n",
+ "HTTP/1.1 302 Found\r\n"
+ "Location: http://www.example.nl/\r\n"
+ "Cache-Control: private\r\n",
+ "HTTP/1.1 ABC Found\r\n"
+ "Location: http://www.example.nl/\r\n"
+ "Cache-Control: private\r\n",
+ "HTTP/1.1 302 \177\r\n"
+ "Location: http://www.example.nl/\r\n"
+ "Cache-Control: private\r\n",
+ "HTTP/1.1 302 Found\n\r"
+ "Location: http://www.example.nl/\n\r"
+ "Cache-Control: private\n\r"
+};
+
+unsigned int invalid_response_parse_test_count =
+ N_ELEMENTS(invalid_response_parse_tests);
+
+static void test_http_response_parse_invalid(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < invalid_response_parse_test_count; i++) T_BEGIN {
+ struct istream *input;
+ const char *test;
+ struct http_response_parser *parser;
+ struct http_response *response;
+ const char *response_text, *error;
+ int ret;
+
+ test = invalid_response_parse_tests[i];
+ response_text = test;
+ input = i_stream_create_from_data(response_text, strlen(response_text));
+ parser = http_response_parser_init(input);
+
+ test_begin(t_strdup_printf("http response invalid [%d]", i));
+
+ while ((ret=http_response_parse_next(parser, FALSE, &response, &error)) > 0);
+
+ test_out("parse failure", ret < 0);
+ test_end();
+ http_response_parser_deinit(&parser);
+ } T_END;
+}
+
+int main(void)
+{
+ static void (*test_functions[])(void) = {
+ test_http_response_parse_valid,
+ test_http_response_parse_invalid,
+ NULL
+ };
+ return test_run(test_functions);
+}
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "istream.h"
+#include "ostream.h"
+#include "test-common.h"
+#include "http-response-parser.h"
+
+#include <stdio.h>
+
+// FIXME: debug tool; can be removed
+
+int main(int argc, char **argv)
+{
+ struct istream *input;
+ struct http_response_parser *parser;
+ struct http_response *response;
+ const char *payload, *error = NULL;
+ int ret;
+
+ if (argc < 2)
+ return 1;
+
+ input = i_stream_create_file(argv[1], 32);
+ parser = http_response_parser_init(input);
+
+ payload = NULL;
+ while ((ret=http_response_parse_next(parser, FALSE, &response, &error)) > 0) {
+ printf("RESPONSE: %u %s\n", response->status, response->reason);
+ }
+
+ printf("RET: %d %s\n", ret, error);
+
+ http_response_parser_deinit(&parser);
+}
+
--- /dev/null
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "strfuncs.h"
+#include "str-sanitize.h"
+#include "istream.h"
+#include "ostream.h"
+#include "test-common.h"
+#include "http-transfer.h"
+
+#include <time.h>
+
+struct http_transfer_chunked_test {
+ const char *in;
+ const char *out;
+};
+
+/* Valid transfer_chunked tests */
+struct http_transfer_chunked_test valid_transfer_chunked_tests[] = {
+ { .in = "1E\r\n"
+ "This is a simple test payload."
+ "\r\n"
+ "0\r\n"
+ "\r\n",
+ .out =
+ "This is a simple test payload."
+ },
+ { .in = "20\r\n"
+ "This is a longer test payload..."
+ "\r\n"
+ "23\r\n"
+ "...spread over two separate chunks."
+ "\r\n"
+ "0\r\n"
+ "\r\n",
+ .out =
+ "This is a longer test payload..."
+ "...spread over two separate chunks."
+ },
+ { .in = "26\r\n"
+ "This is an even longer test payload..."
+ "\r\n"
+ "27\r\n"
+ "...spread over three separate chunks..."
+ "\r\n"
+ "1F\r\n"
+ "...and also includes a trailer."
+ "\r\n"
+ "0\r\n"
+ "Checksum: adgfef3fdaf3daf3dfaf3ff3fdag\r\n"
+ "X-Dovecot: Whatever\r\n"
+ "\r\n",
+ .out =
+ "This is an even longer test payload..."
+ "...spread over three separate chunks..."
+ "...and also includes a trailer."
+ },
+ { .in = "26\n"
+ "This is an even longer test payload..."
+ "\n"
+ "27\n"
+ "...spread over three separate chunks..."
+ "\n"
+ "1F\n"
+ "...and also includes a trailer."
+ "\n"
+ "0\n"
+ "Checksum: adgfef3fdaf3daf3dfaf3ff3fdag\n"
+ "X-Dovecot: Whatever\n"
+ "\n",
+ .out =
+ "This is an even longer test payload..."
+ "...spread over three separate chunks..."
+ "...and also includes a trailer."
+ }
+};
+
+unsigned int valid_transfer_chunked_test_count =
+ N_ELEMENTS(valid_transfer_chunked_tests);
+
+static void test_http_transfer_chunked_valid(void)
+{
+ struct istream *input, *chunked;
+ struct ostream *output;
+ buffer_t *payload_buffer;
+ unsigned int i;
+
+ payload_buffer = buffer_create_dynamic(default_pool, 1024);
+
+ for (i = 0; i < valid_transfer_chunked_test_count; i++) T_BEGIN {
+ const char *in, *out, *stream_out;
+
+ in = valid_transfer_chunked_tests[i].in;
+ out = valid_transfer_chunked_tests[i].out;
+
+ test_begin(t_strdup_printf("http transfer_chunked valid [%d]", i));
+
+ input = i_stream_create_from_data(in, strlen(in));
+ chunked = http_transfer_chunked_istream_create(input);
+
+ buffer_set_used_size(payload_buffer, 0);
+ output = o_stream_create_buffer(payload_buffer);
+ test_out("payload read",
+ o_stream_send_istream(output, chunked));
+ o_stream_destroy(&output);
+ stream_out = str_c(payload_buffer);
+
+ test_out(t_strdup_printf("response->payload = %s",
+ str_sanitize(stream_out, 80)),
+ strcmp(stream_out, out) == 0);
+ test_end();
+ } T_END;
+
+ buffer_free(&payload_buffer);
+}
+
+int main(void)
+{
+ static void (*test_functions[])(void) = {
+ test_http_transfer_chunked_valid,
+ NULL
+ };
+ return test_run(test_functions);
+}
projects / thirdparty / dovecot / core.git / commitdiff
