From: Yann Ylavic Date: Tue, 12 May 2020 12:20:57 +0000 (+0000) Subject: mod_proxy_http: handle Upgrade requests and upgraded protocol forwarding. X-Git-Tag: 2.5.0-alpha2-ci-test-only~1445 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1dc3c88d41a5362c6c0852b63ac85bc996ca157e;p=thirdparty%2Fapache%2Fhttpd.git mod_proxy_http: handle Upgrade requests and upgraded protocol forwarding. If the request Upgrade header matches the worker upgrade= parameter and the backend switches the protocol, do the tunneling in mod_proxy_http. This allows to keep the protocol to HTTP until the backend really switches the protocol, and apply usual output filters. When configured to forward Upgrade mechanism, we want the backend to be able to announce its Upgrade protocol to the client (e.g. with 426 Upgrade Required response) and thus forward back the Upgrade header that matches the one(s) configured in the worker upgrade= parameter. modules/proxy/mod_proxy.h: modules/proxy/proxy_util.c: ap_proxy_worker_can_upgrade(): added helper to determine whether a proxy worker is configured to forward an Upgrade protocol. include/ap_mmn.h: Bump MMN minor for ap_proxy_worker_can_upgrade(). modules/proxy/mod_proxy.c: set_worker_param(): handle worker parameter upgrade=ANY as upgrade=* (should the "any" protocol scheme be something some day..). modules/proxy/mod_proxy_wstunnel.c: proxy_wstunnel_handler(): use ap_proxy_worker_can_upgrade() to match the Upgrade header. Axe handling of upgrade=NONE, it makes no sense to Upgrade a connection if the client did not ask for it, nor to configure mod_proxy_wstunnel to use a worker with upgrade=NONE by the way. modules/proxy/mod_proxy_http.c: proxy_http_req_t: add fields force10 (force HTTP/1.0) and upgrade (value of the Upgrade header sent by the client if it matches the configuration, NULL otherwise). proxy_http_handler(): use ap_proxy_worker_can_upgrade() to determine whether the request is electable for end to end protocol upgrading and set req->upgrade accordingly. terminate_headers(): handle Connection and Upgrade headers to send to the backend, according to req->force10 and req->upgrade set before. ap_proxy_http_prefetch(): use req->force10 and terminate_headers(). send_continue_body(): added helper to send the body retained for end to end 100-continue handling. ap_proxy_http_process_response(): use ap_proxy_worker_can_upgrade() to match the response Upgrade header and forward it back if it matches the configured one(s). That is for 101 Switching Protocol obviously but also any other status code which is not overidden, at the backend wish. If the protocol is switching, create a proxy tunnel and run it, using the minimal timeout from the client or backend connection. Github: closes #125 git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1877646 13f79535-47bb-0310-9956-ffa450edef68 --- diff --git a/CHANGES b/CHANGES index c6d70951a10..9a940c3a3e7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,9 @@ -*- coding: utf-8 -*- Changes with Apache 2.5.1 + *) mod_proxy_http: handle Upgrade request, 101 (Switching Protocol) response + and switched protocol forwarding. [Yann Ylavic] + *) mod_ssl: The "ssl_var_lookup" optional function API now takes a const char *name argument and returns a const char * string value. The pool argument must now be non-NULL. [Joe Orton] diff --git a/docs/log-message-tags/next-number b/docs/log-message-tags/next-number index 289150f8934..9f7931cdf3a 100644 --- a/docs/log-message-tags/next-number +++ b/docs/log-message-tags/next-number @@ -1 +1 @@ -10239 +10241 diff --git a/include/ap_mmn.h b/include/ap_mmn.h index 7e5a1f902b3..8ac21c15f41 100644 --- a/include/ap_mmn.h +++ b/include/ap_mmn.h @@ -630,6 +630,7 @@ * ap_check_request_header() * 20200420.0 (2.5.1-dev) Add flags to listen_rec in place of use_specific_errors * 20200420.1 (2.5.1-dev) Add ap_filter_adopt_brigade() + * 20200420.2 (2.5.1-dev) Add ap_proxy_worker_can_upgrade() */ #define MODULE_MAGIC_COOKIE 0x41503235UL /* "AP25" */ @@ -637,7 +638,7 @@ #ifndef MODULE_MAGIC_NUMBER_MAJOR #define MODULE_MAGIC_NUMBER_MAJOR 20200420 #endif -#define MODULE_MAGIC_NUMBER_MINOR 1 /* 0...n */ +#define MODULE_MAGIC_NUMBER_MINOR 2 /* 0...n */ /** * Determine if the server's current MODULE_MAGIC_NUMBER is at least a diff --git a/modules/proxy/mod_proxy.c b/modules/proxy/mod_proxy.c index 71fae13edcb..057d34deec8 100644 --- a/modules/proxy/mod_proxy.c +++ b/modules/proxy/mod_proxy.c @@ -324,7 +324,8 @@ static const char *set_worker_param(apr_pool_t *p, } } else if (!strcasecmp(key, "upgrade")) { - if (PROXY_STRNCPY(worker->s->upgrade, val) != APR_SUCCESS) { + if (PROXY_STRNCPY(worker->s->upgrade, + strcasecmp(val, "ANY") ? val : "*") != APR_SUCCESS) { return apr_psprintf(p, "upgrade protocol length must be < %d characters", (int)sizeof(worker->s->upgrade)); } diff --git a/modules/proxy/mod_proxy.h b/modules/proxy/mod_proxy.h index bb47d9a01fc..4e8c6e07af6 100644 --- a/modules/proxy/mod_proxy.h +++ b/modules/proxy/mod_proxy.h @@ -740,6 +740,17 @@ typedef __declspec(dllimport) const char * PROXY_DECLARE(char *) ap_proxy_worker_name(apr_pool_t *p, proxy_worker *worker); +/** + * Return whether a worker upgrade configuration matches Upgrade header + * @param p memory pool used for displaying worker name + * @param worker the worker + * @param upgrade the Upgrade header to match + * @return 1 (true) or 0 (false) + */ +PROXY_DECLARE(int) ap_proxy_worker_can_upgrade(apr_pool_t *p, + const proxy_worker *worker, + const char *upgrade); + /** * Get the worker from proxy configuration * @param p memory pool used for finding worker diff --git a/modules/proxy/mod_proxy_http.c b/modules/proxy/mod_proxy_http.c index 0e0cf91ae33..1e84aee74fd 100644 --- a/modules/proxy/mod_proxy_http.c +++ b/modules/proxy/mod_proxy_http.c @@ -216,16 +216,6 @@ static void add_cl(apr_pool_t *p, APR_BRIGADE_INSERT_TAIL(header_brigade, e); } -static void terminate_headers(apr_bucket_alloc_t *bucket_alloc, - apr_bucket_brigade *header_brigade) -{ - apr_bucket *e; - - /* add empty line at the end of the headers */ - e = apr_bucket_immortal_create(CRLF_ASCII, 2, bucket_alloc); - APR_BRIGADE_INSERT_TAIL(header_brigade, e); -} - #define MAX_MEM_SPOOL 16384 @@ -254,6 +244,9 @@ typedef struct { rb_methods rb_method; + int force10; + const char *upgrade; + int expecting_100; unsigned int do_100_continue:1, prefetch_nonblocking:1; @@ -559,6 +552,43 @@ static int spool_reqbody_cl(proxy_http_req_t *req, apr_off_t *bytes_spooled) return OK; } +static void terminate_headers(proxy_http_req_t *req) +{ + apr_bucket_alloc_t *bucket_alloc = req->bucket_alloc; + apr_bucket *e; + char *buf; + + /* + * Handle Connection: header if we do HTTP/1.1 request: + * If we plan to close the backend connection sent Connection: close + * otherwise sent Connection: Keep-Alive. + */ + if (!req->force10) { + if (req->upgrade) { + buf = apr_pstrdup(req->p, "Connection: Upgrade" CRLF); + ap_xlate_proto_to_ascii(buf, strlen(buf)); + e = apr_bucket_pool_create(buf, strlen(buf), req->p, bucket_alloc); + APR_BRIGADE_INSERT_TAIL(req->header_brigade, e); + + /* Tell the backend that it can upgrade the connection. */ + buf = apr_pstrcat(req->p, "Upgrade: ", req->upgrade, CRLF, NULL); + } + else if (ap_proxy_connection_reusable(req->backend)) { + buf = apr_pstrdup(req->p, "Connection: Keep-Alive" CRLF); + } + else { + buf = apr_pstrdup(req->p, "Connection: close" CRLF); + } + ap_xlate_proto_to_ascii(buf, strlen(buf)); + e = apr_bucket_pool_create(buf, strlen(buf), req->p, bucket_alloc); + APR_BRIGADE_INSERT_TAIL(req->header_brigade, e); + } + + /* add empty line at the end of the headers */ + e = apr_bucket_immortal_create(CRLF_ASCII, 2, bucket_alloc); + APR_BRIGADE_INSERT_TAIL(req->header_brigade, e); +} + static int ap_proxy_http_prefetch(proxy_http_req_t *req, apr_uri_t *uri, char *url) { @@ -571,20 +601,14 @@ static int ap_proxy_http_prefetch(proxy_http_req_t *req, apr_bucket_brigade *input_brigade = req->input_brigade; apr_bucket_brigade *temp_brigade; apr_bucket *e; - char *buf; apr_status_t status; apr_off_t bytes_read = 0; apr_off_t bytes; - int force10, rv; apr_read_type_e block; + int rv; - if (apr_table_get(r->subprocess_env, "force-proxy-request-1.0")) { - if (req->expecting_100) { - return HTTP_EXPECTATION_FAILED; - } - force10 = 1; - } else { - force10 = 0; + if (req->force10 && req->expecting_100) { + return HTTP_EXPECTATION_FAILED; } rv = ap_proxy_create_hdrbrgd(p, header_brigade, r, p_conn, @@ -768,7 +792,7 @@ static int ap_proxy_http_prefetch(proxy_http_req_t *req, req->rb_method = RB_STREAM_CL; } else if (req->old_te_val) { - if (force10 + if (req->force10 || (apr_table_get(r->subprocess_env, "proxy-sendcl") && !apr_table_get(r->subprocess_env, "proxy-sendchunks") && !apr_table_get(r->subprocess_env, "proxy-sendchunked"))) { @@ -790,7 +814,7 @@ static int ap_proxy_http_prefetch(proxy_http_req_t *req, } req->rb_method = RB_STREAM_CL; } - else if (!force10 + else if (!req->force10 && (apr_table_get(r->subprocess_env, "proxy-sendchunks") || apr_table_get(r->subprocess_env, "proxy-sendchunked")) && !apr_table_get(r->subprocess_env, "proxy-sendcl")) { @@ -834,23 +858,7 @@ static int ap_proxy_http_prefetch(proxy_http_req_t *req, /* Yes I hate gotos. This is the subrequest shortcut */ skip_body: - /* - * Handle Connection: header if we do HTTP/1.1 request: - * If we plan to close the backend connection sent Connection: close - * otherwise sent Connection: Keep-Alive. - */ - if (!force10) { - if (!ap_proxy_connection_reusable(p_conn)) { - buf = apr_pstrdup(p, "Connection: close" CRLF); - } - else { - buf = apr_pstrdup(p, "Connection: Keep-Alive" CRLF); - } - ap_xlate_proto_to_ascii(buf, strlen(buf)); - e = apr_bucket_pool_create(buf, strlen(buf), p, c->bucket_alloc); - APR_BRIGADE_INSERT_TAIL(header_brigade, e); - } - terminate_headers(bucket_alloc, header_brigade); + terminate_headers(req); return OK; } @@ -1163,6 +1171,36 @@ static int add_trailers(void *data, const char *key, const char *val) return 1; } +static int send_continue_body(proxy_http_req_t *req) +{ + int status; + + /* Send the request body (fully). */ + switch(req->rb_method) { + case RB_SPOOL_CL: + case RB_STREAM_CL: + case RB_STREAM_CHUNKED: + status = stream_reqbody(req); + break; + default: + /* Shouldn't happen */ + status = HTTP_INTERNAL_SERVER_ERROR; + break; + } + if (status != OK) { + conn_rec *c = req->r->connection; + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, req->r, + APLOGNO(10154) "pass request body failed " + "to %pI (%s) from %s (%s) with status %i", + req->backend->addr, + req->backend->hostname ? req->backend->hostname : "", + c->client_ip, c->remote_host ? c->remote_host : "", + status); + req->backend->close = 1; + } + return status; +} + static int ap_proxy_http_process_response(proxy_http_req_t *req) { @@ -1173,6 +1211,7 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) proxy_conn_rec *backend = req->backend; conn_rec *origin = req->origin; int do_100_continue = req->do_100_continue; + int status; char *buffer; char fixed_buffer[HUGE_STRING_LEN]; @@ -1244,6 +1283,7 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) origin->local_addr->port)); do { apr_status_t rc; + const char *upgrade = NULL; int major = 0, minor = 0; int toclose = 0; @@ -1422,6 +1462,21 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) */ te = apr_table_get(r->headers_out, "Transfer-Encoding"); + upgrade = apr_table_get(r->headers_out, "Upgrade"); + if (proxy_status == HTTP_SWITCHING_PROTOCOLS) { + if (!upgrade || !req->upgrade || (strcasecmp(req->upgrade, + upgrade) != 0)) { + return ap_proxyerror(r, HTTP_BAD_GATEWAY, + apr_pstrcat(p, "Unexpected Upgrade: ", + upgrade ? upgrade : "n/a", + " (expecting ", + req->upgrade ? req->upgrade + : "n/a", ")", + NULL)); + } + backend->close = 1; + } + /* strip connection listed hop-by-hop headers from response */ toclose = ap_proxy_clear_connection_fn(r, r->headers_out); if (toclose) { @@ -1509,6 +1564,7 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) ap_log_rerror(APLOG_MARK, APLOG_TRACE2, 0, r, "HTTP: received interim %d response", r->status); if (!policy + || upgrade || (!strcasecmp(policy, "RFC") && (proxy_status != HTTP_CONTINUE || (req->expecting_100 = 1)))) { @@ -1569,30 +1625,8 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) major, minor, proxy_status_line); if (do_send_body) { - int status; - - /* Send the request body (fully). */ - switch(req->rb_method) { - case RB_SPOOL_CL: - case RB_STREAM_CL: - case RB_STREAM_CHUNKED: - status = stream_reqbody(req); - break; - default: - /* Shouldn't happen */ - status = HTTP_INTERNAL_SERVER_ERROR; - break; - } + status = send_continue_body(req); if (status != OK) { - ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, - APLOGNO(10154) "pass request body failed " - "to %pI (%s) from %s (%s) with status %i", - backend->addr, - backend->hostname ? backend->hostname : "", - c->client_ip, - c->remote_host ? c->remote_host : "", - status); - backend->close = 1; return status; } } @@ -1615,6 +1649,67 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) do_100_continue = 0; } + if (proxy_status == HTTP_SWITCHING_PROTOCOLS) { + apr_status_t rv; + proxy_tunnel_rec *tunnel; + apr_interval_time_t client_timeout = -1, + backend_timeout = -1; + + /* If we didn't send the full body yet, do it now */ + if (do_100_continue) { + req->expecting_100 = 0; + status = send_continue_body(req); + if (status != OK) { + return status; + } + } + + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, APLOGNO(10239) + "HTTP: tunneling protocol %s", upgrade); + + rv = ap_proxy_tunnel_create(&tunnel, r, origin, "HTTP"); + if (rv != APR_SUCCESS) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, rv, r, APLOGNO(10240) + "can't create tunnel for %s", upgrade); + return HTTP_INTERNAL_SERVER_ERROR; + } + + /* Set timeout to the lowest configured for client or backend */ + apr_socket_timeout_get(backend->sock, &backend_timeout); + apr_socket_timeout_get(ap_get_conn_socket(c), &client_timeout); + if (backend_timeout >= 0 && backend_timeout < client_timeout) { + tunnel->timeout = backend_timeout; + } + else { + tunnel->timeout = client_timeout; + } + + /* Bidirectional non-HTTP stream will confuse mod_reqtimeoout, we + * use a single idle timeout from now on. + */ + ap_remove_input_filter_byhandle(c->input_filters, "reqtimeout"); + + /* Let proxy tunnel forward everything */ + status = ap_proxy_tunnel_run(tunnel); + if (ap_is_HTTP_ERROR(status)) { + /* Tunnel always return HTTP_GATEWAY_TIME_OUT on timeout, + * but we can differentiate between client and backend here. + */ + if (status == HTTP_GATEWAY_TIME_OUT + && tunnel->timeout == client_timeout) { + status = HTTP_REQUEST_TIME_OUT; + } + } + else { + /* Update r->status for custom log */ + status = HTTP_SWITCHING_PROTOCOLS; + } + r->status = status; + + /* We are done with both connections */ + return DONE; + } + if (interim_response) { /* Already forwarded above, read next response */ continue; @@ -1666,6 +1761,12 @@ int ap_proxy_http_process_response(proxy_http_req_t *req) return proxy_status; } + /* Forward back Upgrade header if it matches the configured one(s). */ + if (upgrade && ap_proxy_worker_can_upgrade(p, worker, upgrade)) { + apr_table_setn(r->headers_out, "Connection", "Upgrade"); + apr_table_setn(r->headers_out, "Upgrade", apr_pstrdup(p, upgrade)); + } + r->sent_bodyct = 1; /* * Is it an HTTP/0.9 response or did we maybe preread the 1st line of @@ -2015,6 +2116,17 @@ static int proxy_http_handler(request_rec *r, proxy_worker *worker, dconf = ap_get_module_config(r->per_dir_config, &proxy_module); + if (apr_table_get(r->subprocess_env, "force-proxy-request-1.0")) { + req->force10 = 1; + } + else if (*worker->s->upgrade) { + /* Forward Upgrade header if it matches the configured one(s). */ + const char *upgrade = apr_table_get(r->headers_in, "Upgrade"); + if (upgrade && ap_proxy_worker_can_upgrade(p, worker, upgrade)) { + req->upgrade = upgrade; + } + } + /* We possibly reuse input data prefetched in previous call(s), e.g. for a * balancer fallback scenario, and in this case the 100 continue settings * should be consistent between balancer members. If not, we need to ignore diff --git a/modules/proxy/mod_proxy_wstunnel.c b/modules/proxy/mod_proxy_wstunnel.c index 6640c62ecd2..f6d1e346641 100644 --- a/modules/proxy/mod_proxy_wstunnel.c +++ b/modules/proxy/mod_proxy_wstunnel.c @@ -304,7 +304,7 @@ static int proxy_wstunnel_handler(request_rec *r, proxy_worker *worker, int status; char server_portstr[32]; proxy_conn_rec *backend = NULL; - const char *upgrade_method, *upgrade; + const char *upgrade; char *scheme; apr_pool_t *p = r->pool; char *locurl = url; @@ -323,25 +323,20 @@ static int proxy_wstunnel_handler(request_rec *r, proxy_worker *worker, return DECLINED; } - /* XXX: what's the point of "NONE"? We probably should _always_ check - * that the client wants an Upgrade.. - */ - upgrade_method = *worker->s->upgrade ? worker->s->upgrade : "WebSocket"; - if (ap_cstr_casecmp(upgrade_method, "NONE") != 0) { - upgrade = apr_table_get(r->headers_in, "Upgrade"); - if (!upgrade || (ap_cstr_casecmp(upgrade, upgrade_method) != 0 && - ap_cstr_casecmp(upgrade_method, "ANY") != 0)) { - ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, APLOGNO(02900) - "require upgrade for URL %s " - "(Upgrade header is %s, expecting %s)", - url, upgrade ? upgrade : "missing", upgrade_method); - apr_table_setn(r->err_headers_out, "Connection", "Upgrade"); - apr_table_setn(r->err_headers_out, "Upgrade", upgrade_method); - return HTTP_UPGRADE_REQUIRED; - } - } - else { - upgrade = "WebSocket"; + upgrade = apr_table_get(r->headers_in, "Upgrade"); + if (!upgrade || (*worker->s->upgrade && + !ap_proxy_worker_can_upgrade(p, worker, upgrade)) + || (!*worker->s->upgrade && + ap_cstr_casecmp(upgrade, "WebSocket") != 0)) { + const char *worker_upgrade = *worker->s->upgrade ? worker->s->upgrade + : "WebSocket"; + ap_log_rerror(APLOG_MARK, APLOG_INFO, 0, r, APLOGNO(02900) + "require upgrade for URL %s " + "(Upgrade header is %s, expecting %s)", + url, upgrade ? upgrade : "missing", worker_upgrade); + apr_table_setn(r->err_headers_out, "Connection", "Upgrade"); + apr_table_setn(r->err_headers_out, "Upgrade", worker_upgrade); + return HTTP_UPGRADE_REQUIRED; } uri = apr_palloc(p, sizeof(*uri)); diff --git a/modules/proxy/proxy_util.c b/modules/proxy/proxy_util.c index 23674ae2a8c..5c9f93e7120 100644 --- a/modules/proxy/proxy_util.c +++ b/modules/proxy/proxy_util.c @@ -1652,6 +1652,17 @@ PROXY_DECLARE(char *) ap_proxy_worker_name(apr_pool_t *p, return apr_pstrcat(p, "unix:", worker->s->uds_path, "|", worker->s->name, NULL); } +PROXY_DECLARE(int) ap_proxy_worker_can_upgrade(apr_pool_t *p, + const proxy_worker *worker, + const char *upgrade) +{ + const char *worker_upgrade = worker->s->upgrade; + return (*worker_upgrade + && (strcmp(worker_upgrade, "*") == 0 + || ap_cstr_casecmp(worker_upgrade, upgrade) == 0 + || ap_find_token(p, worker_upgrade, upgrade))); +} + /* * Taken from ap_strcmp_match() : * Match = 0, NoMatch = 1, Abort = -1, Inval = -2