From c9b95c0bb30f88bf00e1ac7e706cf3de271cc6af Mon Sep 17 00:00:00 2001 From: Stefan Eissing Date: Wed, 19 Jun 2024 12:40:06 +0200 Subject: [PATCH] lib: graceful connection shutdown When libcurl discards a connection there are two phases this may go through: "shutdown" and "closing". If a connection is aborted, the shutdown phase is skipped and it is closed right away. The connection filters attached to the connection implement the phases in their `do_shutdown()` and `do_close()` callbacks. Filters carry now a `shutdown` flags next to `connected` to keep track of the shutdown operation. Filters are shut down from top to bottom. If a filter is not connected, its shutdown is skipped. Notable filters that *do* something during shutdown are HTTP/2 and TLS. HTTP/2 sends the GOAWAY frame. TLS sends its close notify and expects to receive a close notify from the server. As sends and receives may EAGAIN on the network, a shutdown is often not successful right away and needs to poll the connection's socket(s). To facilitate this, such connections are placed on a new shutdown list inside the connection cache. Since managing this list requires the cooperation of a multi handle, only the connection cache belonging to a multi handle is used. If a connection was in another cache when being discarded, it is removed there and added to the multi's cache. If no multi handle is available at that time, the connection is shutdown and closed in a one-time, best-effort attempt. When a multi handle is destroyed, all connection still on the shutdown list are discarded with a final shutdown attempt and close. In curl debug builds, the environment variable `CURL_GRACEFUL_SHUTDOWN` can be set to make this graceful with a timeout in milliseconds given by the variable. The shutdown list is limited to the max number of connections configured for a multi cache. Set via CURLMOPT_MAX_TOTAL_CONNECTIONS. When the limit is reached, the oldest connection on the shutdown list is discarded. - In multi_wait() and multi_waitfds(), collect all connection caches involved (each transfer might carry its own) into a temporary list. Let each connection cache on the list contribute sockets and POLLIN/OUT events it's connections are waiting for. - in multi_perform() collect the connection caches the same way and let them peform their maintenance. This will make another non-blocking attempt to shutdown all connections on its shutdown list. - for event based multis (multi->socket_cb set), add the sockets and their poll events via the callback. When `multi_socket()` is invoked for a socket not known by an active transfer, forward this to the multi's cache for processing. On closing a connection, remove its socket(s) via the callback. TLS connection filters MUST NOT send close nofity messages in their `do_close()` implementation. The reason is that a TLS close notify signals a success. When a connection is aborted and skips its shutdown phase, the server needs to see a missing close notify to detect something has gone wrong. A graceful shutdown of FTP's data connection is performed implicitly before regarding the upload/download as complete and continuing on the control connection. For FTP without TLS, there is just the socket close happening. But with TLS, the sent/received close notify signals that the transfer is complete and healthy. Servers like `vsftpd` verify that and reject uploads without a TLS close notify. - added test_19_* for shutdown related tests - test_19_01 and test_19_02 test for TCP RST packets which happen without a graceful shutdown and should no longer appear otherwise. - add test_19_03 for handling shutdowns by the server - add test_19_04 for handling shutdowns by curl - add test_19_05 for event based shutdowny by server - add test_30_06/07 and test_31_06/07 for shutdown checks on FTP up- and downloads. Closes #13976 --- docs/libcurl/libcurl-env-dbg.md | 13 + lib/cf-h2-proxy.c | 20 +- lib/cfilters.c | 31 +- lib/cfilters.h | 1 + lib/conncache.c | 690 ++++++++++++++++-- lib/conncache.h | 46 +- lib/http2.c | 21 +- lib/multi.c | 63 +- lib/multihandle.h | 2 + lib/share.c | 3 +- lib/url.c | 57 +- lib/url.h | 3 +- lib/urldata.h | 3 + lib/vtls/bearssl.c | 13 +- lib/vtls/gtls.c | 8 +- lib/vtls/mbedtls.c | 10 +- lib/vtls/openssl.c | 108 +-- lib/vtls/rustls.c | 13 +- lib/vtls/schannel.c | 13 +- lib/vtls/sectransp.c | 10 +- lib/vtls/vtls.c | 12 +- lib/vtls/vtls_int.h | 1 - lib/vtls/wolfssl.c | 15 +- src/tool_operate.c | 3 + tests/conftest.py | 2 +- tests/data/test1542 | 4 +- tests/http/clients/h2-download.c | 8 +- tests/http/test_19_shutdown.py | 156 ++++ tests/http/test_30_vsftpd.py | 27 + tests/http/test_31_vsftpds.py | 29 + tests/http/testenv/curl.py | 148 +++- tests/http/testenv/env.py | 11 + .../http/testenv/mod_curltest/mod_curltest.c | 43 +- 33 files changed, 1312 insertions(+), 275 deletions(-) create mode 100644 tests/http/test_19_shutdown.py diff --git a/docs/libcurl/libcurl-env-dbg.md b/docs/libcurl/libcurl-env-dbg.md index 2ab8c2b955..9effd41651 100644 --- a/docs/libcurl/libcurl-env-dbg.md +++ b/docs/libcurl/libcurl-env-dbg.md @@ -123,3 +123,16 @@ greater. There is a number of debug levels, refer to *openldap.c* comments. Used to influence the buffer chunk size used for WebSocket encoding and decoding. + +## CURL_FORBID_REUSE + +Used to set the CURLOPT_FORBID_REUSE flag on each transfer initiated +by the curl command line tool. The value of the environment variable +does not matter. + +## CURL_GRACEFUL_SHUTDOWN + +Make a blocking, graceful shutdown of all remaining connections when +a multi handle is destroyed. This implicitly triggers for easy handles +that are run via easy_perform. The value of the environment variable +gives the shutdown timeout in milliseconds. \ No newline at end of file diff --git a/lib/cf-h2-proxy.c b/lib/cf-h2-proxy.c index 1352079f02..9519763fa8 100644 --- a/lib/cf-h2-proxy.c +++ b/lib/cf-h2-proxy.c @@ -183,7 +183,6 @@ struct cf_h2_proxy_ctx { BIT(conn_closed); BIT(rcvd_goaway); BIT(sent_goaway); - BIT(shutdown); BIT(nw_out_blocked); }; @@ -1172,14 +1171,17 @@ static CURLcode cf_h2_proxy_shutdown(struct Curl_cfilter *cf, struct Curl_easy *data, bool *done) { struct cf_h2_proxy_ctx *ctx = cf->ctx; + struct cf_call_data save; CURLcode result; int rv; - if(!cf->connected || !ctx->h2 || ctx->shutdown) { + if(!cf->connected || !ctx->h2 || cf->shutdown || ctx->conn_closed) { *done = TRUE; return CURLE_OK; } + CF_DATA_SAVE(save, cf, data); + if(!ctx->sent_goaway) { rv = nghttp2_submit_goaway(ctx->h2, NGHTTP2_FLAG_NONE, 0, 0, @@ -1187,7 +1189,8 @@ static CURLcode cf_h2_proxy_shutdown(struct Curl_cfilter *cf, if(rv) { failf(data, "nghttp2_submit_goaway() failed: %s(%d)", nghttp2_strerror(rv), rv); - return CURLE_SEND_ERROR; + result = CURLE_SEND_ERROR; + goto out; } ctx->sent_goaway = TRUE; } @@ -1198,9 +1201,12 @@ static CURLcode cf_h2_proxy_shutdown(struct Curl_cfilter *cf, if(!result && nghttp2_session_want_read(ctx->h2)) result = proxy_h2_progress_ingress(cf, data); - *done = !result && !nghttp2_session_want_write(ctx->h2) && - !nghttp2_session_want_read(ctx->h2); - ctx->shutdown = (result || *done); + *done = (ctx->conn_closed || + (!result && !nghttp2_session_want_write(ctx->h2) && + !nghttp2_session_want_read(ctx->h2))); +out: + CF_DATA_RESTORE(cf, save); + cf->shutdown = (result || *done); return result; } @@ -1240,7 +1246,7 @@ static void cf_h2_proxy_adjust_pollset(struct Curl_cfilter *cf, Curl_pollset_set(data, ps, sock, want_recv, want_send); CF_DATA_RESTORE(cf, save); } - else if(ctx->sent_goaway && !ctx->shutdown) { + else if(ctx->sent_goaway && !cf->shutdown) { /* shutdown in progress */ CF_DATA_SAVE(save, cf, data); want_send = nghttp2_session_want_write(ctx->h2); diff --git a/lib/cfilters.c b/lib/cfilters.c index c21e5cbd4b..9a5c3578d7 100644 --- a/lib/cfilters.c +++ b/lib/cfilters.c @@ -186,8 +186,11 @@ CURLcode Curl_conn_shutdown(struct Curl_easy *data, int sockindex, bool *done) struct curltime now; DEBUGASSERT(data->conn); - /* it is valid to call that without filters being present */ + /* Get the first connected filter that is not shut down already. */ cf = data->conn->cfilter[sockindex]; + while(cf && (!cf->connected || cf->shutdown)) + cf = cf->next; + if(!cf) { *done = TRUE; return CURLE_OK; @@ -209,17 +212,20 @@ CURLcode Curl_conn_shutdown(struct Curl_easy *data, int sockindex, bool *done) } while(cf) { - bool cfdone = FALSE; - result = cf->cft->do_shutdown(cf, data, &cfdone); - if(result) { - CURL_TRC_CF(data, cf, "shut down failed with %d", result); - return result; - } - else if(!cfdone) { - CURL_TRC_CF(data, cf, "shut down not done yet"); - return CURLE_OK; + if(!cf->shutdown) { + bool cfdone = FALSE; + result = cf->cft->do_shutdown(cf, data, &cfdone); + if(result) { + CURL_TRC_CF(data, cf, "shut down failed with %d", result); + return result; + } + else if(!cfdone) { + CURL_TRC_CF(data, cf, "shut down not done yet"); + return CURLE_OK; + } + CURL_TRC_CF(data, cf, "shut down successfully"); + cf->shutdown = TRUE; } - CURL_TRC_CF(data, cf, "shut down successfully"); cf = cf->next; } *done = (!result); @@ -502,6 +508,9 @@ void Curl_conn_cf_adjust_pollset(struct Curl_cfilter *cf, /* Get the lowest not-connected filter, if there are any */ while(cf && !cf->connected && cf->next && !cf->next->connected) cf = cf->next; + /* Skip all filters that have already shut down */ + while(cf && cf->shutdown) + cf = cf->next; /* From there on, give all filters a chance to adjust the pollset. * Lower filters are called later, so they may override */ while(cf) { diff --git a/lib/cfilters.h b/lib/cfilters.h index d7d886045a..e4e0fa5fa2 100644 --- a/lib/cfilters.h +++ b/lib/cfilters.h @@ -223,6 +223,7 @@ struct Curl_cfilter { struct connectdata *conn; /* the connection this filter belongs to */ int sockindex; /* the index the filter is installed at */ BIT(connected); /* != 0 iff this filter is connected */ + BIT(shutdown); /* != 0 iff this filter has shut down */ }; /* Default implementations for the type functions, implementing nop. */ diff --git a/lib/conncache.c b/lib/conncache.c index 2b5ee4f25f..cc9765742b 100644 --- a/lib/conncache.c +++ b/lib/conncache.c @@ -29,13 +29,17 @@ #include "urldata.h" #include "url.h" +#include "cfilters.h" #include "progress.h" #include "multiif.h" #include "sendf.h" #include "conncache.h" +#include "http_negotiate.h" +#include "http_ntlm.h" #include "share.h" #include "sigpipe.h" #include "connect.h" +#include "select.h" #include "strcase.h" /* The last 3 #include files should be in this order */ @@ -45,6 +49,24 @@ #define HASHKEY_SIZE 128 +static void connc_discard_conn(struct conncache *connc, + struct Curl_easy *last_data, + struct connectdata *conn, + bool aborted); +static void connc_disconnect(struct Curl_easy *data, + struct connectdata *conn, + struct conncache *connc, + bool do_shutdown); +static void connc_run_conn_shutdown(struct Curl_easy *data, + struct connectdata *conn, + bool *done); +static void connc_run_conn_shutdown_handler(struct Curl_easy *data, + struct connectdata *conn); +static CURLcode connc_update_shutdown_ev(struct Curl_multi *multi, + struct Curl_easy *data, + struct connectdata *conn); +static void connc_shutdown_all(struct conncache *connc, int timeout_ms); + static CURLcode bundle_create(struct connectbundle **bundlep) { DEBUGASSERT(*bundlep == NULL); @@ -100,25 +122,35 @@ static void free_bundle_hash_entry(void *freethis) bundle_destroy(b); } -int Curl_conncache_init(struct conncache *connc, size_t size) +int Curl_conncache_init(struct conncache *connc, + struct Curl_multi *multi, size_t size) { /* allocate a new easy handle to use when closing cached connections */ connc->closure_handle = curl_easy_init(); if(!connc->closure_handle) return 1; /* bad */ connc->closure_handle->state.internal = true; + #ifdef DEBUGBUILD + if(getenv("CURL_DEBUG")) + connc->closure_handle->set.verbose = true; +#endif Curl_hash_init(&connc->hash, size, Curl_hash_str, Curl_str_key_compare, free_bundle_hash_entry); connc->closure_handle->state.conn_cache = connc; + connc->multi = multi; + Curl_llist_init(&connc->shutdowns.conn_list, NULL); return 0; /* good */ } void Curl_conncache_destroy(struct conncache *connc) { - if(connc) + if(connc) { Curl_hash_destroy(&connc->hash); + connc->multi = NULL; + DEBUGASSERT(!Curl_llist_count(&connc->shutdowns.conn_list)); + } } /* creates a key to find a bundle for this connection */ @@ -180,15 +212,14 @@ Curl_conncache_find_bundle(struct Curl_easy *data, return bundle; } -static void *conncache_add_bundle(struct conncache *connc, - char *key, - struct connectbundle *bundle) +static void *connc_add_bundle(struct conncache *connc, + char *key, struct connectbundle *bundle) { return Curl_hash_add(&connc->hash, key, strlen(key), bundle); } -static void conncache_remove_bundle(struct conncache *connc, - struct connectbundle *bundle) +static void connc_remove_bundle(struct conncache *connc, + struct connectbundle *bundle) { struct Curl_hash_iterator iter; struct Curl_hash_element *he; @@ -231,7 +262,7 @@ CURLcode Curl_conncache_add_conn(struct Curl_easy *data) hashkey(conn, key, sizeof(key)); - if(!conncache_add_bundle(data->state.conn_cache, key, bundle)) { + if(!connc_add_bundle(data->state.conn_cache, key, bundle)) { bundle_destroy(bundle); result = CURLE_OUT_OF_MEMORY; goto unlock; @@ -252,6 +283,23 @@ unlock: return result; } +static void connc_remove_conn(struct conncache *connc, + struct connectdata *conn) +{ + struct connectbundle *bundle = conn->bundle; + + /* The bundle pointer can be NULL, since this function can be called + due to a failed connection attempt, before being added to a bundle */ + if(bundle) { + bundle_remove_conn(bundle, conn); + if(connc && bundle->num_connections == 0) + connc_remove_bundle(connc, bundle); + conn->bundle = NULL; /* removed from it */ + if(connc) + connc->num_conn--; + } +} + /* * Removes the connectdata object from the connection cache, but the transfer * still owns this connection. @@ -262,28 +310,16 @@ unlock: void Curl_conncache_remove_conn(struct Curl_easy *data, struct connectdata *conn, bool lock) { - struct connectbundle *bundle = conn->bundle; struct conncache *connc = data->state.conn_cache; - /* The bundle pointer can be NULL, since this function can be called - due to a failed connection attempt, before being added to a bundle */ - if(bundle) { - if(lock) { - CONNCACHE_LOCK(data); - } - bundle_remove_conn(bundle, conn); - if(bundle->num_connections == 0) - conncache_remove_bundle(connc, bundle); - conn->bundle = NULL; /* removed from it */ - if(connc) { - connc->num_conn--; - DEBUGF(infof(data, "The cache now contains %zu members", - connc->num_conn)); - } - if(lock) { - CONNCACHE_UNLOCK(data); - } - } + if(lock) + CONNCACHE_LOCK(data); + connc_remove_conn(connc, conn); + if(lock) + CONNCACHE_UNLOCK(data); + if(connc) + DEBUGF(infof(data, "The cache now contains %zu members", + connc->num_conn)); } /* This function iterates the entire connection cache and calls the function @@ -345,7 +381,7 @@ bool Curl_conncache_foreach(struct Curl_easy *data, up a cache! */ static struct connectdata * -conncache_find_first_connection(struct conncache *connc) +connc_find_first_connection(struct conncache *connc) { struct Curl_hash_iterator iter; struct Curl_hash_element *he; @@ -394,8 +430,7 @@ bool Curl_conncache_return_conn(struct Curl_easy *data, important that details from this (unrelated) disconnect does not taint meta-data in the data handle. */ struct conncache *connc = data->state.conn_cache; - Curl_disconnect(connc->closure_handle, conn_candidate, - /* dead_connection */ FALSE); + connc_disconnect(NULL, conn_candidate, connc, TRUE); } } @@ -516,33 +551,606 @@ Curl_conncache_extract_oldest(struct Curl_easy *data) return conn_candidate; } -void Curl_conncache_close_all_connections(struct conncache *connc) +static void connc_shutdown_discard_all(struct conncache *connc) +{ + struct Curl_llist_element *e = connc->shutdowns.conn_list.head; + struct connectdata *conn; + + if(!e) + return; + + DEBUGF(infof(connc->closure_handle, "conncache_shutdown_discard_all")); + DEBUGASSERT(!connc->shutdowns.iter_locked); + connc->shutdowns.iter_locked = TRUE; + while(e) { + conn = e->ptr; + Curl_llist_remove(&connc->shutdowns.conn_list, e, NULL); + DEBUGF(infof(connc->closure_handle, "discard connection #%" + CURL_FORMAT_CURL_OFF_T, conn->connection_id)); + connc_disconnect(NULL, conn, connc, FALSE); + e = connc->shutdowns.conn_list.head; + } + connc->shutdowns.iter_locked = FALSE; +} + +static void connc_close_all(struct conncache *connc) { + struct Curl_easy *data = connc->closure_handle; struct connectdata *conn; + int timeout_ms = 0; SIGPIPE_VARIABLE(pipe_st); - if(!connc->closure_handle) + + if(!data) return; - conn = conncache_find_first_connection(connc); + /* Move all connections to the shutdown list */ + conn = connc_find_first_connection(connc); while(conn) { - sigpipe_ignore(connc->closure_handle, &pipe_st); + connc_remove_conn(connc, conn); + sigpipe_ignore(data, &pipe_st); /* This will remove the connection from the cache */ connclose(conn, "kill all"); Curl_conncache_remove_conn(connc->closure_handle, conn, TRUE); - Curl_disconnect(connc->closure_handle, conn, FALSE); + connc_discard_conn(connc, connc->closure_handle, conn, FALSE); sigpipe_restore(&pipe_st); - conn = conncache_find_first_connection(connc); + conn = connc_find_first_connection(connc); } - sigpipe_ignore(connc->closure_handle, &pipe_st); + /* Just for testing, run graceful shutdown */ +#ifdef DEBUGBUILD + { + char *p = getenv("CURL_GRACEFUL_SHUTDOWN"); + if(p) { + long l = strtol(p, NULL, 10); + if(l > 0 && l < INT_MAX) + timeout_ms = (int)l; + } + } +#endif + connc_shutdown_all(connc, timeout_ms); + + /* discard all connections in the shutdown list */ + connc_shutdown_discard_all(connc); - Curl_hostcache_clean(connc->closure_handle, - connc->closure_handle->dns.hostcache); - Curl_close(&connc->closure_handle); + sigpipe_ignore(data, &pipe_st); + Curl_hostcache_clean(data, data->dns.hostcache); + Curl_close(&data); sigpipe_restore(&pipe_st); } +void Curl_conncache_close_all_connections(struct conncache *connc) +{ + connc_close_all(connc); +} + +static void connc_shutdown_discard_oldest(struct conncache *connc) +{ + struct Curl_llist_element *e; + struct connectdata *conn; + SIGPIPE_VARIABLE(pipe_st); + + DEBUGASSERT(!connc->shutdowns.iter_locked); + if(connc->shutdowns.iter_locked) + return; + + e = connc->shutdowns.conn_list.head; + if(e) { + conn = e->ptr; + Curl_llist_remove(&connc->shutdowns.conn_list, e, NULL); + sigpipe_ignore(connc->closure_handle, &pipe_st); + connc_disconnect(NULL, conn, connc, FALSE); + sigpipe_restore(&pipe_st); + } +} + +static void connc_discard_conn(struct conncache *connc, + struct Curl_easy *last_data, + struct connectdata *conn, + bool aborted) +{ + /* `last_data`, if present, is the transfer that last worked with + * the connection. It is present when the connection is being shut down + * via `Curl_conncache_discard_conn()`, e.g. when the transfer failed + * or does not allow connection reuse. + * Using the original handle is necessary for shutting down the protocol + * handler belonging to the connection. Protocols like 'file:' rely on + * being invoked to clean up their allocations in the easy handle. + * When a connection comes from the cache, the transfer is no longer + * there and we use the cache's own closure handle. + */ + struct Curl_easy *data = last_data? last_data : connc->closure_handle; + bool done = FALSE; + + DEBUGASSERT(connc); + DEBUGASSERT(!conn->bundle); + + /* + * If this connection isn't marked to force-close, leave it open if there + * are other users of it + */ + if(CONN_INUSE(conn) && !aborted) { + DEBUGF(infof(data, "[CCACHE] not discarding #%" CURL_FORMAT_CURL_OFF_T + " still in use by %zu transfers", conn->connection_id, + CONN_INUSE(conn))); + return; + } + + /* treat the connection as aborted in CONNECT_ONLY situations, we do + * not know what the APP did with it. */ + if(conn->connect_only) + aborted = TRUE; + conn->bits.aborted = aborted; + + /* We do not shutdown dead connections. The term 'dead' can be misleading + * here, as we also mark errored connections/transfers as 'dead'. + * If we do a shutdown for an aborted transfer, the server might think + * it was successful otherwise (for example an ftps: upload). This is + * not what we want. */ + if(aborted) + done = TRUE; + else if(!done) { + /* Attempt to shutdown the connection right away. */ + Curl_attach_connection(data, conn); + connc_run_conn_shutdown(data, conn, &done); + DEBUGF(infof(data, "[CCACHE] shutdown #%" CURL_FORMAT_CURL_OFF_T + ", done=%d",conn->connection_id, done)); + Curl_detach_connection(data); + } + + if(done) { + connc_disconnect(data, conn, connc, FALSE); + return; + } + + DEBUGASSERT(!connc->shutdowns.iter_locked); + if(connc->shutdowns.iter_locked) { + DEBUGF(infof(data, "[CCACHE] discarding #%" CURL_FORMAT_CURL_OFF_T + ", list locked", conn->connection_id)); + connc_disconnect(data, conn, connc, FALSE); + return; + } + + /* Add the connection to our shutdown list for non-blocking shutdown + * during multi processing. */ + if(data->multi && data->multi->max_shutdown_connections > 0 && + (data->multi->max_shutdown_connections >= + (long)Curl_llist_count(&connc->shutdowns.conn_list))) { + DEBUGF(infof(data, "[CCACHE] discarding oldest shutdown connection " + "due to limit of %ld", + data->multi->max_shutdown_connections)); + connc_shutdown_discard_oldest(connc); + } + + if(data->multi && data->multi->socket_cb) { + DEBUGASSERT(connc == &data->multi->conn_cache); + if(connc_update_shutdown_ev(data->multi, data, conn)) { + DEBUGF(infof(data, "[CCACHE] update events for shutdown failed, " + "discarding #%" CURL_FORMAT_CURL_OFF_T, + conn->connection_id)); + connc_disconnect(data, conn, connc, FALSE); + return; + } + } + + Curl_llist_append(&connc->shutdowns.conn_list, conn, &conn->bundle_node); + DEBUGF(infof(data, "[CCACHE] added #%" CURL_FORMAT_CURL_OFF_T + " to shutdown list of length %zu", conn->connection_id, + Curl_llist_count(&connc->shutdowns.conn_list))); + + /* Forget what this transfer last polled, the connection is ours now. + * If we do not clear this, the event handling for `data` will tell + * the callback to remove the connection socket after we return here. */ + memset(&data->last_poll, 0, sizeof(data->last_poll)); +} + +void Curl_conncache_disconnect(struct Curl_easy *data, + struct connectdata *conn, + bool aborted) +{ + DEBUGASSERT(data); + /* Connection must no longer be in and connection cache */ + DEBUGASSERT(!conn->bundle); + + if(data->multi) { + /* Add it to the multi's conncache for shutdown handling */ + infof(data, "%s connection #%" CURL_FORMAT_CURL_OFF_T, + aborted? "closing" : "shutting down", conn->connection_id); + connc_discard_conn(&data->multi->conn_cache, data, conn, aborted); + } + else { + /* No multi available. Make a best-effort shutdown + close */ + infof(data, "closing connection #%" CURL_FORMAT_CURL_OFF_T, + conn->connection_id); + DEBUGASSERT(!conn->bundle); + connc_run_conn_shutdown_handler(data, conn); + connc_disconnect(data, conn, NULL, !aborted); + } +} + +static void connc_run_conn_shutdown_handler(struct Curl_easy *data, + struct connectdata *conn) +{ + if(!conn->bits.shutdown_handler) { + if(conn->dns_entry) { + Curl_resolv_unlock(data, conn->dns_entry); + conn->dns_entry = NULL; + } + + /* Cleanup NTLM connection-related data */ + Curl_http_auth_cleanup_ntlm(conn); + + /* Cleanup NEGOTIATE connection-related data */ + Curl_http_auth_cleanup_negotiate(conn); + + if(conn->handler && conn->handler->disconnect) { + /* This is set if protocol-specific cleanups should be made */ + DEBUGF(infof(data, "connection #%" CURL_FORMAT_CURL_OFF_T + ", shutdown protocol handler (aborted=%d)", + conn->connection_id, conn->bits.aborted)); + conn->handler->disconnect(data, conn, conn->bits.aborted); + } + + /* possible left-overs from the async name resolvers */ + Curl_resolver_cancel(data); + + conn->bits.shutdown_handler = TRUE; + } +} + +static void connc_run_conn_shutdown(struct Curl_easy *data, + struct connectdata *conn, + bool *done) +{ + CURLcode r1, r2; + bool done1, done2; + + /* We expect to be attached when called */ + DEBUGASSERT(data->conn == conn); + + connc_run_conn_shutdown_handler(data, conn); + + if(conn->bits.shutdown_filters) { + *done = TRUE; + return; + } + + if(!conn->connect_only && Curl_conn_is_connected(conn, FIRSTSOCKET)) + r1 = Curl_conn_shutdown(data, FIRSTSOCKET, &done1); + else { + r1 = CURLE_OK; + done1 = TRUE; + } + + if(!conn->connect_only && Curl_conn_is_connected(conn, SECONDARYSOCKET)) + r2 = Curl_conn_shutdown(data, SECONDARYSOCKET, &done2); + else { + r2 = CURLE_OK; + done2 = TRUE; + } + + /* we are done when any failed or both report success */ + *done = (r1 || r2 || (done1 && done2)); + if(*done) + conn->bits.shutdown_filters = TRUE; +} + +CURLcode Curl_conncache_add_pollfds(struct conncache *connc, + struct curl_pollfds *cpfds) +{ + CURLcode result = CURLE_OK; + + DEBUGASSERT(!connc->shutdowns.iter_locked); + connc->shutdowns.iter_locked = TRUE; + if(connc->shutdowns.conn_list.head) { + struct Curl_llist_element *e; + struct easy_pollset ps; + struct connectdata *conn; + + for(e = connc->shutdowns.conn_list.head; e; e = e->next) { + conn = e->ptr; + memset(&ps, 0, sizeof(ps)); + Curl_attach_connection(connc->closure_handle, conn); + Curl_conn_adjust_pollset(connc->closure_handle, &ps); + Curl_detach_connection(connc->closure_handle); + + result = Curl_pollfds_add_ps(cpfds, &ps); + if(result) { + Curl_pollfds_cleanup(cpfds); + goto out; + } + } + } +out: + connc->shutdowns.iter_locked = FALSE; + return result; +} + +CURLcode Curl_conncache_add_waitfds(struct conncache *connc, + struct curl_waitfds *cwfds) +{ + CURLcode result = CURLE_OK; + + DEBUGASSERT(!connc->shutdowns.iter_locked); + connc->shutdowns.iter_locked = TRUE; + if(connc->shutdowns.conn_list.head) { + struct Curl_llist_element *e; + struct easy_pollset ps; + struct connectdata *conn; + + for(e = connc->shutdowns.conn_list.head; e; e = e->next) { + conn = e->ptr; + memset(&ps, 0, sizeof(ps)); + Curl_attach_connection(connc->closure_handle, conn); + Curl_conn_adjust_pollset(connc->closure_handle, &ps); + Curl_detach_connection(connc->closure_handle); + + result = Curl_waitfds_add_ps(cwfds, &ps); + if(result) + goto out; + } + } +out: + connc->shutdowns.iter_locked = FALSE; + return result; +} + +static void connc_perform(struct conncache *connc) +{ + struct Curl_easy *data = connc->closure_handle; + struct Curl_llist_element *e = connc->shutdowns.conn_list.head; + struct Curl_llist_element *enext; + struct connectdata *conn; + bool done; + + if(!e) + return; + + DEBUGASSERT(!connc->shutdowns.iter_locked); + DEBUGF(infof(data, "[CCACHE] perform, %zu connections being shutdown", + Curl_llist_count(&connc->shutdowns.conn_list))); + connc->shutdowns.iter_locked = TRUE; + while(e) { + enext = e->next; + conn = e->ptr; + Curl_attach_connection(data, conn); + connc_run_conn_shutdown(data, conn, &done); + DEBUGF(infof(data, "[CCACHE] shutdown #%" CURL_FORMAT_CURL_OFF_T + ", done=%d", conn->connection_id, done)); + Curl_detach_connection(data); + if(done) { + Curl_llist_remove(&connc->shutdowns.conn_list, e, NULL); + connc_disconnect(NULL, conn, connc, FALSE); + } + e = enext; + } + connc->shutdowns.iter_locked = FALSE; +} + +void Curl_conncache_multi_perform(struct Curl_multi *multi) +{ + connc_perform(&multi->conn_cache); +} + + +/* + * Disconnects the given connection. Note the connection may not be the + * primary connection, like when freeing room in the connection cache or + * killing of a dead old connection. + * + * A connection needs an easy handle when closing down. We support this passed + * in separately since the connection to get closed here is often already + * disassociated from an easy handle. + * + * This function MUST NOT reset state in the Curl_easy struct if that + * isn't strictly bound to the life-time of *this* particular connection. + * + */ +static void connc_disconnect(struct Curl_easy *data, + struct connectdata *conn, + struct conncache *connc, + bool do_shutdown) +{ + bool done; + + /* there must be a connection to close */ + DEBUGASSERT(conn); + /* it must be removed from the connection cache */ + DEBUGASSERT(!conn->bundle); + /* there must be an associated transfer */ + DEBUGASSERT(data || connc); + if(!data) + data = connc->closure_handle; + + /* the transfer must be detached from the connection */ + DEBUGASSERT(data && !data->conn); + + if(connc && connc->multi && connc->multi->socket_cb) { + unsigned int i; + for(i = 0; i < 2; ++i) { + if(CURL_SOCKET_BAD == conn->sock[i]) + continue; + /* remove all connection's sockets from event handling */ + connc->multi->in_callback = TRUE; + connc->multi->socket_cb(data, conn->sock[i], CURL_POLL_REMOVE, + connc->multi->socket_userp, NULL); + connc->multi->in_callback = FALSE; + } + } + + Curl_attach_connection(data, conn); + + connc_run_conn_shutdown_handler(data, conn); + if(do_shutdown) { + /* Make a last attempt to shutdown handlers and filters, if + * not done so already. */ + connc_run_conn_shutdown(data, conn, &done); + } + + if(connc) + DEBUGF(infof(data, "[CCACHE] closing #%" CURL_FORMAT_CURL_OFF_T, + conn->connection_id)); + else + DEBUGF(infof(data, "closing connection #%" CURL_FORMAT_CURL_OFF_T, + conn->connection_id)); + Curl_conn_close(data, SECONDARYSOCKET); + Curl_conn_close(data, FIRSTSOCKET); + Curl_detach_connection(data); + + Curl_conn_free(data, conn); +} + + +static CURLcode connc_update_shutdown_ev(struct Curl_multi *multi, + struct Curl_easy *data, + struct connectdata *conn) +{ + struct easy_pollset ps; + unsigned int i; + int rc; + + DEBUGASSERT(data); + DEBUGASSERT(multi); + DEBUGASSERT(multi->socket_cb); + + memset(&ps, 0, sizeof(ps)); + Curl_attach_connection(data, conn); + Curl_conn_adjust_pollset(data, &ps); + Curl_detach_connection(data); + + if(!ps.num) + return CURLE_FAILED_INIT; + + for(i = 0; i < ps.num; ++i) { + DEBUGF(infof(data, "[CCACHE] set socket=%" CURL_FORMAT_SOCKET_T + " events=%d on #%" CURL_FORMAT_CURL_OFF_T, + ps.sockets[i], ps.actions[i], conn->connection_id)); + multi->in_callback = TRUE; + rc = multi->socket_cb(data, ps.sockets[i], ps.actions[i], + multi->socket_userp, NULL); + multi->in_callback = FALSE; + if(rc == -1) + return CURLE_FAILED_INIT; + } + + return CURLE_OK; +} + +void Curl_conncache_multi_socket(struct Curl_multi *multi, + curl_socket_t s, int ev_bitmask) +{ + struct conncache *connc = &multi->conn_cache; + struct Curl_easy *data = connc->closure_handle; + struct Curl_llist_element *e = connc->shutdowns.conn_list.head; + struct connectdata *conn; + bool done; + + (void)ev_bitmask; + DEBUGASSERT(multi->socket_cb); + if(!e) + return; + + connc->shutdowns.iter_locked = TRUE; + while(e) { + conn = e->ptr; + if(s == conn->sock[FIRSTSOCKET] || s == conn->sock[SECONDARYSOCKET]) { + Curl_attach_connection(data, conn); + connc_run_conn_shutdown(data, conn, &done); + DEBUGF(infof(data, "[CCACHE] shutdown #%" CURL_FORMAT_CURL_OFF_T + ", done=%d", conn->connection_id, done)); + Curl_detach_connection(data); + if(done || connc_update_shutdown_ev(multi, data, conn)) { + Curl_llist_remove(&connc->shutdowns.conn_list, e, NULL); + connc_disconnect(NULL, conn, connc, FALSE); + } + break; + } + e = e->next; + } + connc->shutdowns.iter_locked = FALSE; +} + +void Curl_conncache_multi_close_all(struct Curl_multi *multi) +{ + connc_close_all(&multi->conn_cache); +} + + +#define NUM_POLLS_ON_STACK 10 + +static CURLcode connc_shutdown_wait(struct conncache *connc, int timeout_ms) +{ + struct pollfd a_few_on_stack[NUM_POLLS_ON_STACK]; + struct curl_pollfds cpfds; + CURLcode result; + + Curl_pollfds_init(&cpfds, a_few_on_stack, NUM_POLLS_ON_STACK); + + result = Curl_conncache_add_pollfds(connc, &cpfds); + if(result) + goto out; + + Curl_poll(cpfds.pfds, cpfds.n, CURLMIN(timeout_ms, 1000)); + +out: + Curl_pollfds_cleanup(&cpfds); + return result; +} + +static void connc_shutdown_all(struct conncache *connc, int timeout_ms) +{ + struct Curl_easy *data = connc->closure_handle; + struct connectdata *conn; + struct curltime started = Curl_now(); + + if(!data) + return; + (void)data; + + DEBUGF(infof(data, "conncache shutdown all")); + + /* Move all connections into the shutdown queue */ + conn = connc_find_first_connection(connc); + while(conn) { + /* This will remove the connection from the cache */ + DEBUGF(infof(data, "moving connection %" CURL_FORMAT_CURL_OFF_T + " to shutdown queue", conn->connection_id)); + connc_remove_conn(connc, conn); + connc_discard_conn(connc, NULL, conn, FALSE); + conn = connc_find_first_connection(connc); + } + + DEBUGASSERT(!connc->shutdowns.iter_locked); + while(connc->shutdowns.conn_list.head) { + timediff_t timespent; + int remain_ms; + + connc_perform(connc); + + if(!connc->shutdowns.conn_list.head) { + DEBUGF(infof(data, "conncache shutdown ok")); + break; + } + + /* wait for activity, timeout or "nothing" */ + timespent = Curl_timediff(Curl_now(), started); + if(timespent >= (timediff_t)timeout_ms) { + DEBUGF(infof(data, "conncache shutdown %s", + (timeout_ms > 0)? "timeout" : "best effort done")); + break; + } + + remain_ms = timeout_ms - (int)timespent; + if(connc_shutdown_wait(connc, remain_ms)) { + DEBUGF(infof(data, "conncache shutdown all, abort")); + break; + } + } + + /* Due to errors/timeout, we might come here without being full ydone. */ + connc_shutdown_discard_all(connc); +} + #if 0 /* Useful for debugging the connection cache */ void Curl_conncache_print(struct conncache *connc) diff --git a/lib/conncache.h b/lib/conncache.h index 295057e42f..f9ee3fa2b7 100644 --- a/lib/conncache.h +++ b/lib/conncache.h @@ -35,6 +35,14 @@ #include "timeval.h" struct connectdata; +struct curl_pollfds; +struct curl_waitfds; +struct Curl_multi; + +struct connshutdowns { + struct Curl_llist conn_list; /* The connectdata to shut down */ + BIT(iter_locked); /* TRUE while iterating the list */ +}; struct conncache { struct Curl_hash hash; @@ -42,8 +50,10 @@ struct conncache { curl_off_t next_connection_id; curl_off_t next_easy_id; struct curltime last_cleanup; + struct connshutdowns shutdowns; /* handle used for closing cached connections */ struct Curl_easy *closure_handle; + struct Curl_multi *multi; /* Optional, set if cache belongs to multi */ }; #define BUNDLE_NO_MULTIUSE -1 @@ -84,8 +94,12 @@ struct connectbundle { struct Curl_llist conn_list; /* The connectdata members of the bundle */ }; -/* returns 1 on error, 0 is fine */ -int Curl_conncache_init(struct conncache *, size_t size); +/* Init the cache, pass multi only if cache is owned by it. + * returns 1 on error, 0 is fine. + */ +int Curl_conncache_init(struct conncache *, + struct Curl_multi *multi, + size_t size); void Curl_conncache_destroy(struct conncache *connc); /* return the correct bundle, to a host or a proxy */ @@ -119,4 +133,32 @@ Curl_conncache_extract_oldest(struct Curl_easy *data); void Curl_conncache_close_all_connections(struct conncache *connc); void Curl_conncache_print(struct conncache *connc); +/** + * Tear down the connection. If `aborted` is FALSE, the connection + * will be shut down first before discarding. If the shutdown + * is not immediately complete, the connection + * will be placed into the cache's shutdown queue. + */ +void Curl_conncache_disconnect(struct Curl_easy *data, + struct connectdata *conn, + bool aborted); + +/** + * Add sockets and POLLIN/OUT flags for connections handled by the cache. + */ +CURLcode Curl_conncache_add_pollfds(struct conncache *connc, + struct curl_pollfds *cpfds); +CURLcode Curl_conncache_add_waitfds(struct conncache *connc, + struct curl_waitfds *cwfds); + +/** + * Perform maintenance on connections in the cache. Specifically, + * progress the shutdown of connections in the queue. + */ +void Curl_conncache_multi_perform(struct Curl_multi *multi); + +void Curl_conncache_multi_socket(struct Curl_multi *multi, + curl_socket_t s, int ev_bitmask); +void Curl_conncache_multi_close_all(struct Curl_multi *multi); + #endif /* HEADER_CURL_CONNCACHE_H */ diff --git a/lib/http2.c b/lib/http2.c index 6a30354d63..7fb1d88bd9 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -140,7 +140,6 @@ struct cf_h2_ctx { BIT(rcvd_goaway); BIT(sent_goaway); BIT(enable_push); - BIT(shutdown); BIT(nw_out_blocked); }; @@ -2375,7 +2374,7 @@ static void cf_h2_adjust_pollset(struct Curl_cfilter *cf, Curl_pollset_set(data, ps, sock, want_recv, want_send); CF_DATA_RESTORE(cf, save); } - else if(ctx->sent_goaway && !ctx->shutdown) { + else if(ctx->sent_goaway && !cf->shutdown) { /* shutdown in progress */ CF_DATA_SAVE(save, cf, data); want_send = nghttp2_session_want_write(ctx->h2); @@ -2467,14 +2466,17 @@ static CURLcode cf_h2_shutdown(struct Curl_cfilter *cf, struct Curl_easy *data, bool *done) { struct cf_h2_ctx *ctx = cf->ctx; + struct cf_call_data save; CURLcode result; int rv; - if(!cf->connected || !ctx->h2 || ctx->shutdown) { + if(!cf->connected || !ctx->h2 || cf->shutdown || ctx->conn_closed) { *done = TRUE; return CURLE_OK; } + CF_DATA_SAVE(save, cf, data); + if(!ctx->sent_goaway) { rv = nghttp2_submit_goaway(ctx->h2, NGHTTP2_FLAG_NONE, ctx->local_max_sid, 0, @@ -2482,7 +2484,8 @@ static CURLcode cf_h2_shutdown(struct Curl_cfilter *cf, if(rv) { failf(data, "nghttp2_submit_goaway() failed: %s(%d)", nghttp2_strerror(rv), rv); - return CURLE_SEND_ERROR; + result = CURLE_SEND_ERROR; + goto out; } ctx->sent_goaway = TRUE; } @@ -2493,9 +2496,13 @@ static CURLcode cf_h2_shutdown(struct Curl_cfilter *cf, if(!result && nghttp2_session_want_read(ctx->h2)) result = h2_progress_ingress(cf, data, 0); - *done = !result && !nghttp2_session_want_write(ctx->h2) && - !nghttp2_session_want_read(ctx->h2); - ctx->shutdown = (result || *done); + *done = (ctx->conn_closed || + (!result && !nghttp2_session_want_write(ctx->h2) && + !nghttp2_session_want_read(ctx->h2))); + +out: + CF_DATA_RESTORE(cf, save); + cf->shutdown = (result || *done); return result; } diff --git a/lib/multi.c b/lib/multi.c index 421db8465c..d510e43bc3 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -410,7 +410,7 @@ struct Curl_multi *Curl_multi_handle(size_t hashsize, /* socket hash */ Curl_hash_init(&multi->proto_hash, 23, Curl_hash_str, Curl_str_key_compare, ph_freeentry); - if(Curl_conncache_init(&multi->conn_cache, chashsize)) + if(Curl_conncache_init(&multi->conn_cache, multi, chashsize)) goto error; Curl_llist_init(&multi->msglist, NULL); @@ -1248,6 +1248,7 @@ CURLMcode curl_multi_waitfds(struct Curl_multi *multi, struct Curl_easy *data; struct curl_waitfds cwfds; struct easy_pollset ps; + CURLMcode result = CURLM_OK; if(!ufds) return CURLM_BAD_FUNCTION_ARGUMENT; @@ -1262,13 +1263,21 @@ CURLMcode curl_multi_waitfds(struct Curl_multi *multi, memset(&ps, 0, sizeof(ps)); for(data = multi->easyp; data; data = data->next) { multi_getsock(data, &ps); - if(Curl_waitfds_add_ps(&cwfds, &ps)) - return CURLM_OUT_OF_MEMORY; + if(Curl_waitfds_add_ps(&cwfds, &ps)) { + result = CURLM_OUT_OF_MEMORY; + goto out; + } } + if(Curl_conncache_add_waitfds(&multi->conn_cache, &cwfds)) { + result = CURLM_OUT_OF_MEMORY; + goto out; + } + +out: if(fd_count) *fd_count = cwfds.n; - return CURLM_OK; + return result; } #ifdef USE_WINSOCK @@ -1305,6 +1314,7 @@ static CURLMcode multi_wait(struct Curl_multi *multi, struct pollfd a_few_on_stack[NUM_POLLS_ON_STACK]; struct curl_pollfds cpfds; unsigned int curl_nfds = 0; /* how many pfds are for curl transfers */ + CURLMcode result = CURLM_OK; #ifdef USE_WINSOCK WSANETWORKEVENTS wsa_events; DEBUGASSERT(multi->wsa_event != WSA_INVALID_EVENT); @@ -1329,11 +1339,16 @@ static CURLMcode multi_wait(struct Curl_multi *multi, for(data = multi->easyp; data; data = data->next) { multi_getsock(data, &ps); if(Curl_pollfds_add_ps(&cpfds, &ps)) { - Curl_pollfds_cleanup(&cpfds); - return CURLM_OUT_OF_MEMORY; + result = CURLM_OUT_OF_MEMORY; + goto out; } } + if(Curl_conncache_add_pollfds(&multi->conn_cache, &cpfds)) { + result = CURLM_OUT_OF_MEMORY; + goto out; + } + curl_nfds = cpfds.n; /* what curl internally uses in cpfds */ /* Add external file descriptions from poll-like struct curl_waitfd */ for(i = 0; i < extra_nfds; i++) { @@ -1345,8 +1360,8 @@ static CURLMcode multi_wait(struct Curl_multi *multi, if(extra_fds[i].events & CURL_WAIT_POLLOUT) events |= POLLOUT; if(Curl_pollfds_add_sock(&cpfds, extra_fds[i].fd, events)) { - Curl_pollfds_cleanup(&cpfds); - return CURLM_OUT_OF_MEMORY; + result = CURLM_OUT_OF_MEMORY; + goto out; } } @@ -1364,8 +1379,8 @@ static CURLMcode multi_wait(struct Curl_multi *multi, } if(mask) { if(WSAEventSelect(cpfds.pfds[i].fd, multi->wsa_event, mask) != 0) { - Curl_pollfds_cleanup(&cpfds); - return CURLM_INTERNAL_ERROR; + result = CURLM_OUT_OF_MEMORY; + goto out; } } } @@ -1375,8 +1390,8 @@ static CURLMcode multi_wait(struct Curl_multi *multi, #ifndef USE_WINSOCK if(use_wakeup && multi->wakeup_pair[0] != CURL_SOCKET_BAD) { if(Curl_pollfds_add_sock(&cpfds, multi->wakeup_pair[0], POLLIN)) { - Curl_pollfds_cleanup(&cpfds); - return CURLM_OUT_OF_MEMORY; + result = CURLM_OUT_OF_MEMORY; + goto out; } } #endif @@ -1405,8 +1420,8 @@ static CURLMcode multi_wait(struct Curl_multi *multi, pollrc = Curl_poll(cpfds.pfds, cpfds.n, timeout_ms); /* wait... */ #endif if(pollrc < 0) { - Curl_pollfds_cleanup(&cpfds); - return CURLM_UNRECOVERABLE_POLL; + result = CURLM_UNRECOVERABLE_POLL; + goto out; } if(pollrc > 0) { @@ -1524,8 +1539,9 @@ static CURLMcode multi_wait(struct Curl_multi *multi, } } +out: Curl_pollfds_cleanup(&cpfds); - return CURLM_OK; + return result; } CURLMcode curl_multi_wait(struct Curl_multi *multi, @@ -2695,6 +2711,7 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles) /* the current node might be unlinked in multi_runsingle(), get the next pointer now */ struct Curl_easy *datanext = data->next; + if(data->set.no_signal != nosig) { sigpipe_restore(&pipe_st); sigpipe_ignore(data, &pipe_st); @@ -2703,11 +2720,14 @@ CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles) result = multi_runsingle(multi, &now, data); if(result) returncode = result; + data = datanext; /* operate on next handle */ } while(data); sigpipe_restore(&pipe_st); } + Curl_conncache_multi_perform(multi); + /* * Simply remove all expired timers from the splay since handles are dealt * with unconditionally by this function and curl_multi_timeout() requires @@ -2796,7 +2816,7 @@ CURLMcode curl_multi_cleanup(struct Curl_multi *multi) } /* Close all the connections in the connection cache */ - Curl_conncache_close_all_connections(&multi->conn_cache); + Curl_conncache_multi_close_all(multi); sockhash_destroy(&multi->sockhash); Curl_hash_destroy(&multi->proto_hash); @@ -2877,7 +2897,6 @@ static CURLMcode singlesocket(struct Curl_multi *multi, /* Fill in the 'current' struct with the state as it is now: what sockets to supervise and for what actions */ multi_getsock(data, &cur_poll); - /* We have 0 .. N sockets already and we get to know about the 0 .. M sockets we should have from now on. Detect the differences, remove no longer supervised ones and add new ones */ @@ -3155,13 +3174,16 @@ static CURLMcode multi_socket(struct Curl_multi *multi, if(s != CURL_SOCKET_TIMEOUT) { struct Curl_sh_entry *entry = sh_getentry(&multi->sockhash, s); - if(!entry) + if(!entry) { /* Unmatched socket, we can't act on it but we ignore this fact. In real-world tests it has been proved that libevent can in fact give the application actions even though the socket was just previously asked to get removed, so thus we better survive stray socket actions and just move on. */ - ; + /* The socket might come from a connection that is being shut down + * by the multi's conncache. */ + Curl_conncache_multi_socket(multi, s, ev_bitmask); + } else { struct Curl_hash_iterator iter; struct Curl_hash_element *he; @@ -3294,6 +3316,9 @@ CURLMcode curl_multi_setopt(struct Curl_multi *multi, break; case CURLMOPT_MAX_TOTAL_CONNECTIONS: multi->max_total_connections = va_arg(param, long); + /* for now, let this also decide the max number of connections + * in shutdown handling */ + multi->max_shutdown_connections = va_arg(param, long); break; /* options formerly used for pipelining */ case CURLMOPT_MAX_PIPELINE_LENGTH: diff --git a/lib/multihandle.h b/lib/multihandle.h index d7cf3b0f12..bfc8ce4fa5 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -148,6 +148,8 @@ struct Curl_multi { long max_total_connections; /* if >0, a fixed limit of the maximum number of connections in total */ + long max_shutdown_connections; /* if >0, a fixed limit of the maximum number + of connections in shutdown handling */ /* timer callback and user data pointer for the *socket() API */ curl_multi_timer_callback timer_cb; diff --git a/lib/share.c b/lib/share.c index 8fa5cda00f..9b5ba1fc39 100644 --- a/lib/share.c +++ b/lib/share.c @@ -26,6 +26,7 @@ #include #include "urldata.h" +#include "connect.h" #include "share.h" #include "psl.h" #include "vtls/vtls.h" @@ -119,7 +120,7 @@ curl_share_setopt(struct Curl_share *share, CURLSHoption option, ...) break; case CURL_LOCK_DATA_CONNECT: - if(Curl_conncache_init(&share->conn_cache, 103)) + if(Curl_conncache_init(&share->conn_cache, NULL, 103)) res = CURLSHE_NOMEM; break; diff --git a/lib/url.c b/lib/url.c index d3dcfd38d1..7438be7b81 100644 --- a/lib/url.c +++ b/lib/url.c @@ -556,19 +556,7 @@ CURLcode Curl_open(struct Curl_easy **curl) return result; } -static void conn_shutdown(struct Curl_easy *data) -{ - DEBUGASSERT(data); - infof(data, "Closing connection"); - - /* possible left-overs from the async name resolvers */ - Curl_resolver_cancel(data); - - Curl_conn_close(data, SECONDARYSOCKET); - Curl_conn_close(data, FIRSTSOCKET); -} - -static void conn_free(struct Curl_easy *data, struct connectdata *conn) +void Curl_conn_free(struct Curl_easy *data, struct connectdata *conn) { size_t i; @@ -620,11 +608,9 @@ static void conn_free(struct Curl_easy *data, struct connectdata *conn) * * This function MUST NOT reset state in the Curl_easy struct if that * isn't strictly bound to the life-time of *this* particular connection. - * */ - void Curl_disconnect(struct Curl_easy *data, - struct connectdata *conn, bool dead_connection) + struct connectdata *conn, bool aborted) { /* there must be a connection to close */ DEBUGASSERT(conn); @@ -639,13 +625,14 @@ void Curl_disconnect(struct Curl_easy *data, DEBUGASSERT(!data->conn); DEBUGF(infof(data, "Curl_disconnect(conn #%" - CURL_FORMAT_CURL_OFF_T ", dead=%d)", - conn->connection_id, dead_connection)); + CURL_FORMAT_CURL_OFF_T ", aborted=%d)", + conn->connection_id, aborted)); + /* * If this connection isn't marked to force-close, leave it open if there * are other users of it */ - if(CONN_INUSE(conn) && !dead_connection) { + if(CONN_INUSE(conn) && !aborted) { DEBUGF(infof(data, "Curl_disconnect when inuse: %zu", CONN_INUSE(conn))); return; } @@ -662,23 +649,10 @@ void Curl_disconnect(struct Curl_easy *data, Curl_http_auth_cleanup_negotiate(conn); if(conn->connect_only) - /* treat the connection as dead in CONNECT_ONLY situations */ - dead_connection = TRUE; - - /* temporarily attach the connection to this transfer handle for the - disconnect and shutdown */ - Curl_attach_connection(data, conn); - - if(conn->handler && conn->handler->disconnect) - /* This is set if protocol-specific cleanups should be made */ - conn->handler->disconnect(data, conn, dead_connection); - - conn_shutdown(data); - - /* detach it again */ - Curl_detach_connection(data); + /* treat the connection as aborted in CONNECT_ONLY situations */ + aborted = TRUE; - conn_free(data, conn); + Curl_conncache_disconnect(data, conn, aborted); } /* @@ -824,6 +798,7 @@ static bool prune_if_dead(struct connectdata *conn, * any time (HTTP/2 PING for example), the protocol handler needs * to install its own `connection_check` callback. */ + DEBUGF(infof(data, "connection has input pending, not reusable")); dead = TRUE; } Curl_detach_connection(data); @@ -881,8 +856,8 @@ static void prune_dead_connections(struct Curl_easy *data) /* connection previously removed from cache in prune_if_dead() */ - /* disconnect it */ - Curl_disconnect(data, pruned, TRUE); + /* disconnect it, do not treat as aborted */ + Curl_disconnect(data, pruned, FALSE); } CONNCACHE_LOCK(data); data->state.conn_cache->last_cleanup = now; @@ -1294,8 +1269,8 @@ ConnectionExists(struct Curl_easy *data, infof(data, "Multiplexed connection found"); } else if(prune_if_dead(check, data)) { - /* disconnect it */ - Curl_disconnect(data, check, TRUE); + /* disconnect it, do not treat as aborted */ + Curl_disconnect(data, check, FALSE); continue; } @@ -3333,7 +3308,7 @@ static void reuse_conn(struct Curl_easy *data, /* reuse init */ existing->bits.reuse = TRUE; /* yes, we're reusing here */ - conn_free(data, temp); + Curl_conn_free(data, temp); } /** @@ -3678,7 +3653,7 @@ static CURLcode create_conn(struct Curl_easy *data, if(!connections_available) { infof(data, "No connections available."); - conn_free(data, conn); + Curl_conn_free(data, conn); *in_connect = NULL; result = CURLE_NO_CONNECTION_AVAILABLE; diff --git a/lib/url.h b/lib/url.h index 198a00ad17..55f9b15855 100644 --- a/lib/url.h +++ b/lib/url.h @@ -38,9 +38,10 @@ CURLcode Curl_uc_to_curlcode(CURLUcode uc); CURLcode Curl_close(struct Curl_easy **datap); /* opposite of curl_open() */ CURLcode Curl_connect(struct Curl_easy *, bool *async, bool *protocol_connect); void Curl_disconnect(struct Curl_easy *data, - struct connectdata *, bool dead_connection); + struct connectdata *, bool aborted); CURLcode Curl_setup_conn(struct Curl_easy *data, bool *protocol_done); +void Curl_conn_free(struct Curl_easy *data, struct connectdata *conn); CURLcode Curl_parse_login_details(const char *login, const size_t len, char **userptr, char **passwdptr, char **optionsptr); diff --git a/lib/urldata.h b/lib/urldata.h index e7ce95b41c..bb7a0f6fa2 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -546,6 +546,9 @@ struct ConnectBits { accept() */ BIT(parallel_connect); /* set TRUE when a parallel connect attempt has started (happy eyeballs) */ + BIT(aborted); /* connection was aborted, e.g. in unclean state */ + BIT(shutdown_handler); /* connection shutdown: handler shut down */ + BIT(shutdown_filters); /* connection shutdown: filters shut down */ }; struct hostname { diff --git a/lib/vtls/bearssl.c b/lib/vtls/bearssl.c index f6dfe7269e..97e66c1dd8 100644 --- a/lib/vtls/bearssl.c +++ b/lib/vtls/bearssl.c @@ -1080,7 +1080,7 @@ static CURLcode bearssl_shutdown(struct Curl_cfilter *cf, CURLcode result; DEBUGASSERT(backend); - if(!backend->active || connssl->shutdown) { + if(!backend->active || cf->shutdown) { *done = TRUE; return CURLE_OK; } @@ -1101,7 +1101,7 @@ static CURLcode bearssl_shutdown(struct Curl_cfilter *cf, else CURL_TRC_CF(data, cf, "shutdown error: %d", result); - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -1112,15 +1112,10 @@ static void bearssl_close(struct Curl_cfilter *cf, struct Curl_easy *data) (struct bearssl_ssl_backend_data *)connssl->backend; size_t i; + (void)data; DEBUGASSERT(backend); - if(backend->active) { - if(!connssl->shutdown) { - bool done; - bearssl_shutdown(cf, data, TRUE, &done); - } - backend->active = FALSE; - } + backend->active = FALSE; if(backend->anchors) { for(i = 0; i < backend->anchors_len; ++i) free(backend->anchors[i].dn.data); diff --git a/lib/vtls/gtls.c b/lib/vtls/gtls.c index bf31097edd..2393739d85 100644 --- a/lib/vtls/gtls.c +++ b/lib/vtls/gtls.c @@ -1822,7 +1822,7 @@ static CURLcode gtls_shutdown(struct Curl_cfilter *cf, size_t i; DEBUGASSERT(backend); - if(!backend->gtls.session || connssl->shutdown) { + if(!backend->gtls.session || cf->shutdown) { *done = TRUE; goto out; } @@ -1876,7 +1876,7 @@ static CURLcode gtls_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -1891,10 +1891,6 @@ static void gtls_close(struct Curl_cfilter *cf, DEBUGASSERT(backend); CURL_TRC_CF(data, cf, "close"); if(backend->gtls.session) { - if(!connssl->shutdown) { - bool done; - gtls_shutdown(cf, data, TRUE, &done); - } gnutls_deinit(backend->gtls.session); backend->gtls.session = NULL; } diff --git a/lib/vtls/mbedtls.c b/lib/vtls/mbedtls.c index 3748709d4c..e167c81e59 100644 --- a/lib/vtls/mbedtls.c +++ b/lib/vtls/mbedtls.c @@ -1274,7 +1274,7 @@ static CURLcode mbedtls_shutdown(struct Curl_cfilter *cf, DEBUGASSERT(backend); - if(!backend->initialized || connssl->shutdown) { + if(!backend->initialized || cf->shutdown) { *done = TRUE; return CURLE_OK; } @@ -1346,7 +1346,7 @@ static CURLcode mbedtls_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -1356,13 +1356,9 @@ static void mbedtls_close(struct Curl_cfilter *cf, struct Curl_easy *data) struct mbed_ssl_backend_data *backend = (struct mbed_ssl_backend_data *)connssl->backend; + (void)data; DEBUGASSERT(backend); if(backend->initialized) { - if(!connssl->shutdown) { - bool done; - mbedtls_shutdown(cf, data, TRUE, &done); - } - mbedtls_pk_free(&backend->pk); mbedtls_x509_crt_free(&backend->clicert); mbedtls_x509_crt_free(&backend->cacert); diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c index a7f5f9a032..e96ee73ccc 100644 --- a/lib/vtls/openssl.c +++ b/lib/vtls/openssl.c @@ -1880,9 +1880,10 @@ static CURLcode ossl_shutdown(struct Curl_cfilter *cf, char buf[1024]; int nread, err; unsigned long sslerr; + size_t i; DEBUGASSERT(octx); - if(!octx->ssl || connssl->shutdown) { + if(!octx->ssl || cf->shutdown) { *done = TRUE; goto out; } @@ -1893,14 +1894,19 @@ static CURLcode ossl_shutdown(struct Curl_cfilter *cf, /* We have not started the shutdown from our side yet. Check * if the server already sent us one. */ ERR_clear_error(); - nread = SSL_read(octx->ssl, buf, (int)sizeof(buf)); + for(i = 0; i < 10; ++i) { + nread = SSL_read(octx->ssl, buf, (int)sizeof(buf)); + CURL_TRC_CF(data, cf, "SSL shutdown not sent, read -> %d", nread); + if(nread <= 0) + break; + } err = SSL_get_error(octx->ssl, nread); if(!nread && err == SSL_ERROR_ZERO_RETURN) { bool input_pending; /* Yes, it did. */ if(!send_shutdown) { - connssl->shutdown = TRUE; CURL_TRC_CF(data, cf, "SSL shutdown received, not sending"); + *done = TRUE; goto out; } else if(!cf->next->cft->is_alive(cf->next, data, &input_pending)) { @@ -1908,59 +1914,65 @@ static CURLcode ossl_shutdown(struct Curl_cfilter *cf, * seems not interested to see our close notify, so do not * send it. We are done. */ connssl->peer_closed = TRUE; - connssl->shutdown = TRUE; CURL_TRC_CF(data, cf, "peer closed connection"); + *done = TRUE; goto out; } } + if(send_shutdown && SSL_shutdown(octx->ssl) == 1) { + CURL_TRC_CF(data, cf, "SSL shutdown finished"); + *done = TRUE; + goto out; + } } - if(send_shutdown && SSL_shutdown(octx->ssl) == 1) { - CURL_TRC_CF(data, cf, "SSL shutdown finished"); + /* SSL should now have started the shutdown from our side. Since it + * was not complete, we are lacking the close notify from the server. */ + for(i = 0; i < 10; ++i) { + ERR_clear_error(); + nread = SSL_read(octx->ssl, buf, (int)sizeof(buf)); + CURL_TRC_CF(data, cf, "SSL shutdown read -> %d", nread); + if(nread <= 0) + break; + } + if(SSL_get_shutdown(octx->ssl) & SSL_RECEIVED_SHUTDOWN) { + CURL_TRC_CF(data, cf, "SSL shutdown received, finished"); *done = TRUE; goto out; } - else { - size_t i; - /* SSL should now have started the shutdown from our side. Since it - * was not complete, we are lacking the close notify from the server. */ - for(i = 0; i < 10; ++i) { - ERR_clear_error(); - nread = SSL_read(octx->ssl, buf, (int)sizeof(buf)); - if(nread <= 0) - break; - } - err = SSL_get_error(octx->ssl, nread); - switch(err) { - case SSL_ERROR_ZERO_RETURN: /* no more data */ - CURL_TRC_CF(data, cf, "SSL shutdown received"); - *done = TRUE; - break; - case SSL_ERROR_NONE: /* just did not get anything */ - case SSL_ERROR_WANT_READ: - /* SSL has send its notify and now wants to read the reply - * from the server. We are not really interested in that. */ - CURL_TRC_CF(data, cf, "SSL shutdown sent, want receive"); - connssl->io_need = CURL_SSL_IO_NEED_RECV; - break; - case SSL_ERROR_WANT_WRITE: - CURL_TRC_CF(data, cf, "SSL shutdown send blocked"); - connssl->io_need = CURL_SSL_IO_NEED_SEND; - break; - default: - sslerr = ERR_get_error(); - CURL_TRC_CF(data, cf, "SSL shutdown, error: '%s', errno %d", - (sslerr ? - ossl_strerror(sslerr, buf, sizeof(buf)) : - SSL_ERROR_to_str(err)), - SOCKERRNO); - result = CURLE_RECV_ERROR; - break; - } + err = SSL_get_error(octx->ssl, nread); + switch(err) { + case SSL_ERROR_ZERO_RETURN: /* no more data */ + CURL_TRC_CF(data, cf, "SSL shutdown not received, but closed"); + *done = TRUE; + break; + case SSL_ERROR_NONE: /* just did not get anything */ + case SSL_ERROR_WANT_READ: + /* SSL has send its notify and now wants to read the reply + * from the server. We are not really interested in that. */ + CURL_TRC_CF(data, cf, "SSL shutdown sent, want receive"); + connssl->io_need = CURL_SSL_IO_NEED_RECV; + break; + case SSL_ERROR_WANT_WRITE: + CURL_TRC_CF(data, cf, "SSL shutdown send blocked"); + connssl->io_need = CURL_SSL_IO_NEED_SEND; + break; + default: + /* Server seems to have closed the connection without sending us + * a close notify. */ + sslerr = ERR_get_error(); + CURL_TRC_CF(data, cf, "SSL shutdown, ignore recv error: '%s', errno %d", + (sslerr ? + ossl_strerror(sslerr, buf, sizeof(buf)) : + SSL_ERROR_to_str(err)), + SOCKERRNO); + *done = TRUE; + result = CURLE_OK; + break; } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -1973,14 +1985,6 @@ static void ossl_close(struct Curl_cfilter *cf, struct Curl_easy *data) DEBUGASSERT(octx); if(octx->ssl) { - /* Send the TLS shutdown if have not done so already and are still - * connected *and* if the peer did not already close the connection. */ - if(cf->connected && !connssl->shutdown && - cf->next && cf->next->connected && !connssl->peer_closed) { - bool done; - (void)ossl_shutdown(cf, data, TRUE, &done); - } - SSL_free(octx->ssl); octx->ssl = NULL; } diff --git a/lib/vtls/rustls.c b/lib/vtls/rustls.c index b1fe131b7e..c4d20da77d 100644 --- a/lib/vtls/rustls.c +++ b/lib/vtls/rustls.c @@ -742,7 +742,7 @@ cr_shutdown(struct Curl_cfilter *cf, size_t i; DEBUGASSERT(backend); - if(!backend->conn || connssl->shutdown) { + if(!backend->conn || cf->shutdown) { *done = TRUE; goto out; } @@ -793,7 +793,7 @@ cr_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -804,16 +804,9 @@ cr_close(struct Curl_cfilter *cf, struct Curl_easy *data) struct rustls_ssl_backend_data *backend = (struct rustls_ssl_backend_data *)connssl->backend; + (void)data; DEBUGASSERT(backend); if(backend->conn) { - /* Send the TLS shutdown if have not done so already and are still - * connected *and* if the peer did not already close the connection. */ - if(cf->connected && !connssl->shutdown && - cf->next && cf->next->connected && !connssl->peer_closed) { - bool done; - (void)cr_shutdown(cf, data, TRUE, &done); - } - rustls_connection_free(backend->conn); backend->conn = NULL; } diff --git a/lib/vtls/schannel.c b/lib/vtls/schannel.c index d709479205..28bd0f8005 100644 --- a/lib/vtls/schannel.c +++ b/lib/vtls/schannel.c @@ -2482,7 +2482,7 @@ static CURLcode schannel_shutdown(struct Curl_cfilter *cf, (struct schannel_ssl_backend_data *)connssl->backend; CURLcode result = CURLE_OK; - if(connssl->shutdown) { + if(cf->shutdown) { *done = TRUE; return CURLE_OK; } @@ -2499,7 +2499,7 @@ static CURLcode schannel_shutdown(struct Curl_cfilter *cf, connssl->peer.hostname, connssl->peer.port); } - if(!backend->ctxt || connssl->shutdown) { + if(!backend->ctxt || cf->shutdown) { *done = TRUE; goto out; } @@ -2606,7 +2606,7 @@ static CURLcode schannel_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -2619,13 +2619,6 @@ static void schannel_close(struct Curl_cfilter *cf, struct Curl_easy *data) DEBUGASSERT(data); DEBUGASSERT(backend); - if(backend->cred && backend->ctxt && - cf->connected && !connssl->shutdown && - cf->next && cf->next->connected && !connssl->peer_closed) { - bool done; - (void)schannel_shutdown(cf, data, TRUE, &done); - } - /* free SSPI Schannel API security context handle */ if(backend->ctxt) { DEBUGF(infof(data, "schannel: clear security context handle")); diff --git a/lib/vtls/sectransp.c b/lib/vtls/sectransp.c index e6e4a47b1e..27f663969d 100644 --- a/lib/vtls/sectransp.c +++ b/lib/vtls/sectransp.c @@ -2575,7 +2575,7 @@ static CURLcode sectransp_shutdown(struct Curl_cfilter *cf, size_t i; DEBUGASSERT(backend); - if(!backend->ssl_ctx || connssl->shutdown) { + if(!backend->ssl_ctx || cf->shutdown) { *done = TRUE; goto out; } @@ -2638,7 +2638,7 @@ static CURLcode sectransp_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -2654,12 +2654,6 @@ static void sectransp_close(struct Curl_cfilter *cf, struct Curl_easy *data) if(backend->ssl_ctx) { CURL_TRC_CF(data, cf, "close"); - if(cf->connected && !connssl->shutdown && - cf->next && cf->next->connected && !connssl->peer_closed) { - bool done; - (void)sectransp_shutdown(cf, data, TRUE, &done); - } - #if CURL_BUILD_MAC_10_8 || CURL_BUILD_IOS if(SSLCreateContext) CFRelease(backend->ssl_ctx); diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c index 20a5fbc9db..7cada8b90a 100644 --- a/lib/vtls/vtls.c +++ b/lib/vtls/vtls.c @@ -1757,17 +1757,17 @@ static CURLcode ssl_cf_shutdown(struct Curl_cfilter *cf, struct Curl_easy *data, bool *done) { - struct ssl_connect_data *connssl = cf->ctx; - struct cf_call_data save; CURLcode result = CURLE_OK; *done = TRUE; - if(!connssl->shutdown) { + if(!cf->shutdown) { + struct cf_call_data save; + CF_DATA_SAVE(save, cf, data); result = Curl_ssl->shut_down(cf, data, TRUE, done); CURL_TRC_CF(data, cf, "cf_shutdown -> %d, done=%d", result, *done); CF_DATA_RESTORE(cf, save); - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); } return result; } @@ -2052,7 +2052,7 @@ static CURLcode vtls_shutdown_blocking(struct Curl_cfilter *cf, timediff_t timeout_ms; int what, loop = 10; - if(connssl->shutdown) { + if(cf->shutdown) { *done = TRUE; return CURLE_OK; } @@ -2091,7 +2091,7 @@ static CURLcode vtls_shutdown_blocking(struct Curl_cfilter *cf, } out: CF_DATA_RESTORE(cf, save); - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } diff --git a/lib/vtls/vtls_int.h b/lib/vtls/vtls_int.h index 310999c8f1..ea41abdda7 100644 --- a/lib/vtls/vtls_int.h +++ b/lib/vtls/vtls_int.h @@ -94,7 +94,6 @@ struct ssl_connect_data { int io_need; /* TLS signals special SEND/RECV needs */ BIT(use_alpn); /* if ALPN shall be used in handshake */ BIT(peer_closed); /* peer has closed connection */ - BIT(shutdown); /* graceful close notify finished */ }; diff --git a/lib/vtls/wolfssl.c b/lib/vtls/wolfssl.c index 94a009e5b2..a0b598d0a7 100644 --- a/lib/vtls/wolfssl.c +++ b/lib/vtls/wolfssl.c @@ -1357,7 +1357,7 @@ static CURLcode wolfssl_shutdown(struct Curl_cfilter *cf, int nread, err; DEBUGASSERT(wctx); - if(!wctx->handle || connssl->shutdown) { + if(!wctx->handle || cf->shutdown) { *done = TRUE; goto out; } @@ -1374,17 +1374,17 @@ static CURLcode wolfssl_shutdown(struct Curl_cfilter *cf, bool input_pending; /* Yes, it did. */ if(!send_shutdown) { - connssl->shutdown = TRUE; CURL_TRC_CF(data, cf, "SSL shutdown received, not sending"); + *done = TRUE; goto out; } else if(!cf->next->cft->is_alive(cf->next, data, &input_pending)) { /* Server closed the connection after its closy notify. It * seems not interested to see our close notify, so do not * send it. We are done. */ - connssl->peer_closed = TRUE; - connssl->shutdown = TRUE; CURL_TRC_CF(data, cf, "peer closed connection"); + connssl->peer_closed = TRUE; + *done = TRUE; goto out; } } @@ -1435,7 +1435,7 @@ static CURLcode wolfssl_shutdown(struct Curl_cfilter *cf, } out: - connssl->shutdown = (result || *done); + cf->shutdown = (result || *done); return result; } @@ -1450,11 +1450,6 @@ static void wolfssl_close(struct Curl_cfilter *cf, struct Curl_easy *data) DEBUGASSERT(backend); if(backend->handle) { - if(cf->connected && !connssl->shutdown && - cf->next && cf->next->connected && !connssl->peer_closed) { - bool done; - (void)wolfssl_shutdown(cf, data, TRUE, &done); - } wolfSSL_free(backend->handle); backend->handle = NULL; } diff --git a/src/tool_operate.c b/src/tool_operate.c index 673eeed603..342c469031 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -2579,6 +2579,9 @@ static CURLcode serial_transfers(struct GlobalConfig *global, } start = tvnow(); #ifdef DEBUGBUILD + if(getenv("CURL_FORBID_REUSE")) + (void)curl_easy_setopt(per->curl, CURLOPT_FORBID_REUSE, 1L); + if(global->test_event_based) result = curl_easy_perform_ev(per->curl); else diff --git a/tests/conftest.py b/tests/conftest.py index f1e066256f..691d6b6188 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def pytest_report_header(config): ]) if env.has_vsftpd(): report.extend([ - f' VsFTPD: {env.vsftpd_version()}, ftp:{env.ftp_port}' + f' VsFTPD: {env.vsftpd_version()}, ftp:{env.ftp_port}, ftps:{env.ftps_port}' ]) return '\n'.join(report) diff --git a/tests/data/test1542 b/tests/data/test1542 index f9806cda81..0454d76dde 100644 --- a/tests/data/test1542 +++ b/tests/data/test1542 @@ -58,11 +58,11 @@ Accept: */* == Info: Connection #0 to host %HOSTIP left intact == Info: Connection #0 to host %HOSTIP left intact == Info: Connection #0 to host %HOSTIP left intact -== Info: Closing connection +== Info: shutting down connection #0 == Info: Connection #1 to host %HOSTIP left intact -$_ = '' if (($_ !~ /left intact/) && ($_ !~ /Closing connection/)) +$_ = '' if (($_ !~ /left intact/) && ($_ !~ /(closing|shutting down) connection #\d+/)) diff --git a/tests/http/clients/h2-download.c b/tests/http/clients/h2-download.c index 74aac6a8ee..7198398891 100644 --- a/tests/http/clients/h2-download.c +++ b/tests/http/clients/h2-download.c @@ -159,6 +159,7 @@ struct transfer { static size_t transfer_count = 1; static struct transfer *transfers; +static int forbid_reuse = 0; static struct transfer *get_transfer_for_easy(CURL *easy) { @@ -239,6 +240,8 @@ static int setup(CURL *hnd, const char *url, struct transfer *t, curl_easy_setopt(hnd, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(hnd, CURLOPT_XFERINFOFUNCTION, my_progress_cb); curl_easy_setopt(hnd, CURLOPT_XFERINFODATA, t); + if(forbid_reuse) + curl_easy_setopt(hnd, CURLOPT_FORBID_REUSE, 1L); /* please be verbose */ if(verbose) { @@ -288,7 +291,7 @@ int main(int argc, char *argv[]) int http_version = CURL_HTTP_VERSION_2_0; int ch; - while((ch = getopt(argc, argv, "ahm:n:A:F:P:V:")) != -1) { + while((ch = getopt(argc, argv, "afhm:n:A:F:P:V:")) != -1) { switch(ch) { case 'h': usage(NULL); @@ -296,6 +299,9 @@ int main(int argc, char *argv[]) case 'a': abort_paused = 1; break; + case 'f': + forbid_reuse = 1; + break; case 'm': max_parallel = (size_t)strtol(optarg, NULL, 10); break; diff --git a/tests/http/test_19_shutdown.py b/tests/http/test_19_shutdown.py new file mode 100644 index 0000000000..de23fa5a26 --- /dev/null +++ b/tests/http/test_19_shutdown.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +#*************************************************************************** +# _ _ ____ _ +# Project ___| | | | _ \| | +# / __| | | | |_) | | +# | (__| |_| | _ <| |___ +# \___|\___/|_| \_\_____| +# +# Copyright (C) Daniel Stenberg, , et al. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at https://curl.se/docs/copyright.html. +# +# You may opt to use, copy, modify, merge, publish, distribute and/or sell +# copies of the Software, and permit persons to whom the Software is +# furnished to do so, under the terms of the COPYING file. +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +# KIND, either express or implied. +# +# SPDX-License-Identifier: curl +# +########################################################################### +# +import difflib +import filecmp +import logging +import os +import re +from datetime import timedelta +import pytest + +from testenv import Env, CurlClient, LocalClient + + +log = logging.getLogger(__name__) + + +class TestShutdown: + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env, httpd, nghttpx): + if env.have_h3(): + nghttpx.start_if_needed() + httpd.clear_extra_configs() + httpd.reload() + + @pytest.fixture(autouse=True, scope='class') + def _class_scope(self, env, httpd): + indir = httpd.docs_dir + env.make_data_file(indir=indir, fname="data-10k", fsize=10*1024) + env.make_data_file(indir=indir, fname="data-100k", fsize=100*1024) + env.make_data_file(indir=indir, fname="data-1m", fsize=1024*1024) + + # check with `tcpdump` that we see curl TCP RST packets + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + @pytest.mark.parametrize("proto", ['http/1.1']) + def test_19_01_check_tcp_rst(self, env: Env, httpd, repeat, proto): + if env.ci_run: + pytest.skip("seems not to work in CI") + curl = CurlClient(env=env) + url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]' + r = curl.http_download(urls=[url], alpn_proto=proto, with_tcpdump=True, extra_args=[ + '--parallel' + ]) + r.check_response(http_status=200, count=2) + assert r.tcpdump + assert len(r.tcpdump.stats) != 0, f'Expected TCP RSTs packets: {r.tcpdump.stderr}' + + # check with `tcpdump` that we do NOT see TCP RST when CURL_GRACEFUL_SHUTDOWN set + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) + def test_19_02_check_shutdown(self, env: Env, httpd, repeat, proto): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') + curl = CurlClient(env=env, run_env={ + 'CURL_GRACEFUL_SHUTDOWN': '2000', + 'CURL_DEBUG': 'ssl' + }) + url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]' + r = curl.http_download(urls=[url], alpn_proto=proto, with_tcpdump=True, extra_args=[ + '--parallel' + ]) + r.check_response(http_status=200, count=2) + assert r.tcpdump + assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets' + + # run downloads where the server closes the connection after each request + @pytest.mark.parametrize("proto", ['http/1.1']) + def test_19_03_shutdown_by_server(self, env: Env, httpd, repeat, proto): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') + count = 10 + curl = CurlClient(env=env, run_env={ + 'CURL_GRACEFUL_SHUTDOWN': '2000', + 'CURL_DEBUG': 'ssl' + }) + url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak/?'\ + f'id=[0-{count-1}]&with_cl&close' + r = curl.http_download(urls=[url], alpn_proto=proto) + r.check_response(http_status=200, count=count) + shutdowns = [l for l in r.trace_lines if re.match(r'.*CCACHE\] shutdown #\d+, done=1', l)] + assert len(shutdowns) == count, f'{shutdowns}' + + # run downloads with CURLOPT_FORBID_REUSE set, meaning *we* close + # the connection after each request + @pytest.mark.parametrize("proto", ['http/1.1']) + def test_19_04_shutdown_by_curl(self, env: Env, httpd, proto, repeat): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') + count = 10 + docname = 'data.json' + url = f'https://localhost:{env.https_port}/{docname}' + client = LocalClient(name='h2-download', env=env, run_env={ + 'CURL_GRACEFUL_SHUTDOWN': '2000', + 'CURL_DEBUG': 'ssl' + }) + if not client.exists(): + pytest.skip(f'example client not built: {client.name}') + r = client.run(args=[ + '-n', f'{count}', '-f', '-V', proto, url + ]) + r.check_exit_code(0) + shutdowns = [l for l in r.trace_lines if re.match(r'.*CCACHE\] shutdown #\d+, done=1', l)] + assert len(shutdowns) == count, f'{shutdowns}' + + # run event-based downloads with CURLOPT_FORBID_REUSE set, meaning *we* close + # the connection after each request + @pytest.mark.parametrize("proto", ['http/1.1']) + def test_19_05_event_shutdown_by_server(self, env: Env, httpd, proto, repeat): + if not env.curl_is_debug(): + pytest.skip('only works for curl debug builds') + count = 10 + curl = CurlClient(env=env, run_env={ + # forbid connection reuse to trigger shutdowns after transfer + 'CURL_FORBID_REUSE': '1', + # make socket receives block 50% of the time to delay shutdown + 'CURL_DBG_SOCK_RBLOCK': '50', + 'CURL_DEBUG': 'ssl' + }) + url = f'https://{env.authority_for(env.domain1, proto)}/curltest/tweak/?'\ + f'id=[0-{count-1}]&with_cl&' + r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[ + '--test-event' + ]) + r.check_response(http_status=200, count=count) + # check that we closed all connections + closings = [l for l in r.trace_lines if re.match(r'.*CCACHE\] closing #\d+', l)] + assert len(closings) == count, f'{closings}' + # check that all connection sockets were removed from event + removes = [l for l in r.trace_lines if re.match(r'.*socket cb: socket \d+ REMOVED', l)] + assert len(removes) == count, f'{removes}' + + diff --git a/tests/http/test_30_vsftpd.py b/tests/http/test_30_vsftpd.py index 11b8902792..477b27affd 100644 --- a/tests/http/test_30_vsftpd.py +++ b/tests/http/test_30_vsftpd.py @@ -136,6 +136,33 @@ class TestVsFTPD: if os.path.exists(path): return os.remove(path) + # check with `tcpdump` if curl causes any TCP RST packets + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + def test_30_06_shutdownh_download(self, env: Env, vsftpd: VsFTPD, repeat): + docname = 'data-1k' + curl = CurlClient(env=env) + count = 1 + url = f'ftp://{env.ftp_domain}:{vsftpd.port}/{docname}?[0-{count-1}]' + r = curl.ftp_get(urls=[url], with_stats=True, with_tcpdump=True) + r.check_stats(count=count, http_status=226) + assert r.tcpdump + assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets' + + # check with `tcpdump` if curl causes any TCP RST packets + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + def test_30_07_shutdownh_upload(self, env: Env, vsftpd: VsFTPD, repeat): + docname = 'upload-1k' + curl = CurlClient(env=env) + srcfile = os.path.join(env.gen_dir, docname) + dstfile = os.path.join(vsftpd.docs_dir, docname) + self._rmf(dstfile) + count = 1 + url = f'ftp://{env.ftp_domain}:{vsftpd.port}/' + r = curl.ftp_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True) + r.check_stats(count=count, http_status=226) + assert r.tcpdump + assert len(r.tcpdump.stats) == 0, f'Unexpected TCP RSTs packets' + def check_downloads(self, client, srcfile: str, count: int, complete: bool = True): for i in range(count): diff --git a/tests/http/test_31_vsftpds.py b/tests/http/test_31_vsftpds.py index 2283812473..2434677c0c 100644 --- a/tests/http/test_31_vsftpds.py +++ b/tests/http/test_31_vsftpds.py @@ -143,6 +143,35 @@ class TestVsFTPD: if os.path.exists(path): return os.remove(path) + # check with `tcpdump` if curl causes any TCP RST packets + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + def test_31_06_shutdownh_download(self, env: Env, vsftpds: VsFTPD, repeat): + docname = 'data-1k' + curl = CurlClient(env=env) + count = 1 + url = f'ftp://{env.ftp_domain}:{vsftpds.port}/{docname}?[0-{count-1}]' + r = curl.ftp_ssl_get(urls=[url], with_stats=True, with_tcpdump=True) + r.check_stats(count=count, http_status=226) + # vsftp closes control connection without niceties, + # disregard RST packets it sent from its port to curl + assert len(r.tcpdump.stats_excluding(src_port=env.ftps_port)) == 0, f'Unexpected TCP RSTs packets' + + # check with `tcpdump` if curl causes any TCP RST packets + @pytest.mark.skipif(condition=not Env.tcpdump(), reason="tcpdump not available") + def test_31_07_shutdownh_upload(self, env: Env, vsftpds: VsFTPD, repeat): + docname = 'upload-1k' + curl = CurlClient(env=env) + srcfile = os.path.join(env.gen_dir, docname) + dstfile = os.path.join(vsftpds.docs_dir, docname) + self._rmf(dstfile) + count = 1 + url = f'ftp://{env.ftp_domain}:{vsftpds.port}/' + r = curl.ftp_ssl_upload(urls=[url], fupload=f'{srcfile}', with_stats=True, with_tcpdump=True) + r.check_stats(count=count, http_status=226) + # vsftp closes control connection without niceties, + # disregard RST packets it sent from its port to curl + assert len(r.tcpdump.stats_excluding(src_port=env.ftps_port)) == 0, f'Unexpected TCP RSTs packets' + def check_downloads(self, client, srcfile: str, count: int, complete: bool = True): for i in range(count): diff --git a/tests/http/testenv/curl.py b/tests/http/testenv/curl.py index f407864643..f89b2c9a8e 100644 --- a/tests/http/testenv/curl.py +++ b/tests/http/testenv/curl.py @@ -27,6 +27,10 @@ import json import logging import os +import sys +import time +from threading import Thread + import psutil import re import shutil @@ -103,6 +107,85 @@ class RunProfile: f'stats={self.stats}]' +class RunTcpDump: + + def __init__(self, env, run_dir): + self._env = env + self._run_dir = run_dir + self._proc = None + self._stdoutfile = os.path.join(self._run_dir, 'tcpdump.out') + self._stderrfile = os.path.join(self._run_dir, 'tcpdump.err') + + @property + def stats(self) -> Optional[List[str]]: + if self._proc: + raise Exception('tcpdump still running') + lines = [] + for l in open(self._stdoutfile).readlines(): + if re.match(r'.* IP 127\.0\.0\.1\.\d+ [<>] 127\.0\.0\.1\.\d+:.*', l): + lines.append(l) + return lines + + def stats_excluding(self, src_port) -> Optional[List[str]]: + if self._proc: + raise Exception('tcpdump still running') + lines = [] + for l in self.stats: + if not re.match(r'.* IP 127\.0\.0\.1\.' + str(src_port) + ' >.*', l): + lines.append(l) + return lines + + @property + def stderr(self) -> List[str]: + if self._proc: + raise Exception('tcpdump still running') + lines = [] + return open(self._stderrfile).readlines() + + def sample(self): + # not sure how to make that detection reliable for all platforms + local_if = 'lo0' if sys.platform.startswith('darwin') else 'lo' + try: + tcpdump = self._env.tcpdump() + if tcpdump is None: + raise Exception('tcpdump not available') + # look with tcpdump for TCP RST packets which indicate + # we did not shut down connections cleanly + args = [] + # at least on Linux, we need root permissions to run tcpdump + if sys.platform.startswith('linux'): + args.append('sudo') + args.extend([ + tcpdump, '-i', local_if, '-n', 'tcp[tcpflags] & (tcp-rst)!=0' + ]) + with open(self._stdoutfile, 'w') as cout: + with open(self._stderrfile, 'w') as cerr: + self._proc = subprocess.Popen(args, stdout=cout, stderr=cerr, + text=True, cwd=self._run_dir, + shell=False) + assert self._proc + assert self._proc.returncode is None + while self._proc: + try: + self._proc.wait(timeout=1) + except subprocess.TimeoutExpired: + pass + except Exception as e: + log.error(f'Tcpdump: {e}') + + def start(self): + def do_sample(): + self.sample() + t = Thread(target=do_sample) + t.start() + + def finish(self): + if self._proc: + time.sleep(1) + self._proc.terminate() + self._proc = None + + class ExecResult: def __init__(self, args: List[str], exit_code: int, @@ -110,13 +193,15 @@ class ExecResult: duration: Optional[timedelta] = None, with_stats: bool = False, exception: Optional[str] = None, - profile: Optional[RunProfile] = None): + profile: Optional[RunProfile] = None, + tcpdump: Optional[RunTcpDump] = None): self._args = args self._exit_code = exit_code self._exception = exception self._stdout = stdout self._stderr = stderr self._profile = profile + self._tcpdump = tcpdump self._duration = duration if duration is not None else timedelta() self._response = None self._responses = [] @@ -185,6 +270,10 @@ class ExecResult: def profile(self) -> Optional[RunProfile]: return self._profile + @property + def tcpdump(self) -> Optional[RunTcpDump]: + return self._tcpdump + @property def response(self) -> Optional[Dict]: return self._response @@ -359,8 +448,11 @@ class CurlClient: 'h3': '--http3-only', } - def __init__(self, env: Env, run_dir: Optional[str] = None, - timeout: Optional[float] = None, silent: bool = False): + def __init__(self, env: Env, + run_dir: Optional[str] = None, + timeout: Optional[float] = None, + silent: bool = False, + run_env: Optional[Dict[str, str]] = None): self.env = env self._timeout = timeout if timeout else env.test_timeout self._curl = os.environ['CURL'] if 'CURL' in os.environ else env.curl @@ -370,6 +462,7 @@ class CurlClient: self._headerfile = f'{self._run_dir}/curl.headers' self._log_path = f'{self._run_dir}/curl.log' self._silent = silent + self._run_env = run_env self._rmrf(self._run_dir) self._mkpath(self._run_dir) @@ -418,18 +511,21 @@ class CurlClient: alpn_proto: Optional[str] = None, def_tracing: bool = True, with_stats: bool = False, - with_profile: bool = False): + with_profile: bool = False, + with_tcpdump: bool = False): return self._raw(url, options=extra_args, with_stats=with_stats, alpn_proto=alpn_proto, def_tracing=def_tracing, - with_profile=with_profile) + with_profile=with_profile, + with_tcpdump=with_tcpdump) def http_download(self, urls: List[str], alpn_proto: Optional[str] = None, with_stats: bool = True, with_headers: bool = False, with_profile: bool = False, + with_tcpdump: bool = False, no_save: bool = False, extra_args: List[str] = None): if extra_args is None: @@ -452,13 +548,15 @@ class CurlClient: return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, with_stats=with_stats, with_headers=with_headers, - with_profile=with_profile) + with_profile=with_profile, + with_tcpdump=with_tcpdump) def http_upload(self, urls: List[str], data: str, alpn_proto: Optional[str] = None, with_stats: bool = True, with_headers: bool = False, with_profile: bool = False, + with_tcpdump: bool = False, extra_args: Optional[List[str]] = None): if extra_args is None: extra_args = [] @@ -472,7 +570,8 @@ class CurlClient: return self._raw(urls, alpn_proto=alpn_proto, options=extra_args, with_stats=with_stats, with_headers=with_headers, - with_profile=with_profile) + with_profile=with_profile, + with_tcpdump=with_tcpdump) def http_delete(self, urls: List[str], alpn_proto: Optional[str] = None, @@ -541,6 +640,7 @@ class CurlClient: def ftp_get(self, urls: List[str], with_stats: bool = True, with_profile: bool = False, + with_tcpdump: bool = False, no_save: bool = False, extra_args: List[str] = None): if extra_args is None: @@ -563,11 +663,13 @@ class CurlClient: return self._raw(urls, options=extra_args, with_stats=with_stats, with_headers=False, - with_profile=with_profile) + with_profile=with_profile, + with_tcpdump=with_tcpdump) def ftp_ssl_get(self, urls: List[str], with_stats: bool = True, with_profile: bool = False, + with_tcpdump: bool = False, no_save: bool = False, extra_args: List[str] = None): if extra_args is None: @@ -577,11 +679,13 @@ class CurlClient: ]) return self.ftp_get(urls=urls, with_stats=with_stats, with_profile=with_profile, no_save=no_save, + with_tcpdump=with_tcpdump, extra_args=extra_args) def ftp_upload(self, urls: List[str], fupload, with_stats: bool = True, with_profile: bool = False, + with_tcpdump: bool = False, extra_args: List[str] = None): if extra_args is None: extra_args = [] @@ -595,11 +699,13 @@ class CurlClient: return self._raw(urls, options=extra_args, with_stats=with_stats, with_headers=False, - with_profile=with_profile) + with_profile=with_profile, + with_tcpdump=with_tcpdump) def ftp_ssl_upload(self, urls: List[str], fupload, with_stats: bool = True, with_profile: bool = False, + with_tcpdump: bool = False, extra_args: List[str] = None): if extra_args is None: extra_args = [] @@ -608,6 +714,7 @@ class CurlClient: ]) return self.ftp_upload(urls=urls, fupload=fupload, with_stats=with_stats, with_profile=with_profile, + with_tcpdump=with_tcpdump, extra_args=extra_args) def response_file(self, idx: int): @@ -625,14 +732,18 @@ class CurlClient: my_args.extend(args) return self._run(args=my_args, with_stats=with_stats, with_profile=with_profile) - def _run(self, args, intext='', with_stats: bool = False, with_profile: bool = True): + def _run(self, args, intext='', with_stats: bool = False, + with_profile: bool = True, with_tcpdump: bool = False): self._rmf(self._stdoutfile) self._rmf(self._stderrfile) self._rmf(self._headerfile) - started_at = datetime.now() exception = None profile = None + tcpdump = None started_at = datetime.now() + if with_tcpdump: + tcpdump = RunTcpDump(self.env, self._run_dir) + tcpdump.start() try: with open(self._stdoutfile, 'w') as cout: with open(self._stderrfile, 'w') as cerr: @@ -641,7 +752,8 @@ class CurlClient: if self._timeout else None log.info(f'starting: {args}') p = subprocess.Popen(args, stderr=cerr, stdout=cout, - cwd=self._run_dir, shell=False) + cwd=self._run_dir, shell=False, + env=self._run_env) profile = RunProfile(p.pid, started_at, self._run_dir) if intext is not None and False: p.communicate(input=intext.encode(), timeout=1) @@ -663,7 +775,8 @@ class CurlClient: p = subprocess.run(args, stderr=cerr, stdout=cout, cwd=self._run_dir, shell=False, input=intext.encode() if intext else None, - timeout=self._timeout) + timeout=self._timeout, + env=self._run_env) exitcode = p.returncode except subprocess.TimeoutExpired: now = datetime.now() @@ -672,13 +785,15 @@ class CurlClient: f'(configured {self._timeout}s): {args}') exitcode = -1 exception = 'TimeoutExpired' + if tcpdump: + tcpdump.finish() coutput = open(self._stdoutfile).readlines() cerrput = open(self._stderrfile).readlines() return ExecResult(args=args, exit_code=exitcode, exception=exception, stdout=coutput, stderr=cerrput, duration=datetime.now() - started_at, with_stats=with_stats, - profile=profile) + profile=profile, tcpdump=tcpdump) def _raw(self, urls, intext='', timeout=None, options=None, insecure=False, alpn_proto: Optional[str] = None, @@ -686,13 +801,14 @@ class CurlClient: with_stats=False, with_headers=True, def_tracing=True, - with_profile=False): + with_profile=False, + with_tcpdump=False): args = self._complete_args( urls=urls, timeout=timeout, options=options, insecure=insecure, alpn_proto=alpn_proto, force_resolve=force_resolve, with_headers=with_headers, def_tracing=def_tracing) r = self._run(args, intext=intext, with_stats=with_stats, - with_profile=with_profile) + with_profile=with_profile, with_tcpdump=with_tcpdump) if r.exit_code == 0 and with_headers: self._parse_headerfile(self._headerfile, r=r) if r.json: diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py index 067fe4f3e6..2cd6432364 100644 --- a/tests/http/testenv/env.py +++ b/tests/http/testenv/env.py @@ -27,6 +27,7 @@ import logging import os import re +import shutil import socket import subprocess import sys @@ -203,6 +204,8 @@ class EnvConfig: except Exception as e: self.vsftpd = None + self._tcpdump = shutil.which('tcpdump') + @property def httpd_version(self): if self._httpd_version is None and self.apxs is not None: @@ -264,6 +267,10 @@ class EnvConfig: def vsftpd_version(self): return self._vsftpd_version + @property + def tcpdmp(self) -> Optional[str]: + return self._tcpdump + class Env: @@ -383,6 +390,10 @@ class Env: def vsftpd_version() -> str: return Env.CONFIG.vsftpd_version + @staticmethod + def tcpdump() -> Optional[str]: + return Env.CONFIG.tcpdmp + def __init__(self, pytestconfig=None): self._verbose = pytestconfig.option.verbose \ if pytestconfig is not None else 0 diff --git a/tests/http/testenv/mod_curltest/mod_curltest.c b/tests/http/testenv/mod_curltest/mod_curltest.c index 09f12256a5..1c495a2c03 100644 --- a/tests/http/testenv/mod_curltest/mod_curltest.c +++ b/tests/http/testenv/mod_curltest/mod_curltest.c @@ -324,10 +324,11 @@ static int curltest_tweak_handler(request_rec *r) int i, chunks = 3, error_bucket = 1; size_t chunk_size = sizeof(buffer); const char *request_id = "none"; - apr_time_t delay = 0, chunk_delay = 0; + apr_time_t delay = 0, chunk_delay = 0, close_delay = 0; apr_array_header_t *args = NULL; int http_status = 200; apr_status_t error = APR_SUCCESS, body_error = APR_SUCCESS; + int close_conn = 0, with_cl = 0; if(strcmp(r->handler, "curltest-tweak")) { return DECLINED; @@ -405,6 +406,21 @@ static int curltest_tweak_handler(request_rec *r) continue; } } + else if(!strcmp("close_delay", arg)) { + rv = duration_parse(&close_delay, val, "s"); + if(APR_SUCCESS == rv) { + continue; + } + } + } + else if(!strcmp("close", arg)) { + /* we are asked to close the connection */ + close_conn = 1; + continue; + } + else if(!strcmp("with_cl", arg)) { + with_cl = 1; + continue; } ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "query parameter not " "understood: '%s' in %s", @@ -417,10 +433,15 @@ static int curltest_tweak_handler(request_rec *r) ap_log_rerror(APLOG_MARK, APLOG_TRACE1, 0, r, "error_handler: processing " "request, %s", r->args? r->args : "(no args)"); r->status = http_status; - r->clength = -1; - r->chunked = (r->proto_num >= HTTP_VERSION(1,1)); + r->clength = with_cl? (chunks * chunk_size) : -1; + r->chunked = (r->proto_num >= HTTP_VERSION(1,1)) && !with_cl; apr_table_setn(r->headers_out, "request-id", request_id); - apr_table_unset(r->headers_out, "Content-Length"); + if(r->clength >= 0) { + apr_table_set(r->headers_out, "Content-Length", + apr_ltoa(r->pool, (long)r->clength)); + } + else + apr_table_unset(r->headers_out, "Content-Length"); /* Discourage content-encodings */ apr_table_unset(r->headers_out, "Content-Encoding"); apr_table_setn(r->subprocess_env, "no-brotli", "1"); @@ -467,9 +488,19 @@ static int curltest_tweak_handler(request_rec *r) "error_handler: response passed"); cleanup: + if(close_conn) { + if(close_delay) { + b = apr_bucket_flush_create(c->bucket_alloc); + APR_BRIGADE_INSERT_TAIL(bb, b); + rv = ap_pass_brigade(r->output_filters, bb); + apr_brigade_cleanup(bb); + apr_sleep(close_delay); + } + r->connection->keepalive = AP_CONN_CLOSE; + } ap_log_rerror(APLOG_MARK, APLOG_TRACE1, rv, r, - "error_handler: request cleanup, r->status=%d, aborted=%d", - r->status, c->aborted); + "error_handler: request cleanup, r->status=%d, aborted=%d, " + "close=%d", r->status, c->aborted, close_conn); if(rv == APR_SUCCESS) { return OK; } -- 2.47.3