From: Stefan Eissing Date: Mon, 12 Aug 2024 09:28:19 +0000 (+0200) Subject: urldata: introduce `data->mid`, a unique identifier inside a multi X-Git-Tag: curl-8_10_0~271 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=22d292b3eca1dcbdc590f07e7fe07d1132312a1c;p=thirdparty%2Fcurl.git urldata: introduce `data->mid`, a unique identifier inside a multi `data->id` is unique in *most* situations, but not in all. If a libcurl application uses more than one connection cache, they will overlap. This is a rare situations, but libcurl apps do crazy things. However, for informative things, like tracing, `data->id` is superior, since it assigns new ids in curl's serial curl_easy_perform() use. Introduce `data->mid` which is a unique identifer inside one multi instance, assigned on multi_add_handle() and cleared on multi_remove_handle(). Use the `mid` in DoH operations and also in h2/h3 stream hashes. Reported-by: 罗朝辉 Fixes #14414 Closes #14499 --- diff --git a/lib/doh.c b/lib/doh.c index ecb8cbe646..9adb885688 100644 --- a/lib/doh.c +++ b/lib/doh.c @@ -214,19 +214,28 @@ static void local_print_buf(struct Curl_easy *data, /* called from multi.c when this DoH transfer is complete */ static int doh_done(struct Curl_easy *doh, CURLcode result) { - struct Curl_easy *data = doh->set.dohfor; - struct dohdata *dohp = data->req.doh; - /* so one of the DoH request done for the 'data' transfer is now complete! */ - dohp->pending--; - infof(doh, "a DoH request is completed, %u to go", dohp->pending); - if(result) - infof(doh, "DoH request %s", curl_easy_strerror(result)); + struct Curl_easy *data; - if(!dohp->pending) { - /* DoH completed */ - curl_slist_free_all(dohp->headers); - dohp->headers = NULL; - Curl_expire(data, 0, EXPIRE_RUN_NOW); + data = Curl_multi_get_handle(doh->multi, doh->set.dohfor_mid); + if(!data) { + DEBUGF(infof(doh, "doh_done: xfer for mid=%" CURL_FORMAT_CURL_OFF_T + " not found", doh->set.dohfor_mid)); + DEBUGASSERT(0); + } + else { + struct dohdata *dohp = data->req.doh; + /* one of the DoH request done for the 'data' transfer is now complete! */ + dohp->pending--; + infof(doh, "a DoH request is completed, %u to go", dohp->pending); + if(result) + infof(doh, "DoH request %s", curl_easy_strerror(result)); + + if(!dohp->pending) { + /* DoH completed */ + curl_slist_free_all(dohp->headers); + dohp->headers = NULL; + Curl_expire(data, 0, EXPIRE_RUN_NOW); + } } return 0; } @@ -368,8 +377,7 @@ static CURLcode dohprobe(struct Curl_easy *data, } doh->set.fmultidone = doh_done; - doh->set.dohfor = data; /* identify for which transfer this is done */ - p->easy = doh; + doh->set.dohfor_mid = data->mid; /* for which transfer this is done */ /* DoH handles must not inherit private_data. The handles may be passed to the user via callbacks and the user will be able to identify them as @@ -379,6 +387,8 @@ static CURLcode dohprobe(struct Curl_easy *data, if(curl_multi_add_handle(multi, doh)) goto error; + + p->easy_mid = doh->mid; } else goto error; @@ -402,6 +412,7 @@ struct Curl_addrinfo *Curl_doh(struct Curl_easy *data, CURLcode result = CURLE_OK; struct dohdata *dohp; struct connectdata *conn = data->conn; + size_t i; #ifdef USE_HTTPSRR /* for now, this is only used when ECH is enabled */ # ifdef USE_ECH @@ -420,6 +431,10 @@ struct Curl_addrinfo *Curl_doh(struct Curl_easy *data, if(!dohp) return NULL; + for(i = 0; i < DOH_PROBE_SLOTS; ++i) { + dohp->probe[i].easy_mid = -1; + } + conn->bits.doh = TRUE; dohp->host = hostname; dohp->port = port; @@ -1299,8 +1314,8 @@ CURLcode Curl_doh_is_resolved(struct Curl_easy *data, if(!dohp) return CURLE_OUT_OF_MEMORY; - if(!dohp->probe[DOH_PROBE_SLOT_IPADDR_V4].easy && - !dohp->probe[DOH_PROBE_SLOT_IPADDR_V6].easy) { + if(dohp->probe[DOH_PROBE_SLOT_IPADDR_V4].easy_mid < 0 && + dohp->probe[DOH_PROBE_SLOT_IPADDR_V6].easy_mid < 0) { failf(data, "Could not DoH-resolve: %s", data->state.async.hostname); return CONN_IS_PROXIED(data->conn)?CURLE_COULDNT_RESOLVE_PROXY: CURLE_COULDNT_RESOLVE_HOST; @@ -1408,16 +1423,27 @@ CURLcode Curl_doh_is_resolved(struct Curl_easy *data, void Curl_doh_close(struct Curl_easy *data) { struct dohdata *doh = data->req.doh; - if(doh) { + if(doh && data->multi) { + struct Curl_easy *probe_data; + curl_off_t mid; size_t slot; for(slot = 0; slot < DOH_PROBE_SLOTS; slot++) { - if(!doh->probe[slot].easy) + mid = doh->probe[slot].easy_mid; + if(mid < 0) + continue; + doh->probe[slot].easy_mid = -1; + /* should have been called before data is removed from multi handle */ + DEBUGASSERT(data->multi); + probe_data = data->multi? Curl_multi_get_handle(data->multi, mid) : NULL; + if(!probe_data) { + DEBUGF(infof(data, "Curl_doh_close: xfer for mid=%" + CURL_FORMAT_CURL_OFF_T " not found!", + doh->probe[slot].easy_mid)); continue; + } /* data->multi might already be reset at this time */ - if(doh->probe[slot].easy->multi) - curl_multi_remove_handle(doh->probe[slot].easy->multi, - doh->probe[slot].easy); - Curl_close(&doh->probe[slot].easy); + curl_multi_remove_handle(data->multi, probe_data); + Curl_close(&probe_data); } } } diff --git a/lib/doh.h b/lib/doh.h index 318c19520b..14d82a1051 100644 --- a/lib/doh.h +++ b/lib/doh.h @@ -60,7 +60,7 @@ typedef enum { /* one of these for each DoH request */ struct dnsprobe { - CURL *easy; + curl_off_t easy_mid; /* multi id of easy handle doing the lookup */ DNStype dnstype; unsigned char dohbuffer[512]; size_t dohlen; diff --git a/lib/http2.c b/lib/http2.c index 6b4aabca70..ec01628a8e 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -136,7 +136,7 @@ struct cf_h2_ctx { struct bufc_pool stream_bufcp; /* spares for stream buffers */ struct dynbuf scratch; /* scratch buffer for temp use */ - struct Curl_hash streams; /* hash of `data->id` to `h2_stream_ctx` */ + struct Curl_hash streams; /* hash of `data->mid` to `h2_stream_ctx` */ size_t drain_total; /* sum of all stream's UrlState drain */ uint32_t max_concurrent_streams; uint32_t goaway_error; /* goaway error code from server */ @@ -224,7 +224,7 @@ struct h2_stream_ctx { }; #define H2_STREAM_CTX(ctx,data) ((struct h2_stream_ctx *)(\ - data? Curl_hash_offt_get(&(ctx)->streams, (data)->id) : NULL)) + data? Curl_hash_offt_get(&(ctx)->streams, (data)->mid) : NULL)) static struct h2_stream_ctx *h2_stream_ctx_create(struct cf_h2_ctx *ctx) { @@ -387,7 +387,7 @@ static CURLcode http2_data_setup(struct Curl_cfilter *cf, if(!stream) return CURLE_OUT_OF_MEMORY; - if(!Curl_hash_offt_set(&ctx->streams, data->id, stream)) { + if(!Curl_hash_offt_set(&ctx->streams, data->mid, stream)) { h2_stream_ctx_free(stream); return CURLE_OUT_OF_MEMORY; } @@ -425,7 +425,7 @@ static void http2_data_done(struct Curl_cfilter *cf, struct Curl_easy *data) nghttp2_session_send(ctx->h2); } - Curl_hash_offt_remove(&ctx->streams, data->id); + Curl_hash_offt_remove(&ctx->streams, data->mid); } static int h2_client_new(struct Curl_cfilter *cf, @@ -2010,9 +2010,8 @@ static ssize_t cf_h2_recv(struct Curl_cfilter *cf, struct Curl_easy *data, * (unlikely) or the transfer has been done, cleaned up its resources, but * a read() is called anyway. It is not clear what the calling sequence * is for such a case. */ - failf(data, "[%zd-%zd], http/2 recv on a transfer never opened " - "or already cleared", (ssize_t)data->id, - (ssize_t)cf->conn->connection_id); + failf(data, "http/2 recv on a transfer never opened " + "or already cleared, mid=%" CURL_FORMAT_CURL_OFF_T, data->mid); *err = CURLE_HTTP2; return -1; } diff --git a/lib/multi.c b/lib/multi.c index e6b323f1ab..8732837f3b 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -592,9 +592,16 @@ CURLMcode curl_multi_add_handle(struct Curl_multi *multi, data->set.server_response_timeout; data->state.conn_cache->closure_handle->set.no_signal = data->set.no_signal; + + /* the identifier inside the connection cache */ data->id = data->state.conn_cache->next_easy_id++; if(data->state.conn_cache->next_easy_id <= 0) data->state.conn_cache->next_easy_id = 0; + /* the identifier inside the multi instance */ + data->mid = multi->next_easy_mid++; + if(multi->next_easy_mid <= 0) + multi->next_easy_mid = 0; + CONNCACHE_UNLOCK(data); multi_warn_debug(multi, data); @@ -904,8 +911,6 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, since we are not part of that multi handle anymore */ data->state.conn_cache = NULL; - data->multi = NULL; /* clear the association to this multi handle */ - /* make sure there is no pending message in the queue sent from this easy handle */ for(e = Curl_llist_head(&multi->msglist); e; e = Curl_node_next(e)) { @@ -918,6 +923,9 @@ CURLMcode curl_multi_remove_handle(struct Curl_multi *multi, } } + data->multi = NULL; /* clear the association to this multi handle */ + data->mid = -1; + /* NOTE NOTE NOTE We do not touch the easy handle here! */ multi->num_easy--; /* one less to care about now */ @@ -3891,3 +3899,32 @@ static void multi_xfer_bufs_free(struct Curl_multi *multi) multi->xfer_ulbuf_len = 0; multi->xfer_ulbuf_borrowed = FALSE; } + +struct Curl_easy *Curl_multi_get_handle(struct Curl_multi *multi, + curl_off_t mid) +{ + + if(mid >= 0) { + struct Curl_easy *data; + struct Curl_llist_node *e; + + for(e = Curl_llist_head(&multi->process); e; e = Curl_node_next(e)) { + data = Curl_node_elem(e); + if(data->mid == mid) + return data; + } + /* may be in msgsent queue */ + for(e = Curl_llist_head(&multi->msgsent); e; e = Curl_node_next(e)) { + data = Curl_node_elem(e); + if(data->mid == mid) + return data; + } + /* may be in pending queue */ + for(e = Curl_llist_head(&multi->pending); e; e = Curl_node_next(e)) { + data = Curl_node_elem(e); + if(data->mid == mid) + return data; + } + } + return NULL; +} diff --git a/lib/multihandle.h b/lib/multihandle.h index 0649e623aa..ad78c8c013 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -96,6 +96,7 @@ struct Curl_multi { struct Curl_llist process; /* not in PENDING or MSGSENT */ struct Curl_llist pending; /* in PENDING */ struct Curl_llist msgsent; /* in MSGSENT */ + curl_off_t next_easy_mid; /* next multi-id for easy handle added */ /* callback function and user data pointer for the *socket() API */ curl_socket_callback socket_cb; diff --git a/lib/multiif.h b/lib/multiif.h index d3c12baee5..6722e4b2d6 100644 --- a/lib/multiif.h +++ b/lib/multiif.h @@ -153,4 +153,10 @@ CURLcode Curl_multi_xfer_ulbuf_borrow(struct Curl_easy *data, */ void Curl_multi_xfer_ulbuf_release(struct Curl_easy *data, char *buf); +/** + * Get the transfer handle for the given id. Returns NULL if not found. + */ +struct Curl_easy *Curl_multi_get_handle(struct Curl_multi *multi, + curl_off_t id); + #endif /* HEADER_CURL_MULTIIF_H */ diff --git a/lib/request.c b/lib/request.c index 123ee6dc34..d56aee1551 100644 --- a/lib/request.c +++ b/lib/request.c @@ -100,6 +100,9 @@ CURLcode Curl_req_done(struct SingleRequest *req, if(!aborted) (void)req_flush(data); Curl_client_reset(data); +#ifndef CURL_DISABLE_DOH + Curl_doh_close(data); +#endif return CURLE_OK; } diff --git a/lib/url.c b/lib/url.c index d5d7e2fcca..abf9bc14ac 100644 --- a/lib/url.c +++ b/lib/url.c @@ -536,6 +536,10 @@ CURLcode Curl_open(struct Curl_easy **curl) data->state.recent_conn_id = -1; /* and not assigned an id yet */ data->id = -1; + data->mid = -1; +#ifndef CURL_DISABLE_DOH + data->set.dohfor_mid = -1; +#endif data->progress.flags |= PGRS_HIDE; data->state.current_speed = -1; /* init to negative == impossible */ @@ -3581,7 +3585,7 @@ static CURLcode create_conn(struct Curl_easy *data, Curl_disconnect(data, conn_candidate, FALSE); else #ifndef CURL_DISABLE_DOH - if(data->set.dohfor) + if(data->set.dohfor_mid >= 0) infof(data, "Allowing DoH to override max connection limit"); else #endif diff --git a/lib/urldata.h b/lib/urldata.h index bc7ee989ad..096665440c 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1751,7 +1751,7 @@ struct UserDefined { long upkeep_interval_ms; /* Time between calls for connection upkeep. */ multidone_func fmultidone; #ifndef CURL_DISABLE_DOH - struct Curl_easy *dohfor; /* this is a DoH request for that transfer */ + curl_off_t dohfor_mid; /* this is a DoH request for that transfer */ #endif CURLU *uh; /* URL handle for the current parsed URL */ #ifndef CURL_DISABLE_HTTP @@ -1907,8 +1907,14 @@ struct Curl_easy { other using the same cache. For easier tracking in log output. This may wrap around after LONG_MAX to 0 again, so it - has no uniqueness guarantee for very large processings. */ + has no uniqueness guarantee for very large processings. + Note: it has no uniqueness either IFF more than one connection cache + is used by the libcurl application. */ curl_off_t id; + /* once an easy handle is added to a multi, either explicitly by the + * libcurl application or implicitly during `curl_easy_perform()`, + * a unique identifier inside this one multi instance. */ + curl_off_t mid; struct connectdata *conn; struct Curl_llist_node multi_queue; /* for multihandle list management */ diff --git a/lib/vquic/curl_msh3.c b/lib/vquic/curl_msh3.c index 17e7d402f4..d1afc80a2c 100644 --- a/lib/vquic/curl_msh3.c +++ b/lib/vquic/curl_msh3.c @@ -119,7 +119,7 @@ struct cf_msh3_ctx { struct cf_call_data call_data; struct curltime connect_started; /* time the current attempt started */ struct curltime handshake_at; /* time connect handshake finished */ - struct Curl_hash streams; /* hash `data->id` to `stream_ctx` */ + struct Curl_hash streams; /* hash `data->mid` to `stream_ctx` */ /* Flags written by msh3/msquic thread */ bool handshake_complete; bool handshake_succeeded; @@ -180,7 +180,7 @@ struct stream_ctx { }; #define H3_STREAM_CTX(ctx,data) ((struct stream_ctx *)((data && ctx)? \ - Curl_hash_offt_get(&(ctx)->streams, (data)->id) : NULL)) + Curl_hash_offt_get(&(ctx)->streams, (data)->mid) : NULL)) static void h3_stream_ctx_free(struct stream_ctx *stream) { @@ -213,7 +213,7 @@ static CURLcode h3_data_setup(struct Curl_cfilter *cf, H3_STREAM_RECV_CHUNKS, BUFQ_OPT_SOFT_LIMIT); CURL_TRC_CF(data, cf, "data setup"); - if(!Curl_hash_offt_set(&ctx->streams, data->id, stream)) { + if(!Curl_hash_offt_set(&ctx->streams, data->mid, stream)) { h3_stream_ctx_free(stream); return CURLE_OUT_OF_MEMORY; } @@ -229,7 +229,7 @@ static void h3_data_done(struct Curl_cfilter *cf, struct Curl_easy *data) (void)cf; if(stream) { CURL_TRC_CF(data, cf, "easy handle is done"); - Curl_hash_offt_remove(&ctx->streams, data->id); + Curl_hash_offt_remove(&ctx->streams, data->mid); } } diff --git a/lib/vquic/curl_ngtcp2.c b/lib/vquic/curl_ngtcp2.c index dbcc3065dd..ee438ee643 100644 --- a/lib/vquic/curl_ngtcp2.c +++ b/lib/vquic/curl_ngtcp2.c @@ -132,7 +132,7 @@ struct cf_ngtcp2_ctx { struct curltime reconnect_at; /* time the next attempt should start */ struct bufc_pool stream_bufcp; /* chunk pool for streams */ struct dynbuf scratch; /* temp buffer for header construction */ - struct Curl_hash streams; /* hash `data->id` to `h3_stream_ctx` */ + struct Curl_hash streams; /* hash `data->mid` to `h3_stream_ctx` */ size_t max_stream_window; /* max flow window for one stream */ uint64_t max_idle_ms; /* max idle time for QUIC connection */ uint64_t used_bidi_streams; /* bidi streams we have opened */ @@ -203,7 +203,7 @@ struct h3_stream_ctx { }; #define H3_STREAM_CTX(ctx,data) ((struct h3_stream_ctx *)(\ - data? Curl_hash_offt_get(&(ctx)->streams, (data)->id) : NULL)) + data? Curl_hash_offt_get(&(ctx)->streams, (data)->mid) : NULL)) #define H3_STREAM_CTX_ID(ctx,id) ((struct h3_stream_ctx *)(\ Curl_hash_offt_get(&(ctx)->streams, (id)))) @@ -245,7 +245,7 @@ static CURLcode h3_data_setup(struct Curl_cfilter *cf, stream->sendbuf_len_in_flight = 0; Curl_h1_req_parse_init(&stream->h1, H1_PARSE_DEFAULT_MAX_LINE_LEN); - if(!Curl_hash_offt_set(&ctx->streams, data->id, stream)) { + if(!Curl_hash_offt_set(&ctx->streams, data->mid, stream)) { h3_stream_ctx_free(stream); return CURLE_OUT_OF_MEMORY; } @@ -284,7 +284,7 @@ static void h3_data_done(struct Curl_cfilter *cf, struct Curl_easy *data) CURL_TRC_CF(data, cf, "[%" CURL_PRId64 "] easy handle is done", stream->id); cf_ngtcp2_stream_close(cf, data, stream); - Curl_hash_offt_remove(&ctx->streams, data->id); + Curl_hash_offt_remove(&ctx->streams, data->mid); } } diff --git a/lib/vquic/curl_osslq.c b/lib/vquic/curl_osslq.c index 31469c8d43..157dd9c907 100644 --- a/lib/vquic/curl_osslq.c +++ b/lib/vquic/curl_osslq.c @@ -290,7 +290,7 @@ struct cf_osslq_ctx { struct curltime first_byte_at; /* when first byte was recvd */ struct curltime reconnect_at; /* time the next attempt should start */ struct bufc_pool stream_bufcp; /* chunk pool for streams */ - struct Curl_hash streams; /* hash `data->id` to `h3_stream_ctx` */ + struct Curl_hash streams; /* hash `data->mid` to `h3_stream_ctx` */ size_t max_stream_window; /* max flow window for one stream */ uint64_t max_idle_ms; /* max idle time for QUIC connection */ BIT(initialized); @@ -589,7 +589,7 @@ struct h3_stream_ctx { }; #define H3_STREAM_CTX(ctx,data) ((struct h3_stream_ctx *)(\ - data? Curl_hash_offt_get(&(ctx)->streams, (data)->id) : NULL)) + data? Curl_hash_offt_get(&(ctx)->streams, (data)->mid) : NULL)) static void h3_stream_ctx_free(struct h3_stream_ctx *stream) { @@ -636,7 +636,7 @@ static CURLcode h3_data_setup(struct Curl_cfilter *cf, stream->recv_buf_nonflow = 0; Curl_h1_req_parse_init(&stream->h1, H1_PARSE_DEFAULT_MAX_LINE_LEN); - if(!Curl_hash_offt_set(&ctx->streams, data->id, stream)) { + if(!Curl_hash_offt_set(&ctx->streams, data->mid, stream)) { h3_stream_ctx_free(stream); return CURLE_OUT_OF_MEMORY; } @@ -661,7 +661,7 @@ static void h3_data_done(struct Curl_cfilter *cf, struct Curl_easy *data) stream->closed = TRUE; } - Curl_hash_offt_remove(&ctx->streams, data->id); + Curl_hash_offt_remove(&ctx->streams, data->mid); } } diff --git a/lib/vquic/curl_quiche.c b/lib/vquic/curl_quiche.c index 79963ca388..227e0c93b9 100644 --- a/lib/vquic/curl_quiche.c +++ b/lib/vquic/curl_quiche.c @@ -98,7 +98,7 @@ struct cf_quiche_ctx { struct curltime handshake_at; /* time connect handshake finished */ struct curltime reconnect_at; /* time the next attempt should start */ struct bufc_pool stream_bufcp; /* chunk pool for streams */ - struct Curl_hash streams; /* hash `data->id` to `stream_ctx` */ + struct Curl_hash streams; /* hash `data->mid` to `stream_ctx` */ curl_off_t data_recvd; BIT(initialized); BIT(goaway); /* got GOAWAY from server */ @@ -182,7 +182,7 @@ struct stream_ctx { }; #define H3_STREAM_CTX(ctx,data) ((struct stream_ctx *)(\ - data? Curl_hash_offt_get(&(ctx)->streams, (data)->id) : NULL)) + data? Curl_hash_offt_get(&(ctx)->streams, (data)->mid) : NULL)) static void h3_stream_ctx_free(struct stream_ctx *stream) { @@ -235,7 +235,7 @@ static CURLcode h3_data_setup(struct Curl_cfilter *cf, H3_STREAM_RECV_CHUNKS, BUFQ_OPT_SOFT_LIMIT); Curl_h1_req_parse_init(&stream->h1, H1_PARSE_DEFAULT_MAX_LINE_LEN); - if(!Curl_hash_offt_set(&ctx->streams, data->id, stream)) { + if(!Curl_hash_offt_set(&ctx->streams, data->mid, stream)) { h3_stream_ctx_free(stream); return CURLE_OUT_OF_MEMORY; } @@ -265,7 +265,7 @@ static void h3_data_done(struct Curl_cfilter *cf, struct Curl_easy *data) if(result) CURL_TRC_CF(data, cf, "data_done, flush egress -> %d", result); } - Curl_hash_offt_remove(&ctx->streams, data->id); + Curl_hash_offt_remove(&ctx->streams, data->mid); } }