1 #define USE_THE_REPOSITORY_VARIABLE
2 #define DISABLE_SIGN_COMPARE_WARNINGS
4 #include "git-compat-util.h"
5 #include "git-curl-compat.h"
10 #include "run-command.h"
13 #include "credential.h"
18 #include "transport.h"
20 #include "string-list.h"
21 #include "object-file.h"
22 #include "object-store.h"
25 static struct trace_key trace_curl
= TRACE_KEY_INIT(CURL
);
26 static int trace_curl_data
= 1;
27 static int trace_curl_redact
= 1;
28 long int git_curl_ipresolve
= CURL_IPRESOLVE_WHATEVER
;
31 ssize_t http_post_buffer
= 16 * LARGE_PACKET_MAX
;
33 static int min_curl_sessions
= 1;
34 static int curl_session_count
;
35 static int max_requests
= -1;
37 static CURL
*curl_default
;
39 #define PREV_BUF_SIZE 4096
41 char curl_errorstr
[CURL_ERROR_SIZE
];
43 static int curl_ssl_verify
= -1;
44 static int curl_ssl_try
;
45 static char *curl_http_version
;
46 static char *ssl_cert
;
47 static char *ssl_cert_type
;
48 static char *ssl_cipherlist
;
49 static char *ssl_version
;
54 { "sslv2", CURL_SSLVERSION_SSLv2
},
55 { "sslv3", CURL_SSLVERSION_SSLv3
},
56 { "tlsv1", CURL_SSLVERSION_TLSv1
},
57 { "tlsv1.0", CURL_SSLVERSION_TLSv1_0
},
58 { "tlsv1.1", CURL_SSLVERSION_TLSv1_1
},
59 { "tlsv1.2", CURL_SSLVERSION_TLSv1_2
},
60 { "tlsv1.3", CURL_SSLVERSION_TLSv1_3
},
63 static char *ssl_key_type
;
64 static char *ssl_capath
;
65 static char *curl_no_proxy
;
66 static char *ssl_pinnedkey
;
67 static char *ssl_cainfo
;
68 static long curl_low_speed_limit
= -1;
69 static long curl_low_speed_time
= -1;
70 static int curl_ftp_no_epsv
;
71 static char *curl_http_proxy
;
72 static char *http_proxy_authmethod
;
74 static char *http_proxy_ssl_cert
;
75 static char *http_proxy_ssl_key
;
76 static char *http_proxy_ssl_ca_info
;
77 static struct credential proxy_cert_auth
= CREDENTIAL_INIT
;
78 static int proxy_ssl_cert_password_required
;
83 } proxy_authmethods
[] = {
84 { "basic", CURLAUTH_BASIC
},
85 { "digest", CURLAUTH_DIGEST
},
86 { "negotiate", CURLAUTH_GSSNEGOTIATE
},
87 { "ntlm", CURLAUTH_NTLM
},
88 { "anyauth", CURLAUTH_ANY
},
90 * CURLAUTH_DIGEST_IE has no corresponding command-line option in
91 * curl(1) and is not included in CURLAUTH_ANY, so we leave it out
95 #ifdef CURLGSSAPI_DELEGATION_FLAG
96 static char *curl_deleg
;
99 long curl_deleg_param
;
100 } curl_deleg_levels
[] = {
101 { "none", CURLGSSAPI_DELEGATION_NONE
},
102 { "policy", CURLGSSAPI_DELEGATION_POLICY_FLAG
},
103 { "always", CURLGSSAPI_DELEGATION_FLAG
},
107 static long curl_tcp_keepidle
= -1;
108 static long curl_tcp_keepintvl
= -1;
109 static long curl_tcp_keepcnt
= -1;
111 enum proactive_auth
{
112 PROACTIVE_AUTH_NONE
= 0,
113 PROACTIVE_AUTH_IF_CREDENTIALS
,
115 PROACTIVE_AUTH_BASIC
,
118 static struct credential proxy_auth
= CREDENTIAL_INIT
;
119 static const char *curl_proxyuserpwd
;
120 static char *curl_cookie_file
;
121 static int curl_save_cookies
;
122 struct credential http_auth
= CREDENTIAL_INIT
;
123 static enum proactive_auth http_proactive_auth
;
124 static char *user_agent
;
125 static int curl_empty_auth
= -1;
127 enum http_follow_config http_follow_config
= HTTP_FOLLOW_INITIAL
;
129 static struct credential cert_auth
= CREDENTIAL_INIT
;
130 static int ssl_cert_password_required
;
131 static unsigned long http_auth_methods
= CURLAUTH_ANY
;
132 static int http_auth_methods_restricted
;
133 /* Modes for which empty_auth cannot actually help us. */
134 static unsigned long empty_auth_useless
=
139 static struct curl_slist
*pragma_header
;
140 static struct string_list extra_http_headers
= STRING_LIST_INIT_DUP
;
142 static struct curl_slist
*host_resolutions
;
144 static struct active_request_slot
*active_queue_head
;
146 static char *cached_accept_language
;
148 static char *http_ssl_backend
;
150 static int http_schannel_check_revoke
= 1;
152 * With the backend being set to `schannel`, setting sslCAinfo would override
153 * the Certificate Store in cURL v7.60.0 and later, which is not what we want
156 static int http_schannel_use_ssl_cainfo
;
158 static int always_auth_proactively(void)
160 return http_proactive_auth
!= PROACTIVE_AUTH_NONE
&&
161 http_proactive_auth
!= PROACTIVE_AUTH_IF_CREDENTIALS
;
164 size_t fread_buffer(char *ptr
, size_t eltsize
, size_t nmemb
, void *buffer_
)
166 size_t size
= eltsize
* nmemb
;
167 struct buffer
*buffer
= buffer_
;
169 if (size
> buffer
->buf
.len
- buffer
->posn
)
170 size
= buffer
->buf
.len
- buffer
->posn
;
171 memcpy(ptr
, buffer
->buf
.buf
+ buffer
->posn
, size
);
172 buffer
->posn
+= size
;
174 return size
/ eltsize
;
177 int seek_buffer(void *clientp
, curl_off_t offset
, int origin
)
179 struct buffer
*buffer
= clientp
;
181 if (origin
!= SEEK_SET
)
182 BUG("seek_buffer only handles SEEK_SET");
183 if (offset
< 0 || offset
>= buffer
->buf
.len
) {
184 error("curl seek would be outside of buffer");
185 return CURL_SEEKFUNC_FAIL
;
188 buffer
->posn
= offset
;
189 return CURL_SEEKFUNC_OK
;
192 size_t fwrite_buffer(char *ptr
, size_t eltsize
, size_t nmemb
, void *buffer_
)
194 size_t size
= eltsize
* nmemb
;
195 struct strbuf
*buffer
= buffer_
;
197 strbuf_add(buffer
, ptr
, size
);
202 * A folded header continuation line starts with any number of spaces or
203 * horizontal tab characters (SP or HTAB) as per RFC 7230 section 3.2.
204 * It is not a continuation line if the line starts with any other character.
206 static inline int is_hdr_continuation(const char *ptr
, const size_t size
)
208 return size
&& (*ptr
== ' ' || *ptr
== '\t');
211 static size_t fwrite_wwwauth(char *ptr
, size_t eltsize
, size_t nmemb
, void *p UNUSED
)
213 size_t size
= eltsize
* nmemb
;
214 struct strvec
*values
= &http_auth
.wwwauth_headers
;
215 struct strbuf buf
= STRBUF_INIT
;
220 * Header lines may not come NULL-terminated from libcurl so we must
221 * limit all scans to the maximum length of the header line, or leverage
222 * strbufs for all operations.
224 * In addition, it is possible that header values can be split over
225 * multiple lines as per RFC 7230. 'Line folding' has been deprecated
226 * but older servers may still emit them. A continuation header field
227 * value is identified as starting with a space or horizontal tab.
229 * The formal definition of a header field as given in RFC 7230 is:
231 * header-field = field-name ":" OWS field-value OWS
234 * field-value = *( field-content / obs-fold )
235 * field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ]
236 * field-vchar = VCHAR / obs-text
238 * obs-fold = CRLF 1*( SP / HTAB )
239 * ; obsolete line folding
240 * ; see Section 3.2.4
243 /* Start of a new WWW-Authenticate header */
244 if (skip_iprefix_mem(ptr
, size
, "www-authenticate:", &val
, &val_len
)) {
245 strbuf_add(&buf
, val
, val_len
);
248 * Strip the CRLF that should be present at the end of each
249 * field as well as any trailing or leading whitespace from the
254 strvec_push(values
, buf
.buf
);
255 http_auth
.header_is_last_match
= 1;
260 * This line could be a continuation of the previously matched header
261 * field. If this is the case then we should append this value to the
262 * end of the previously consumed value.
264 if (http_auth
.header_is_last_match
&& is_hdr_continuation(ptr
, size
)) {
266 * Trim the CRLF and any leading or trailing from this line.
268 strbuf_add(&buf
, ptr
, size
);
272 * At this point we should always have at least one existing
273 * value, even if it is empty. Do not bother appending the new
274 * value if this continuation header is itself empty.
277 BUG("should have at least one existing header value");
278 } else if (buf
.len
) {
279 char *prev
= xstrdup(values
->v
[values
->nr
- 1]);
281 /* Join two non-empty values with a single space. */
282 const char *const sp
= *prev
? " " : "";
285 strvec_pushf(values
, "%s%s%s", prev
, sp
, buf
.buf
);
292 /* Not a continuation of a previously matched auth header line. */
293 http_auth
.header_is_last_match
= 0;
296 * If this is a HTTP status line and not a header field, this signals
297 * a different HTTP response. libcurl writes all the output of all
298 * response headers of all responses, including redirects.
299 * We only care about the last HTTP request response's headers so clear
300 * the existing array.
302 if (skip_iprefix_mem(ptr
, size
, "http/", &val
, &val_len
))
303 strvec_clear(values
);
306 strbuf_release(&buf
);
310 size_t fwrite_null(char *ptr UNUSED
, size_t eltsize UNUSED
, size_t nmemb
,
316 static struct curl_slist
*object_request_headers(void)
318 return curl_slist_append(http_copy_default_headers(), "Pragma:");
321 static void closedown_active_slot(struct active_request_slot
*slot
)
327 static void finish_active_slot(struct active_request_slot
*slot
)
329 closedown_active_slot(slot
);
330 curl_easy_getinfo(slot
->curl
, CURLINFO_HTTP_CODE
, &slot
->http_code
);
333 (*slot
->finished
) = 1;
335 /* Store slot results so they can be read after the slot is reused */
337 slot
->results
->curl_result
= slot
->curl_result
;
338 slot
->results
->http_code
= slot
->http_code
;
339 curl_easy_getinfo(slot
->curl
, CURLINFO_HTTPAUTH_AVAIL
,
340 &slot
->results
->auth_avail
);
342 curl_easy_getinfo(slot
->curl
, CURLINFO_HTTP_CONNECTCODE
,
343 &slot
->results
->http_connectcode
);
346 /* Run callback if appropriate */
347 if (slot
->callback_func
)
348 slot
->callback_func(slot
->callback_data
);
351 static void xmulti_remove_handle(struct active_request_slot
*slot
)
353 curl_multi_remove_handle(curlm
, slot
->curl
);
356 static void process_curl_messages(void)
359 struct active_request_slot
*slot
;
360 CURLMsg
*curl_message
= curl_multi_info_read(curlm
, &num_messages
);
362 while (curl_message
!= NULL
) {
363 if (curl_message
->msg
== CURLMSG_DONE
) {
364 int curl_result
= curl_message
->data
.result
;
365 slot
= active_queue_head
;
366 while (slot
!= NULL
&&
367 slot
->curl
!= curl_message
->easy_handle
)
370 xmulti_remove_handle(slot
);
371 slot
->curl_result
= curl_result
;
372 finish_active_slot(slot
);
374 fprintf(stderr
, "Received DONE message for unknown request!\n");
377 fprintf(stderr
, "Unknown CURL message received: %d\n",
378 (int)curl_message
->msg
);
380 curl_message
= curl_multi_info_read(curlm
, &num_messages
);
384 static int http_options(const char *var
, const char *value
,
385 const struct config_context
*ctx
, void *data
)
387 if (!strcmp("http.version", var
)) {
388 return git_config_string(&curl_http_version
, var
, value
);
390 if (!strcmp("http.sslverify", var
)) {
391 curl_ssl_verify
= git_config_bool(var
, value
);
394 if (!strcmp("http.sslcipherlist", var
))
395 return git_config_string(&ssl_cipherlist
, var
, value
);
396 if (!strcmp("http.sslversion", var
))
397 return git_config_string(&ssl_version
, var
, value
);
398 if (!strcmp("http.sslcert", var
))
399 return git_config_pathname(&ssl_cert
, var
, value
);
400 if (!strcmp("http.sslcerttype", var
))
401 return git_config_string(&ssl_cert_type
, var
, value
);
402 if (!strcmp("http.sslkey", var
))
403 return git_config_pathname(&ssl_key
, var
, value
);
404 if (!strcmp("http.sslkeytype", var
))
405 return git_config_string(&ssl_key_type
, var
, value
);
406 if (!strcmp("http.sslcapath", var
))
407 return git_config_pathname(&ssl_capath
, var
, value
);
408 if (!strcmp("http.sslcainfo", var
))
409 return git_config_pathname(&ssl_cainfo
, var
, value
);
410 if (!strcmp("http.sslcertpasswordprotected", var
)) {
411 ssl_cert_password_required
= git_config_bool(var
, value
);
414 if (!strcmp("http.ssltry", var
)) {
415 curl_ssl_try
= git_config_bool(var
, value
);
418 if (!strcmp("http.sslbackend", var
)) {
419 free(http_ssl_backend
);
420 http_ssl_backend
= xstrdup_or_null(value
);
424 if (!strcmp("http.schannelcheckrevoke", var
)) {
425 http_schannel_check_revoke
= git_config_bool(var
, value
);
429 if (!strcmp("http.schannelusesslcainfo", var
)) {
430 http_schannel_use_ssl_cainfo
= git_config_bool(var
, value
);
434 if (!strcmp("http.minsessions", var
)) {
435 min_curl_sessions
= git_config_int(var
, value
, ctx
->kvi
);
436 if (min_curl_sessions
> 1)
437 min_curl_sessions
= 1;
440 if (!strcmp("http.maxrequests", var
)) {
441 max_requests
= git_config_int(var
, value
, ctx
->kvi
);
444 if (!strcmp("http.lowspeedlimit", var
)) {
445 curl_low_speed_limit
= git_config_int(var
, value
, ctx
->kvi
);
448 if (!strcmp("http.lowspeedtime", var
)) {
449 curl_low_speed_time
= git_config_int(var
, value
, ctx
->kvi
);
453 if (!strcmp("http.noepsv", var
)) {
454 curl_ftp_no_epsv
= git_config_bool(var
, value
);
457 if (!strcmp("http.proxy", var
))
458 return git_config_string(&curl_http_proxy
, var
, value
);
460 if (!strcmp("http.proxyauthmethod", var
))
461 return git_config_string(&http_proxy_authmethod
, var
, value
);
463 if (!strcmp("http.proxysslcert", var
))
464 return git_config_string(&http_proxy_ssl_cert
, var
, value
);
466 if (!strcmp("http.proxysslkey", var
))
467 return git_config_string(&http_proxy_ssl_key
, var
, value
);
469 if (!strcmp("http.proxysslcainfo", var
))
470 return git_config_string(&http_proxy_ssl_ca_info
, var
, value
);
472 if (!strcmp("http.proxysslcertpasswordprotected", var
)) {
473 proxy_ssl_cert_password_required
= git_config_bool(var
, value
);
477 if (!strcmp("http.cookiefile", var
))
478 return git_config_pathname(&curl_cookie_file
, var
, value
);
479 if (!strcmp("http.savecookies", var
)) {
480 curl_save_cookies
= git_config_bool(var
, value
);
484 if (!strcmp("http.postbuffer", var
)) {
485 http_post_buffer
= git_config_ssize_t(var
, value
, ctx
->kvi
);
486 if (http_post_buffer
< 0)
487 warning(_("negative value for http.postBuffer; defaulting to %d"), LARGE_PACKET_MAX
);
488 if (http_post_buffer
< LARGE_PACKET_MAX
)
489 http_post_buffer
= LARGE_PACKET_MAX
;
493 if (!strcmp("http.useragent", var
))
494 return git_config_string(&user_agent
, var
, value
);
496 if (!strcmp("http.emptyauth", var
)) {
497 if (value
&& !strcmp("auto", value
))
498 curl_empty_auth
= -1;
500 curl_empty_auth
= git_config_bool(var
, value
);
504 if (!strcmp("http.delegation", var
)) {
505 #ifdef CURLGSSAPI_DELEGATION_FLAG
506 return git_config_string(&curl_deleg
, var
, value
);
508 warning(_("Delegation control is not supported with cURL < 7.22.0"));
513 if (!strcmp("http.pinnedpubkey", var
)) {
514 return git_config_pathname(&ssl_pinnedkey
, var
, value
);
517 if (!strcmp("http.extraheader", var
)) {
519 return config_error_nonbool(var
);
520 } else if (!*value
) {
521 string_list_clear(&extra_http_headers
, 0);
523 string_list_append(&extra_http_headers
, value
);
528 if (!strcmp("http.curloptresolve", var
)) {
530 return config_error_nonbool(var
);
531 } else if (!*value
) {
532 curl_slist_free_all(host_resolutions
);
533 host_resolutions
= NULL
;
535 host_resolutions
= curl_slist_append(host_resolutions
, value
);
540 if (!strcmp("http.followredirects", var
)) {
541 if (value
&& !strcmp(value
, "initial"))
542 http_follow_config
= HTTP_FOLLOW_INITIAL
;
543 else if (git_config_bool(var
, value
))
544 http_follow_config
= HTTP_FOLLOW_ALWAYS
;
546 http_follow_config
= HTTP_FOLLOW_NONE
;
550 if (!strcmp("http.proactiveauth", var
)) {
552 return config_error_nonbool(var
);
553 if (!strcmp(value
, "auto"))
554 http_proactive_auth
= PROACTIVE_AUTH_AUTO
;
555 else if (!strcmp(value
, "basic"))
556 http_proactive_auth
= PROACTIVE_AUTH_BASIC
;
557 else if (!strcmp(value
, "none"))
558 http_proactive_auth
= PROACTIVE_AUTH_NONE
;
560 warning(_("Unknown value for http.proactiveauth"));
564 if (!strcmp("http.keepaliveidle", var
)) {
565 curl_tcp_keepidle
= git_config_int(var
, value
, ctx
->kvi
);
568 if (!strcmp("http.keepaliveinterval", var
)) {
569 curl_tcp_keepintvl
= git_config_int(var
, value
, ctx
->kvi
);
572 if (!strcmp("http.keepalivecount", var
)) {
573 curl_tcp_keepcnt
= git_config_int(var
, value
, ctx
->kvi
);
577 /* Fall back on the default ones */
578 return git_default_config(var
, value
, ctx
, data
);
581 static int curl_empty_auth_enabled(void)
583 if (curl_empty_auth
>= 0)
584 return curl_empty_auth
;
587 * In the automatic case, kick in the empty-auth
588 * hack as long as we would potentially try some
589 * method more exotic than "Basic" or "Digest".
591 * But only do this when this is our second or
592 * subsequent request, as by then we know what
593 * methods are available.
595 if (http_auth_methods_restricted
&&
596 (http_auth_methods
& ~empty_auth_useless
))
601 struct curl_slist
*http_append_auth_header(const struct credential
*c
,
602 struct curl_slist
*headers
)
604 if (c
->authtype
&& c
->credential
) {
605 struct strbuf auth
= STRBUF_INIT
;
606 strbuf_addf(&auth
, "Authorization: %s %s",
607 c
->authtype
, c
->credential
);
608 headers
= curl_slist_append(headers
, auth
.buf
);
609 strbuf_release(&auth
);
614 static void init_curl_http_auth(CURL
*result
)
616 if ((!http_auth
.username
|| !*http_auth
.username
) &&
617 (!http_auth
.credential
|| !*http_auth
.credential
)) {
618 if (!always_auth_proactively() && curl_empty_auth_enabled()) {
619 curl_easy_setopt(result
, CURLOPT_USERPWD
, ":");
621 } else if (!always_auth_proactively()) {
623 } else if (http_proactive_auth
== PROACTIVE_AUTH_BASIC
) {
624 strvec_push(&http_auth
.wwwauth_headers
, "Basic");
628 credential_fill(the_repository
, &http_auth
, 1);
630 if (http_auth
.password
) {
631 if (always_auth_proactively()) {
633 * We got a credential without an authtype and we don't
634 * know what's available. Since our only two options at
635 * the moment are auto (which defaults to basic) and
636 * basic, use basic for now.
638 curl_easy_setopt(result
, CURLOPT_HTTPAUTH
, CURLAUTH_BASIC
);
640 curl_easy_setopt(result
, CURLOPT_USERNAME
, http_auth
.username
);
641 curl_easy_setopt(result
, CURLOPT_PASSWORD
, http_auth
.password
);
645 /* *var must be free-able */
646 static void var_override(char **var
, char *value
)
650 *var
= xstrdup(value
);
654 static void set_proxyauth_name_password(CURL
*result
)
656 if (proxy_auth
.password
) {
657 curl_easy_setopt(result
, CURLOPT_PROXYUSERNAME
,
658 proxy_auth
.username
);
659 curl_easy_setopt(result
, CURLOPT_PROXYPASSWORD
,
660 proxy_auth
.password
);
661 } else if (proxy_auth
.authtype
&& proxy_auth
.credential
) {
662 curl_easy_setopt(result
, CURLOPT_PROXYHEADER
,
663 http_append_auth_header(&proxy_auth
, NULL
));
667 static void init_curl_proxy_auth(CURL
*result
)
669 if (proxy_auth
.username
) {
670 if (!proxy_auth
.password
&& !proxy_auth
.credential
)
671 credential_fill(the_repository
, &proxy_auth
, 1);
672 set_proxyauth_name_password(result
);
675 var_override(&http_proxy_authmethod
, getenv("GIT_HTTP_PROXY_AUTHMETHOD"));
677 if (http_proxy_authmethod
) {
679 for (i
= 0; i
< ARRAY_SIZE(proxy_authmethods
); i
++) {
680 if (!strcmp(http_proxy_authmethod
, proxy_authmethods
[i
].name
)) {
681 curl_easy_setopt(result
, CURLOPT_PROXYAUTH
,
682 proxy_authmethods
[i
].curlauth_param
);
686 if (i
== ARRAY_SIZE(proxy_authmethods
)) {
687 warning("unsupported proxy authentication method %s: using anyauth",
688 http_proxy_authmethod
);
689 curl_easy_setopt(result
, CURLOPT_PROXYAUTH
, CURLAUTH_ANY
);
693 curl_easy_setopt(result
, CURLOPT_PROXYAUTH
, CURLAUTH_ANY
);
696 static int has_cert_password(void)
698 if (ssl_cert
== NULL
|| ssl_cert_password_required
!= 1)
700 if (!cert_auth
.password
) {
701 cert_auth
.protocol
= xstrdup("cert");
702 cert_auth
.host
= xstrdup("");
703 cert_auth
.username
= xstrdup("");
704 cert_auth
.path
= xstrdup(ssl_cert
);
705 credential_fill(the_repository
, &cert_auth
, 0);
710 static int has_proxy_cert_password(void)
712 if (http_proxy_ssl_cert
== NULL
|| proxy_ssl_cert_password_required
!= 1)
714 if (!proxy_cert_auth
.password
) {
715 proxy_cert_auth
.protocol
= xstrdup("cert");
716 proxy_cert_auth
.host
= xstrdup("");
717 proxy_cert_auth
.username
= xstrdup("");
718 proxy_cert_auth
.path
= xstrdup(http_proxy_ssl_cert
);
719 credential_fill(the_repository
, &proxy_cert_auth
, 0);
724 /* Return 1 if redactions have been made, 0 otherwise. */
725 static int redact_sensitive_header(struct strbuf
*header
, size_t offset
)
728 const char *sensitive_header
;
730 if (trace_curl_redact
&&
731 (skip_iprefix(header
->buf
+ offset
, "Authorization:", &sensitive_header
) ||
732 skip_iprefix(header
->buf
+ offset
, "Proxy-Authorization:", &sensitive_header
))) {
733 /* The first token is the type, which is OK to log */
734 while (isspace(*sensitive_header
))
736 while (*sensitive_header
&& !isspace(*sensitive_header
))
738 /* Everything else is opaque and possibly sensitive */
739 strbuf_setlen(header
, sensitive_header
- header
->buf
);
740 strbuf_addstr(header
, " <redacted>");
742 } else if (trace_curl_redact
&&
743 skip_iprefix(header
->buf
+ offset
, "Cookie:", &sensitive_header
)) {
744 struct strbuf redacted_header
= STRBUF_INIT
;
747 while (isspace(*sensitive_header
))
750 cookie
= sensitive_header
;
754 char *semicolon
= strstr(cookie
, "; ");
757 equals
= strchrnul(cookie
, '=');
759 /* invalid cookie, just append and continue */
760 strbuf_addstr(&redacted_header
, cookie
);
763 strbuf_add(&redacted_header
, cookie
, equals
- cookie
);
764 strbuf_addstr(&redacted_header
, "=<redacted>");
767 * There are more cookies. (Or, for some
768 * reason, the input string ends in "; ".)
770 strbuf_addstr(&redacted_header
, "; ");
771 cookie
= semicolon
+ strlen("; ");
777 strbuf_setlen(header
, sensitive_header
- header
->buf
);
778 strbuf_addbuf(header
, &redacted_header
);
779 strbuf_release(&redacted_header
);
785 static int match_curl_h2_trace(const char *line
, const char **out
)
790 * curl prior to 8.1.0 gives us:
792 * h2h3 [<header-name>: <header-val>]
794 * Starting in 8.1.0, the first token became just "h2".
796 if (skip_iprefix(line
, "h2h3 [", out
) ||
797 skip_iprefix(line
, "h2 [", out
))
802 * [HTTP/2] [<stream-id>] [<header-name>: <header-val>]
803 * where <stream-id> is numeric.
805 if (skip_iprefix(line
, "[HTTP/2] [", &p
)) {
808 if (skip_prefix(p
, "] [", out
))
815 /* Redact headers in info */
816 static void redact_sensitive_info_header(struct strbuf
*header
)
818 const char *sensitive_header
;
820 if (trace_curl_redact
&&
821 match_curl_h2_trace(header
->buf
, &sensitive_header
)) {
822 if (redact_sensitive_header(header
, sensitive_header
- header
->buf
)) {
823 /* redaction ate our closing bracket */
824 strbuf_addch(header
, ']');
829 static void curl_dump_header(const char *text
, unsigned char *ptr
, size_t size
, int hide_sensitive_header
)
831 struct strbuf out
= STRBUF_INIT
;
832 struct strbuf
**headers
, **header
;
834 strbuf_addf(&out
, "%s, %10.10ld bytes (0x%8.8lx)\n",
835 text
, (long)size
, (long)size
);
836 trace_strbuf(&trace_curl
, &out
);
838 strbuf_add(&out
, ptr
, size
);
839 headers
= strbuf_split_max(&out
, '\n', 0);
841 for (header
= headers
; *header
; header
++) {
842 if (hide_sensitive_header
)
843 redact_sensitive_header(*header
, 0);
844 strbuf_insertstr((*header
), 0, text
);
845 strbuf_insertstr((*header
), strlen(text
), ": ");
846 strbuf_rtrim((*header
));
847 strbuf_addch((*header
), '\n');
848 trace_strbuf(&trace_curl
, (*header
));
850 strbuf_list_free(headers
);
851 strbuf_release(&out
);
854 static void curl_dump_data(const char *text
, unsigned char *ptr
, size_t size
)
857 struct strbuf out
= STRBUF_INIT
;
858 unsigned int width
= 60;
860 strbuf_addf(&out
, "%s, %10.10ld bytes (0x%8.8lx)\n",
861 text
, (long)size
, (long)size
);
862 trace_strbuf(&trace_curl
, &out
);
864 for (i
= 0; i
< size
; i
+= width
) {
868 strbuf_addf(&out
, "%s: ", text
);
869 for (w
= 0; (w
< width
) && (i
+ w
< size
); w
++) {
870 unsigned char ch
= ptr
[i
+ w
];
873 (ch
>= 0x20) && (ch
< 0x80)
876 strbuf_addch(&out
, '\n');
877 trace_strbuf(&trace_curl
, &out
);
879 strbuf_release(&out
);
882 static void curl_dump_info(char *data
, size_t size
)
884 struct strbuf buf
= STRBUF_INIT
;
886 strbuf_add(&buf
, data
, size
);
888 redact_sensitive_info_header(&buf
);
889 trace_printf_key(&trace_curl
, "== Info: %s", buf
.buf
);
891 strbuf_release(&buf
);
894 static int curl_trace(CURL
*handle UNUSED
, curl_infotype type
,
895 char *data
, size_t size
,
899 enum { NO_FILTER
= 0, DO_FILTER
= 1 };
903 curl_dump_info(data
, size
);
905 case CURLINFO_HEADER_OUT
:
906 text
= "=> Send header";
907 curl_dump_header(text
, (unsigned char *)data
, size
, DO_FILTER
);
909 case CURLINFO_DATA_OUT
:
910 if (trace_curl_data
) {
911 text
= "=> Send data";
912 curl_dump_data(text
, (unsigned char *)data
, size
);
915 case CURLINFO_SSL_DATA_OUT
:
916 if (trace_curl_data
) {
917 text
= "=> Send SSL data";
918 curl_dump_data(text
, (unsigned char *)data
, size
);
921 case CURLINFO_HEADER_IN
:
922 text
= "<= Recv header";
923 curl_dump_header(text
, (unsigned char *)data
, size
, NO_FILTER
);
925 case CURLINFO_DATA_IN
:
926 if (trace_curl_data
) {
927 text
= "<= Recv data";
928 curl_dump_data(text
, (unsigned char *)data
, size
);
931 case CURLINFO_SSL_DATA_IN
:
932 if (trace_curl_data
) {
933 text
= "<= Recv SSL data";
934 curl_dump_data(text
, (unsigned char *)data
, size
);
938 default: /* we ignore unknown types by default */
944 void http_trace_curl_no_data(void)
946 trace_override_envvar(&trace_curl
, "1");
950 void setup_curl_trace(CURL
*handle
)
952 if (!trace_want(&trace_curl
))
954 curl_easy_setopt(handle
, CURLOPT_VERBOSE
, 1L);
955 curl_easy_setopt(handle
, CURLOPT_DEBUGFUNCTION
, curl_trace
);
956 curl_easy_setopt(handle
, CURLOPT_DEBUGDATA
, NULL
);
959 static void proto_list_append(struct strbuf
*list
, const char *proto
)
964 strbuf_addch(list
, ',');
965 strbuf_addstr(list
, proto
);
968 static long get_curl_allowed_protocols(int from_user
, struct strbuf
*list
)
972 if (is_transport_allowed("http", from_user
)) {
973 bits
|= CURLPROTO_HTTP
;
974 proto_list_append(list
, "http");
976 if (is_transport_allowed("https", from_user
)) {
977 bits
|= CURLPROTO_HTTPS
;
978 proto_list_append(list
, "https");
980 if (is_transport_allowed("ftp", from_user
)) {
981 bits
|= CURLPROTO_FTP
;
982 proto_list_append(list
, "ftp");
984 if (is_transport_allowed("ftps", from_user
)) {
985 bits
|= CURLPROTO_FTPS
;
986 proto_list_append(list
, "ftps");
992 static int get_curl_http_version_opt(const char *version_string
, long *opt
)
999 { "HTTP/1.1", CURL_HTTP_VERSION_1_1
},
1000 { "HTTP/2", CURL_HTTP_VERSION_2
}
1003 for (i
= 0; i
< ARRAY_SIZE(choice
); i
++) {
1004 if (!strcmp(version_string
, choice
[i
].name
)) {
1005 *opt
= choice
[i
].opt_token
;
1010 warning("unknown value given to http.version: '%s'", version_string
);
1011 return -1; /* not found */
1014 static CURL
*get_curl_handle(void)
1016 CURL
*result
= curl_easy_init();
1019 die("curl_easy_init failed");
1021 if (!curl_ssl_verify
) {
1022 curl_easy_setopt(result
, CURLOPT_SSL_VERIFYPEER
, 0L);
1023 curl_easy_setopt(result
, CURLOPT_SSL_VERIFYHOST
, 0L);
1025 /* Verify authenticity of the peer's certificate */
1026 curl_easy_setopt(result
, CURLOPT_SSL_VERIFYPEER
, 1L);
1027 /* The name in the cert must match whom we tried to connect */
1028 curl_easy_setopt(result
, CURLOPT_SSL_VERIFYHOST
, 2L);
1031 if (curl_http_version
) {
1033 if (!get_curl_http_version_opt(curl_http_version
, &opt
)) {
1034 /* Set request use http version */
1035 curl_easy_setopt(result
, CURLOPT_HTTP_VERSION
, opt
);
1039 curl_easy_setopt(result
, CURLOPT_NETRC
, CURL_NETRC_OPTIONAL
);
1040 curl_easy_setopt(result
, CURLOPT_HTTPAUTH
, CURLAUTH_ANY
);
1042 #ifdef CURLGSSAPI_DELEGATION_FLAG
1045 for (i
= 0; i
< ARRAY_SIZE(curl_deleg_levels
); i
++) {
1046 if (!strcmp(curl_deleg
, curl_deleg_levels
[i
].name
)) {
1047 curl_easy_setopt(result
, CURLOPT_GSSAPI_DELEGATION
,
1048 curl_deleg_levels
[i
].curl_deleg_param
);
1052 if (i
== ARRAY_SIZE(curl_deleg_levels
))
1053 warning("Unknown delegation method '%s': using default",
1058 if (http_ssl_backend
&& !strcmp("schannel", http_ssl_backend
) &&
1059 !http_schannel_check_revoke
) {
1060 curl_easy_setopt(result
, CURLOPT_SSL_OPTIONS
, (long)CURLSSLOPT_NO_REVOKE
);
1063 if (http_proactive_auth
!= PROACTIVE_AUTH_NONE
)
1064 init_curl_http_auth(result
);
1066 if (getenv("GIT_SSL_VERSION"))
1067 ssl_version
= getenv("GIT_SSL_VERSION");
1068 if (ssl_version
&& *ssl_version
) {
1070 for (i
= 0; i
< ARRAY_SIZE(sslversions
); i
++) {
1071 if (!strcmp(ssl_version
, sslversions
[i
].name
)) {
1072 curl_easy_setopt(result
, CURLOPT_SSLVERSION
,
1073 sslversions
[i
].ssl_version
);
1077 if (i
== ARRAY_SIZE(sslversions
))
1078 warning("unsupported ssl version %s: using default",
1082 if (getenv("GIT_SSL_CIPHER_LIST"))
1083 ssl_cipherlist
= getenv("GIT_SSL_CIPHER_LIST");
1084 if (ssl_cipherlist
!= NULL
&& *ssl_cipherlist
)
1085 curl_easy_setopt(result
, CURLOPT_SSL_CIPHER_LIST
,
1089 curl_easy_setopt(result
, CURLOPT_SSLCERT
, ssl_cert
);
1091 curl_easy_setopt(result
, CURLOPT_SSLCERTTYPE
, ssl_cert_type
);
1092 if (has_cert_password())
1093 curl_easy_setopt(result
, CURLOPT_KEYPASSWD
, cert_auth
.password
);
1095 curl_easy_setopt(result
, CURLOPT_SSLKEY
, ssl_key
);
1097 curl_easy_setopt(result
, CURLOPT_SSLKEYTYPE
, ssl_key_type
);
1099 curl_easy_setopt(result
, CURLOPT_CAPATH
, ssl_capath
);
1101 curl_easy_setopt(result
, CURLOPT_PINNEDPUBLICKEY
, ssl_pinnedkey
);
1102 if (http_ssl_backend
&& !strcmp("schannel", http_ssl_backend
) &&
1103 !http_schannel_use_ssl_cainfo
) {
1104 curl_easy_setopt(result
, CURLOPT_CAINFO
, NULL
);
1105 curl_easy_setopt(result
, CURLOPT_PROXY_CAINFO
, NULL
);
1106 } else if (ssl_cainfo
!= NULL
|| http_proxy_ssl_ca_info
!= NULL
) {
1108 curl_easy_setopt(result
, CURLOPT_CAINFO
, ssl_cainfo
);
1109 if (http_proxy_ssl_ca_info
)
1110 curl_easy_setopt(result
, CURLOPT_PROXY_CAINFO
, http_proxy_ssl_ca_info
);
1113 if (curl_low_speed_limit
> 0 && curl_low_speed_time
> 0) {
1114 curl_easy_setopt(result
, CURLOPT_LOW_SPEED_LIMIT
,
1115 curl_low_speed_limit
);
1116 curl_easy_setopt(result
, CURLOPT_LOW_SPEED_TIME
,
1117 curl_low_speed_time
);
1120 curl_easy_setopt(result
, CURLOPT_MAXREDIRS
, 20L);
1121 curl_easy_setopt(result
, CURLOPT_POSTREDIR
, (long)CURL_REDIR_POST_ALL
);
1123 #ifdef GIT_CURL_HAVE_CURLOPT_PROTOCOLS_STR
1125 struct strbuf buf
= STRBUF_INIT
;
1127 get_curl_allowed_protocols(0, &buf
);
1128 curl_easy_setopt(result
, CURLOPT_REDIR_PROTOCOLS_STR
, buf
.buf
);
1131 get_curl_allowed_protocols(-1, &buf
);
1132 curl_easy_setopt(result
, CURLOPT_PROTOCOLS_STR
, buf
.buf
);
1133 strbuf_release(&buf
);
1136 curl_easy_setopt(result
, CURLOPT_REDIR_PROTOCOLS
,
1137 get_curl_allowed_protocols(0, NULL
));
1138 curl_easy_setopt(result
, CURLOPT_PROTOCOLS
,
1139 get_curl_allowed_protocols(-1, NULL
));
1142 if (getenv("GIT_CURL_VERBOSE"))
1143 http_trace_curl_no_data();
1144 setup_curl_trace(result
);
1145 if (getenv("GIT_TRACE_CURL_NO_DATA"))
1146 trace_curl_data
= 0;
1147 if (!git_env_bool("GIT_TRACE_REDACT", 1))
1148 trace_curl_redact
= 0;
1150 curl_easy_setopt(result
, CURLOPT_USERAGENT
,
1151 user_agent
? user_agent
: git_user_agent());
1153 if (curl_ftp_no_epsv
)
1154 curl_easy_setopt(result
, CURLOPT_FTP_USE_EPSV
, 0L);
1157 curl_easy_setopt(result
, CURLOPT_USE_SSL
, CURLUSESSL_TRY
);
1160 * CURL also examines these variables as a fallback; but we need to query
1161 * them here in order to decide whether to prompt for missing password (cf.
1162 * init_curl_proxy_auth()).
1164 * Unlike many other common environment variables, these are historically
1165 * lowercase only. It appears that CURL did not know this and implemented
1166 * only uppercase variants, which was later corrected to take both - with
1167 * the exception of http_proxy, which is lowercase only also in CURL. As
1168 * the lowercase versions are the historical quasi-standard, they take
1169 * precedence here, as in CURL.
1171 if (!curl_http_proxy
) {
1172 if (http_auth
.protocol
&& !strcmp(http_auth
.protocol
, "https")) {
1173 var_override(&curl_http_proxy
, getenv("HTTPS_PROXY"));
1174 var_override(&curl_http_proxy
, getenv("https_proxy"));
1176 var_override(&curl_http_proxy
, getenv("http_proxy"));
1178 if (!curl_http_proxy
) {
1179 var_override(&curl_http_proxy
, getenv("ALL_PROXY"));
1180 var_override(&curl_http_proxy
, getenv("all_proxy"));
1184 if (curl_http_proxy
&& curl_http_proxy
[0] == '\0') {
1186 * Handle case with the empty http.proxy value here to keep
1187 * common code clean.
1188 * NB: empty option disables proxying at all.
1190 curl_easy_setopt(result
, CURLOPT_PROXY
, "");
1191 } else if (curl_http_proxy
) {
1192 struct strbuf proxy
= STRBUF_INIT
;
1194 if (starts_with(curl_http_proxy
, "socks5h"))
1195 curl_easy_setopt(result
,
1196 CURLOPT_PROXYTYPE
, (long)CURLPROXY_SOCKS5_HOSTNAME
);
1197 else if (starts_with(curl_http_proxy
, "socks5"))
1198 curl_easy_setopt(result
,
1199 CURLOPT_PROXYTYPE
, (long)CURLPROXY_SOCKS5
);
1200 else if (starts_with(curl_http_proxy
, "socks4a"))
1201 curl_easy_setopt(result
,
1202 CURLOPT_PROXYTYPE
, (long)CURLPROXY_SOCKS4A
);
1203 else if (starts_with(curl_http_proxy
, "socks"))
1204 curl_easy_setopt(result
,
1205 CURLOPT_PROXYTYPE
, (long)CURLPROXY_SOCKS4
);
1206 else if (starts_with(curl_http_proxy
, "https")) {
1207 curl_easy_setopt(result
, CURLOPT_PROXYTYPE
, (long)CURLPROXY_HTTPS
);
1209 if (http_proxy_ssl_cert
)
1210 curl_easy_setopt(result
, CURLOPT_PROXY_SSLCERT
, http_proxy_ssl_cert
);
1212 if (http_proxy_ssl_key
)
1213 curl_easy_setopt(result
, CURLOPT_PROXY_SSLKEY
, http_proxy_ssl_key
);
1215 if (has_proxy_cert_password())
1216 curl_easy_setopt(result
, CURLOPT_PROXY_KEYPASSWD
, proxy_cert_auth
.password
);
1218 if (strstr(curl_http_proxy
, "://"))
1219 credential_from_url(&proxy_auth
, curl_http_proxy
);
1221 struct strbuf url
= STRBUF_INIT
;
1222 strbuf_addf(&url
, "http://%s", curl_http_proxy
);
1223 credential_from_url(&proxy_auth
, url
.buf
);
1224 strbuf_release(&url
);
1227 if (!proxy_auth
.host
)
1228 die("Invalid proxy URL '%s'", curl_http_proxy
);
1230 strbuf_addstr(&proxy
, proxy_auth
.host
);
1231 if (proxy_auth
.path
) {
1232 curl_version_info_data
*ver
= curl_version_info(CURLVERSION_NOW
);
1234 if (ver
->version_num
< 0x075400)
1235 die("libcurl 7.84 or later is required to support paths in proxy URLs");
1237 if (!starts_with(proxy_auth
.protocol
, "socks"))
1238 die("Invalid proxy URL '%s': only SOCKS proxies support paths",
1241 if (strcasecmp(proxy_auth
.host
, "localhost"))
1242 die("Invalid proxy URL '%s': host must be localhost if a path is present",
1245 strbuf_addch(&proxy
, '/');
1246 strbuf_add_percentencode(&proxy
, proxy_auth
.path
, 0);
1248 curl_easy_setopt(result
, CURLOPT_PROXY
, proxy
.buf
);
1249 strbuf_release(&proxy
);
1251 var_override(&curl_no_proxy
, getenv("NO_PROXY"));
1252 var_override(&curl_no_proxy
, getenv("no_proxy"));
1253 curl_easy_setopt(result
, CURLOPT_NOPROXY
, curl_no_proxy
);
1255 init_curl_proxy_auth(result
);
1257 curl_easy_setopt(result
, CURLOPT_TCP_KEEPALIVE
, 1L);
1259 if (curl_tcp_keepidle
> -1)
1260 curl_easy_setopt(result
, CURLOPT_TCP_KEEPIDLE
,
1262 if (curl_tcp_keepintvl
> -1)
1263 curl_easy_setopt(result
, CURLOPT_TCP_KEEPINTVL
,
1264 curl_tcp_keepintvl
);
1265 #ifdef GIT_CURL_HAVE_CURLOPT_TCP_KEEPCNT
1266 if (curl_tcp_keepcnt
> -1)
1267 curl_easy_setopt(result
, CURLOPT_TCP_KEEPCNT
, curl_tcp_keepcnt
);
1273 static void set_from_env(char **var
, const char *envname
)
1275 const char *val
= getenv(envname
);
1277 FREE_AND_NULL(*var
);
1278 *var
= xstrdup(val
);
1282 static void set_long_from_env(long *var
, const char *envname
)
1284 const char *val
= getenv(envname
);
1288 int saved_errno
= errno
;
1291 tmp
= strtol(val
, &endp
, 10);
1294 warning_errno(_("failed to parse %s"), envname
);
1295 else if (*endp
|| endp
== val
)
1296 warning(_("failed to parse %s"), envname
);
1300 errno
= saved_errno
;
1304 void http_init(struct remote
*remote
, const char *url
, int proactive_auth
)
1306 char *normalized_url
;
1307 struct urlmatch_config config
= URLMATCH_CONFIG_INIT
;
1309 config
.section
= "http";
1311 config
.collect_fn
= http_options
;
1312 config
.cascade_fn
= git_default_config
;
1315 http_is_verbose
= 0;
1316 normalized_url
= url_normalize(url
, &config
.url
);
1318 git_config(urlmatch_config_entry
, &config
);
1319 free(normalized_url
);
1320 string_list_clear(&config
.vars
, 1);
1322 if (http_ssl_backend
) {
1323 const curl_ssl_backend
**backends
;
1324 struct strbuf buf
= STRBUF_INIT
;
1327 switch (curl_global_sslset(-1, http_ssl_backend
, &backends
)) {
1328 case CURLSSLSET_UNKNOWN_BACKEND
:
1329 strbuf_addf(&buf
, _("Unsupported SSL backend '%s'. "
1330 "Supported SSL backends:"),
1332 for (i
= 0; backends
[i
]; i
++)
1333 strbuf_addf(&buf
, "\n\t%s", backends
[i
]->name
);
1335 case CURLSSLSET_NO_BACKENDS
:
1336 die(_("Could not set SSL backend to '%s': "
1337 "cURL was built without SSL backends"),
1339 case CURLSSLSET_TOO_LATE
:
1340 die(_("Could not set SSL backend to '%s': already set"),
1347 if (curl_global_init(CURL_GLOBAL_ALL
) != CURLE_OK
)
1348 die("curl_global_init failed");
1350 if (proactive_auth
&& http_proactive_auth
== PROACTIVE_AUTH_NONE
)
1351 http_proactive_auth
= PROACTIVE_AUTH_IF_CREDENTIALS
;
1353 if (remote
&& remote
->http_proxy
)
1354 curl_http_proxy
= xstrdup(remote
->http_proxy
);
1357 var_override(&http_proxy_authmethod
, remote
->http_proxy_authmethod
);
1359 pragma_header
= curl_slist_append(http_copy_default_headers(),
1360 "Pragma: no-cache");
1363 char *http_max_requests
= getenv("GIT_HTTP_MAX_REQUESTS");
1364 if (http_max_requests
)
1365 max_requests
= atoi(http_max_requests
);
1368 curlm
= curl_multi_init();
1370 die("curl_multi_init failed");
1372 if (getenv("GIT_SSL_NO_VERIFY"))
1373 curl_ssl_verify
= 0;
1375 set_from_env(&ssl_cert
, "GIT_SSL_CERT");
1376 set_from_env(&ssl_cert_type
, "GIT_SSL_CERT_TYPE");
1377 set_from_env(&ssl_key
, "GIT_SSL_KEY");
1378 set_from_env(&ssl_key_type
, "GIT_SSL_KEY_TYPE");
1379 set_from_env(&ssl_capath
, "GIT_SSL_CAPATH");
1380 set_from_env(&ssl_cainfo
, "GIT_SSL_CAINFO");
1382 set_from_env(&user_agent
, "GIT_HTTP_USER_AGENT");
1384 set_long_from_env(&curl_low_speed_limit
, "GIT_HTTP_LOW_SPEED_LIMIT");
1385 set_long_from_env(&curl_low_speed_time
, "GIT_HTTP_LOW_SPEED_TIME");
1387 if (curl_ssl_verify
== -1)
1388 curl_ssl_verify
= 1;
1390 curl_session_count
= 0;
1391 if (max_requests
< 1)
1392 max_requests
= DEFAULT_MAX_REQUESTS
;
1394 set_from_env(&http_proxy_ssl_cert
, "GIT_PROXY_SSL_CERT");
1395 set_from_env(&http_proxy_ssl_key
, "GIT_PROXY_SSL_KEY");
1396 set_from_env(&http_proxy_ssl_ca_info
, "GIT_PROXY_SSL_CAINFO");
1398 if (getenv("GIT_PROXY_SSL_CERT_PASSWORD_PROTECTED"))
1399 proxy_ssl_cert_password_required
= 1;
1401 if (getenv("GIT_CURL_FTP_NO_EPSV"))
1402 curl_ftp_no_epsv
= 1;
1405 credential_from_url(&http_auth
, url
);
1406 if (!ssl_cert_password_required
&&
1407 getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
1408 starts_with(url
, "https://"))
1409 ssl_cert_password_required
= 1;
1412 set_long_from_env(&curl_tcp_keepidle
, "GIT_TCP_KEEPIDLE");
1413 set_long_from_env(&curl_tcp_keepintvl
, "GIT_TCP_KEEPINTVL");
1414 set_long_from_env(&curl_tcp_keepcnt
, "GIT_TCP_KEEPCNT");
1416 curl_default
= get_curl_handle();
1419 void http_cleanup(void)
1421 struct active_request_slot
*slot
= active_queue_head
;
1423 while (slot
!= NULL
) {
1424 struct active_request_slot
*next
= slot
->next
;
1426 xmulti_remove_handle(slot
);
1427 curl_easy_cleanup(slot
->curl
);
1432 active_queue_head
= NULL
;
1434 curl_easy_cleanup(curl_default
);
1436 curl_multi_cleanup(curlm
);
1437 curl_global_cleanup();
1439 string_list_clear(&extra_http_headers
, 0);
1441 curl_slist_free_all(pragma_header
);
1442 pragma_header
= NULL
;
1444 curl_slist_free_all(host_resolutions
);
1445 host_resolutions
= NULL
;
1447 if (curl_http_proxy
) {
1448 free((void *)curl_http_proxy
);
1449 curl_http_proxy
= NULL
;
1452 if (proxy_auth
.password
) {
1453 memset(proxy_auth
.password
, 0, strlen(proxy_auth
.password
));
1454 FREE_AND_NULL(proxy_auth
.password
);
1457 free((void *)curl_proxyuserpwd
);
1458 curl_proxyuserpwd
= NULL
;
1460 free((void *)http_proxy_authmethod
);
1461 http_proxy_authmethod
= NULL
;
1463 if (cert_auth
.password
) {
1464 memset(cert_auth
.password
, 0, strlen(cert_auth
.password
));
1465 FREE_AND_NULL(cert_auth
.password
);
1467 ssl_cert_password_required
= 0;
1469 if (proxy_cert_auth
.password
) {
1470 memset(proxy_cert_auth
.password
, 0, strlen(proxy_cert_auth
.password
));
1471 FREE_AND_NULL(proxy_cert_auth
.password
);
1473 proxy_ssl_cert_password_required
= 0;
1475 FREE_AND_NULL(cached_accept_language
);
1478 struct active_request_slot
*get_active_slot(void)
1480 struct active_request_slot
*slot
= active_queue_head
;
1481 struct active_request_slot
*newslot
;
1485 /* Wait for a slot to open up if the queue is full */
1486 while (active_requests
>= max_requests
) {
1487 curl_multi_perform(curlm
, &num_transfers
);
1488 if (num_transfers
< active_requests
)
1489 process_curl_messages();
1492 while (slot
!= NULL
&& slot
->in_use
)
1496 newslot
= xmalloc(sizeof(*newslot
));
1497 newslot
->curl
= NULL
;
1498 newslot
->in_use
= 0;
1499 newslot
->next
= NULL
;
1501 slot
= active_queue_head
;
1503 active_queue_head
= newslot
;
1505 while (slot
->next
!= NULL
)
1507 slot
->next
= newslot
;
1513 slot
->curl
= curl_easy_duphandle(curl_default
);
1514 curl_session_count
++;
1519 slot
->results
= NULL
;
1520 slot
->finished
= NULL
;
1521 slot
->callback_data
= NULL
;
1522 slot
->callback_func
= NULL
;
1524 if (curl_cookie_file
&& !strcmp(curl_cookie_file
, "-")) {
1525 warning(_("refusing to read cookies from http.cookiefile '-'"));
1526 FREE_AND_NULL(curl_cookie_file
);
1528 curl_easy_setopt(slot
->curl
, CURLOPT_COOKIEFILE
, curl_cookie_file
);
1529 if (curl_save_cookies
&& (!curl_cookie_file
|| !curl_cookie_file
[0])) {
1530 curl_save_cookies
= 0;
1531 warning(_("ignoring http.savecookies for empty http.cookiefile"));
1533 if (curl_save_cookies
)
1534 curl_easy_setopt(slot
->curl
, CURLOPT_COOKIEJAR
, curl_cookie_file
);
1535 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, pragma_header
);
1536 curl_easy_setopt(slot
->curl
, CURLOPT_RESOLVE
, host_resolutions
);
1537 curl_easy_setopt(slot
->curl
, CURLOPT_ERRORBUFFER
, curl_errorstr
);
1538 curl_easy_setopt(slot
->curl
, CURLOPT_CUSTOMREQUEST
, NULL
);
1539 curl_easy_setopt(slot
->curl
, CURLOPT_READFUNCTION
, NULL
);
1540 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, NULL
);
1541 curl_easy_setopt(slot
->curl
, CURLOPT_POSTFIELDS
, NULL
);
1542 curl_easy_setopt(slot
->curl
, CURLOPT_POSTFIELDSIZE
, -1L);
1543 curl_easy_setopt(slot
->curl
, CURLOPT_UPLOAD
, 0L);
1544 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPGET
, 1L);
1545 curl_easy_setopt(slot
->curl
, CURLOPT_FAILONERROR
, 1L);
1546 curl_easy_setopt(slot
->curl
, CURLOPT_RANGE
, NULL
);
1549 * Default following to off unless "ALWAYS" is configured; this gives
1550 * callers a sane starting point, and they can tweak for individual
1551 * HTTP_FOLLOW_* cases themselves.
1553 if (http_follow_config
== HTTP_FOLLOW_ALWAYS
)
1554 curl_easy_setopt(slot
->curl
, CURLOPT_FOLLOWLOCATION
, 1L);
1556 curl_easy_setopt(slot
->curl
, CURLOPT_FOLLOWLOCATION
, 0L);
1558 curl_easy_setopt(slot
->curl
, CURLOPT_IPRESOLVE
, git_curl_ipresolve
);
1559 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPAUTH
, http_auth_methods
);
1560 if (http_auth
.password
|| http_auth
.credential
|| curl_empty_auth_enabled())
1561 init_curl_http_auth(slot
->curl
);
1566 int start_active_slot(struct active_request_slot
*slot
)
1568 CURLMcode curlm_result
= curl_multi_add_handle(curlm
, slot
->curl
);
1571 if (curlm_result
!= CURLM_OK
&&
1572 curlm_result
!= CURLM_CALL_MULTI_PERFORM
) {
1573 warning("curl_multi_add_handle failed: %s",
1574 curl_multi_strerror(curlm_result
));
1581 * We know there must be something to do, since we just added
1584 curl_multi_perform(curlm
, &num_transfers
);
1590 int (*fill
)(void *);
1591 struct fill_chain
*next
;
1594 static struct fill_chain
*fill_cfg
;
1596 void add_fill_function(void *data
, int (*fill
)(void *))
1598 struct fill_chain
*new_fill
= xmalloc(sizeof(*new_fill
));
1599 struct fill_chain
**linkp
= &fill_cfg
;
1600 new_fill
->data
= data
;
1601 new_fill
->fill
= fill
;
1602 new_fill
->next
= NULL
;
1604 linkp
= &(*linkp
)->next
;
1608 void fill_active_slots(void)
1610 struct active_request_slot
*slot
= active_queue_head
;
1612 while (active_requests
< max_requests
) {
1613 struct fill_chain
*fill
;
1614 for (fill
= fill_cfg
; fill
; fill
= fill
->next
)
1615 if (fill
->fill(fill
->data
))
1622 while (slot
!= NULL
) {
1623 if (!slot
->in_use
&& slot
->curl
!= NULL
1624 && curl_session_count
> min_curl_sessions
) {
1625 curl_easy_cleanup(slot
->curl
);
1627 curl_session_count
--;
1633 void step_active_slots(void)
1636 CURLMcode curlm_result
;
1639 curlm_result
= curl_multi_perform(curlm
, &num_transfers
);
1640 } while (curlm_result
== CURLM_CALL_MULTI_PERFORM
);
1641 if (num_transfers
< active_requests
) {
1642 process_curl_messages();
1643 fill_active_slots();
1647 void run_active_slot(struct active_request_slot
*slot
)
1653 struct timeval select_timeout
;
1656 slot
->finished
= &finished
;
1658 step_active_slots();
1662 curl_multi_timeout(curlm
, &curl_timeout
);
1663 if (curl_timeout
== 0) {
1665 } else if (curl_timeout
== -1) {
1666 select_timeout
.tv_sec
= 0;
1667 select_timeout
.tv_usec
= 50000;
1669 select_timeout
.tv_sec
= curl_timeout
/ 1000;
1670 select_timeout
.tv_usec
= (curl_timeout
% 1000) * 1000;
1677 curl_multi_fdset(curlm
, &readfds
, &writefds
, &excfds
, &max_fd
);
1680 * It can happen that curl_multi_timeout returns a pathologically
1681 * long timeout when curl_multi_fdset returns no file descriptors
1682 * to read. See commit message for more details.
1685 (select_timeout
.tv_sec
> 0 ||
1686 select_timeout
.tv_usec
> 50000)) {
1687 select_timeout
.tv_sec
= 0;
1688 select_timeout
.tv_usec
= 50000;
1691 select(max_fd
+1, &readfds
, &writefds
, &excfds
, &select_timeout
);
1696 * The value of slot->finished we set before the loop was used
1697 * to set our "finished" variable when our request completed.
1699 * 1. The slot may not have been reused for another request
1700 * yet, in which case it still has &finished.
1702 * 2. The slot may already be in-use to serve another request,
1703 * which can further be divided into two cases:
1705 * (a) If call run_active_slot() hasn't been called for that
1706 * other request, slot->finished would have been cleared
1707 * by get_active_slot() and has NULL.
1709 * (b) If the request did call run_active_slot(), then the
1710 * call would have updated slot->finished at the beginning
1711 * of this function, and with the clearing of the member
1712 * below, we would find that slot->finished is now NULL.
1714 * In all cases, slot->finished has no useful information to
1715 * anybody at this point. Some compilers warn us for
1716 * attempting to smuggle a pointer that is about to become
1717 * invalid, i.e. &finished. We clear it here to assure them.
1719 slot
->finished
= NULL
;
1722 static void release_active_slot(struct active_request_slot
*slot
)
1724 closedown_active_slot(slot
);
1726 xmulti_remove_handle(slot
);
1727 if (curl_session_count
> min_curl_sessions
) {
1728 curl_easy_cleanup(slot
->curl
);
1730 curl_session_count
--;
1733 fill_active_slots();
1736 void finish_all_active_slots(void)
1738 struct active_request_slot
*slot
= active_queue_head
;
1740 while (slot
!= NULL
)
1742 run_active_slot(slot
);
1743 slot
= active_queue_head
;
1749 /* Helpers for modifying and creating URLs */
1750 static inline int needs_quote(int ch
)
1752 if (((ch
>= 'A') && (ch
<= 'Z'))
1753 || ((ch
>= 'a') && (ch
<= 'z'))
1754 || ((ch
>= '0') && (ch
<= '9'))
1762 static char *quote_ref_url(const char *base
, const char *ref
)
1764 struct strbuf buf
= STRBUF_INIT
;
1768 end_url_with_slash(&buf
, base
);
1770 for (cp
= ref
; (ch
= *cp
) != 0; cp
++)
1771 if (needs_quote(ch
))
1772 strbuf_addf(&buf
, "%%%02x", ch
);
1774 strbuf_addch(&buf
, *cp
);
1776 return strbuf_detach(&buf
, NULL
);
1779 void append_remote_object_url(struct strbuf
*buf
, const char *url
,
1781 int only_two_digit_prefix
)
1783 end_url_with_slash(buf
, url
);
1785 strbuf_addf(buf
, "objects/%.*s/", 2, hex
);
1786 if (!only_two_digit_prefix
)
1787 strbuf_addstr(buf
, hex
+ 2);
1790 char *get_remote_object_url(const char *url
, const char *hex
,
1791 int only_two_digit_prefix
)
1793 struct strbuf buf
= STRBUF_INIT
;
1794 append_remote_object_url(&buf
, url
, hex
, only_two_digit_prefix
);
1795 return strbuf_detach(&buf
, NULL
);
1798 void normalize_curl_result(CURLcode
*result
, long http_code
,
1799 char *errorstr
, size_t errorlen
)
1802 * If we see a failing http code with CURLE_OK, we have turned off
1803 * FAILONERROR (to keep the server's custom error response), and should
1804 * translate the code into failure here.
1806 * Likewise, if we see a redirect (30x code), that means we turned off
1807 * redirect-following, and we should treat the result as an error.
1809 if (*result
== CURLE_OK
&& http_code
>= 300) {
1810 *result
= CURLE_HTTP_RETURNED_ERROR
;
1812 * Normally curl will already have put the "reason phrase"
1813 * from the server into curl_errorstr; unfortunately without
1814 * FAILONERROR it is lost, so we can give only the numeric
1817 xsnprintf(errorstr
, errorlen
,
1818 "The requested URL returned error: %ld",
1823 static int handle_curl_result(struct slot_results
*results
)
1825 normalize_curl_result(&results
->curl_result
, results
->http_code
,
1826 curl_errorstr
, sizeof(curl_errorstr
));
1828 if (results
->curl_result
== CURLE_OK
) {
1829 credential_approve(the_repository
, &http_auth
);
1830 credential_approve(the_repository
, &proxy_auth
);
1831 credential_approve(the_repository
, &cert_auth
);
1833 } else if (results
->curl_result
== CURLE_SSL_CERTPROBLEM
) {
1835 * We can't tell from here whether it's a bad path, bad
1836 * certificate, bad password, or something else wrong
1837 * with the certificate. So we reject the credential to
1838 * avoid caching or saving a bad password.
1840 credential_reject(the_repository
, &cert_auth
);
1842 } else if (results
->curl_result
== CURLE_SSL_PINNEDPUBKEYNOTMATCH
) {
1843 return HTTP_NOMATCHPUBLICKEY
;
1844 } else if (missing_target(results
))
1845 return HTTP_MISSING_TARGET
;
1846 else if (results
->http_code
== 401) {
1847 if ((http_auth
.username
&& http_auth
.password
) ||\
1848 (http_auth
.authtype
&& http_auth
.credential
)) {
1849 if (http_auth
.multistage
) {
1850 credential_clear_secrets(&http_auth
);
1853 credential_reject(the_repository
, &http_auth
);
1854 if (always_auth_proactively())
1855 http_proactive_auth
= PROACTIVE_AUTH_NONE
;
1858 http_auth_methods
&= ~CURLAUTH_GSSNEGOTIATE
;
1859 if (results
->auth_avail
) {
1860 http_auth_methods
&= results
->auth_avail
;
1861 http_auth_methods_restricted
= 1;
1866 if (results
->http_connectcode
== 407)
1867 credential_reject(the_repository
, &proxy_auth
);
1868 if (!curl_errorstr
[0])
1869 strlcpy(curl_errorstr
,
1870 curl_easy_strerror(results
->curl_result
),
1871 sizeof(curl_errorstr
));
1876 int run_one_slot(struct active_request_slot
*slot
,
1877 struct slot_results
*results
)
1879 slot
->results
= results
;
1880 if (!start_active_slot(slot
)) {
1881 xsnprintf(curl_errorstr
, sizeof(curl_errorstr
),
1882 "failed to start HTTP request");
1883 return HTTP_START_FAILED
;
1886 run_active_slot(slot
);
1887 return handle_curl_result(results
);
1890 struct curl_slist
*http_copy_default_headers(void)
1892 struct curl_slist
*headers
= NULL
;
1893 const struct string_list_item
*item
;
1895 for_each_string_list_item(item
, &extra_http_headers
)
1896 headers
= curl_slist_append(headers
, item
->string
);
1901 static CURLcode
curlinfo_strbuf(CURL
*curl
, CURLINFO info
, struct strbuf
*buf
)
1907 ret
= curl_easy_getinfo(curl
, info
, &ptr
);
1909 strbuf_addstr(buf
, ptr
);
1914 * Check for and extract a content-type parameter. "raw"
1915 * should be positioned at the start of the potential
1916 * parameter, with any whitespace already removed.
1918 * "name" is the name of the parameter. The value is appended
1921 static int extract_param(const char *raw
, const char *name
,
1924 size_t len
= strlen(name
);
1926 if (strncasecmp(raw
, name
, len
))
1934 while (*raw
&& !isspace(*raw
) && *raw
!= ';')
1935 strbuf_addch(out
, *raw
++);
1940 * Extract a normalized version of the content type, with any
1941 * spaces suppressed, all letters lowercased, and no trailing ";"
1944 * Note that we will silently remove even invalid whitespace. For
1945 * example, "text / plain" is specifically forbidden by RFC 2616,
1946 * but "text/plain" is the only reasonable output, and this keeps
1949 * If the "charset" argument is not NULL, store the value of any
1950 * charset parameter there.
1953 * "TEXT/PLAIN; charset=utf-8" -> "text/plain", "utf-8"
1954 * "text / plain" -> "text/plain"
1956 static void extract_content_type(struct strbuf
*raw
, struct strbuf
*type
,
1957 struct strbuf
*charset
)
1962 strbuf_grow(type
, raw
->len
);
1963 for (p
= raw
->buf
; *p
; p
++) {
1970 strbuf_addch(type
, tolower(*p
));
1976 strbuf_reset(charset
);
1978 while (isspace(*p
) || *p
== ';')
1980 if (!extract_param(p
, "charset", charset
))
1982 while (*p
&& !isspace(*p
))
1986 if (!charset
->len
&& starts_with(type
->buf
, "text/"))
1987 strbuf_addstr(charset
, "ISO-8859-1");
1990 static void write_accept_language(struct strbuf
*buf
)
1993 * MAX_DECIMAL_PLACES must not be larger than 3. If it is larger than
1994 * that, q-value will be smaller than 0.001, the minimum q-value the
1995 * HTTP specification allows. See
1996 * https://datatracker.ietf.org/doc/html/rfc7231#section-5.3.1 for q-value.
1998 const int MAX_DECIMAL_PLACES
= 3;
1999 const int MAX_LANGUAGE_TAGS
= 1000;
2000 const int MAX_ACCEPT_LANGUAGE_HEADER_SIZE
= 4000;
2001 char **language_tags
= NULL
;
2003 const char *s
= get_preferred_languages();
2005 struct strbuf tag
= STRBUF_INIT
;
2007 /* Don't add Accept-Language header if no language is preferred. */
2012 * Split the colon-separated string of preferred languages into
2013 * language_tags array.
2016 /* collect language tag */
2017 for (; *s
&& (isalnum(*s
) || *s
== '_'); s
++)
2018 strbuf_addch(&tag
, *s
== '_' ? '-' : *s
);
2020 /* skip .codeset, @modifier and any other unnecessary parts */
2021 while (*s
&& *s
!= ':')
2026 REALLOC_ARRAY(language_tags
, num_langs
);
2027 language_tags
[num_langs
- 1] = strbuf_detach(&tag
, NULL
);
2028 if (num_langs
>= MAX_LANGUAGE_TAGS
- 1) /* -1 for '*' */
2033 /* write Accept-Language header into buf */
2035 int last_buf_len
= 0;
2041 REALLOC_ARRAY(language_tags
, num_langs
+ 1);
2042 language_tags
[num_langs
++] = xstrdup("*");
2044 /* compute decimal_places */
2045 for (max_q
= 1, decimal_places
= 0;
2046 max_q
< num_langs
&& decimal_places
<= MAX_DECIMAL_PLACES
;
2047 decimal_places
++, max_q
*= 10)
2050 xsnprintf(q_format
, sizeof(q_format
), ";q=0.%%0%dd", decimal_places
);
2052 strbuf_addstr(buf
, "Accept-Language: ");
2054 for (i
= 0; i
< num_langs
; i
++) {
2056 strbuf_addstr(buf
, ", ");
2058 strbuf_addstr(buf
, language_tags
[i
]);
2061 strbuf_addf(buf
, q_format
, max_q
- i
);
2063 if (buf
->len
> MAX_ACCEPT_LANGUAGE_HEADER_SIZE
) {
2064 strbuf_remove(buf
, last_buf_len
, buf
->len
- last_buf_len
);
2068 last_buf_len
= buf
->len
;
2072 for (i
= 0; i
< num_langs
; i
++)
2073 free(language_tags
[i
]);
2074 free(language_tags
);
2078 * Get an Accept-Language header which indicates user's preferred languages.
2082 * LANGUAGE=ko:en -> "Accept-Language: ko, en; q=0.9, *; q=0.1"
2083 * LANGUAGE=ko_KR.UTF-8:sr@latin -> "Accept-Language: ko-KR, sr; q=0.9, *; q=0.1"
2084 * LANGUAGE=ko LANG=en_US.UTF-8 -> "Accept-Language: ko, *; q=0.1"
2085 * LANGUAGE= LANG=en_US.UTF-8 -> "Accept-Language: en-US, *; q=0.1"
2086 * LANGUAGE= LANG=C -> ""
2088 const char *http_get_accept_language_header(void)
2090 if (!cached_accept_language
) {
2091 struct strbuf buf
= STRBUF_INIT
;
2092 write_accept_language(&buf
);
2094 cached_accept_language
= strbuf_detach(&buf
, NULL
);
2097 return cached_accept_language
;
2100 static void http_opt_request_remainder(CURL
*curl
, off_t pos
)
2103 xsnprintf(buf
, sizeof(buf
), "%"PRIuMAX
"-", (uintmax_t)pos
);
2104 curl_easy_setopt(curl
, CURLOPT_RANGE
, buf
);
2107 /* http_request() targets */
2108 #define HTTP_REQUEST_STRBUF 0
2109 #define HTTP_REQUEST_FILE 1
2111 static int http_request(const char *url
,
2112 void *result
, int target
,
2113 const struct http_get_options
*options
)
2115 struct active_request_slot
*slot
;
2116 struct slot_results results
;
2117 struct curl_slist
*headers
= http_copy_default_headers();
2118 struct strbuf buf
= STRBUF_INIT
;
2119 const char *accept_language
;
2122 slot
= get_active_slot();
2123 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPGET
, 1L);
2126 curl_easy_setopt(slot
->curl
, CURLOPT_NOBODY
, 1L);
2128 curl_easy_setopt(slot
->curl
, CURLOPT_NOBODY
, 0L);
2129 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEDATA
, result
);
2131 if (target
== HTTP_REQUEST_FILE
) {
2132 off_t posn
= ftello(result
);
2133 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
,
2136 http_opt_request_remainder(slot
->curl
, posn
);
2138 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
,
2142 curl_easy_setopt(slot
->curl
, CURLOPT_HEADERFUNCTION
, fwrite_wwwauth
);
2144 accept_language
= http_get_accept_language_header();
2146 if (accept_language
)
2147 headers
= curl_slist_append(headers
, accept_language
);
2149 strbuf_addstr(&buf
, "Pragma:");
2150 if (options
&& options
->no_cache
)
2151 strbuf_addstr(&buf
, " no-cache");
2152 if (options
&& options
->initial_request
&&
2153 http_follow_config
== HTTP_FOLLOW_INITIAL
)
2154 curl_easy_setopt(slot
->curl
, CURLOPT_FOLLOWLOCATION
, 1L);
2156 headers
= curl_slist_append(headers
, buf
.buf
);
2158 /* Add additional headers here */
2159 if (options
&& options
->extra_headers
) {
2160 const struct string_list_item
*item
;
2161 if (options
&& options
->extra_headers
) {
2162 for_each_string_list_item(item
, options
->extra_headers
) {
2163 headers
= curl_slist_append(headers
, item
->string
);
2168 headers
= http_append_auth_header(&http_auth
, headers
);
2170 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
);
2171 curl_easy_setopt(slot
->curl
, CURLOPT_HTTPHEADER
, headers
);
2172 curl_easy_setopt(slot
->curl
, CURLOPT_ENCODING
, "");
2173 curl_easy_setopt(slot
->curl
, CURLOPT_FAILONERROR
, 0L);
2175 ret
= run_one_slot(slot
, &results
);
2177 if (options
&& options
->content_type
) {
2178 struct strbuf raw
= STRBUF_INIT
;
2179 curlinfo_strbuf(slot
->curl
, CURLINFO_CONTENT_TYPE
, &raw
);
2180 extract_content_type(&raw
, options
->content_type
,
2182 strbuf_release(&raw
);
2185 if (options
&& options
->effective_url
)
2186 curlinfo_strbuf(slot
->curl
, CURLINFO_EFFECTIVE_URL
,
2187 options
->effective_url
);
2189 curl_slist_free_all(headers
);
2190 strbuf_release(&buf
);
2196 * Update the "base" url to a more appropriate value, as deduced by
2197 * redirects seen when requesting a URL starting with "url".
2199 * The "asked" parameter is a URL that we asked curl to access, and must begin
2202 * The "got" parameter is the URL that curl reported to us as where we ended
2205 * Returns 1 if we updated the base url, 0 otherwise.
2207 * Our basic strategy is to compare "base" and "asked" to find the bits
2208 * specific to our request. We then strip those bits off of "got" to yield the
2209 * new base. So for example, if our base is "http://example.com/foo.git",
2210 * and we ask for "http://example.com/foo.git/info/refs", we might end up
2211 * with "https://other.example.com/foo.git/info/refs". We would want the
2212 * new URL to become "https://other.example.com/foo.git".
2214 * Note that this assumes a sane redirect scheme. It's entirely possible
2215 * in the example above to end up at a URL that does not even end in
2216 * "info/refs". In such a case we die. There's not much we can do, such a
2217 * scheme is unlikely to represent a real git repository, and failing to
2218 * rewrite the base opens options for malicious redirects to do funny things.
2220 static int update_url_from_redirect(struct strbuf
*base
,
2222 const struct strbuf
*got
)
2227 if (!strcmp(asked
, got
->buf
))
2230 if (!skip_prefix(asked
, base
->buf
, &tail
))
2231 BUG("update_url_from_redirect: %s is not a superset of %s",
2235 if (!strip_suffix_mem(got
->buf
, &new_len
, tail
))
2236 die(_("unable to update url base from redirection:\n"
2242 strbuf_add(base
, got
->buf
, new_len
);
2247 static int http_request_reauth(const char *url
,
2248 void *result
, int target
,
2249 struct http_get_options
*options
)
2254 if (always_auth_proactively())
2255 credential_fill(the_repository
, &http_auth
, 1);
2257 ret
= http_request(url
, result
, target
, options
);
2259 if (ret
!= HTTP_OK
&& ret
!= HTTP_REAUTH
)
2262 if (options
&& options
->effective_url
&& options
->base_url
) {
2263 if (update_url_from_redirect(options
->base_url
,
2264 url
, options
->effective_url
)) {
2265 credential_from_url(&http_auth
, options
->base_url
->buf
);
2266 url
= options
->effective_url
->buf
;
2270 while (ret
== HTTP_REAUTH
&& --i
) {
2272 * The previous request may have put cruft into our output stream; we
2273 * should clear it out before making our next request.
2276 case HTTP_REQUEST_STRBUF
:
2277 strbuf_reset(result
);
2279 case HTTP_REQUEST_FILE
: {
2282 error_errno("unable to flush a file");
2283 return HTTP_START_FAILED
;
2286 if (ftruncate(fileno(f
), 0) < 0) {
2287 error_errno("unable to truncate a file");
2288 return HTTP_START_FAILED
;
2293 BUG("Unknown http_request target");
2296 credential_fill(the_repository
, &http_auth
, 1);
2298 ret
= http_request(url
, result
, target
, options
);
2303 int http_get_strbuf(const char *url
,
2304 struct strbuf
*result
,
2305 struct http_get_options
*options
)
2307 return http_request_reauth(url
, result
, HTTP_REQUEST_STRBUF
, options
);
2311 * Downloads a URL and stores the result in the given file.
2313 * If a previous interrupted download is detected (i.e. a previous temporary
2314 * file is still around) the download is resumed.
2316 int http_get_file(const char *url
, const char *filename
,
2317 struct http_get_options
*options
)
2320 struct strbuf tmpfile
= STRBUF_INIT
;
2323 strbuf_addf(&tmpfile
, "%s.temp", filename
);
2324 result
= fopen(tmpfile
.buf
, "a");
2326 error("Unable to open local file %s", tmpfile
.buf
);
2331 ret
= http_request_reauth(url
, result
, HTTP_REQUEST_FILE
, options
);
2334 if (ret
== HTTP_OK
&& finalize_object_file(tmpfile
.buf
, filename
))
2337 strbuf_release(&tmpfile
);
2341 int http_fetch_ref(const char *base
, struct ref
*ref
)
2343 struct http_get_options options
= {0};
2345 struct strbuf buffer
= STRBUF_INIT
;
2348 options
.no_cache
= 1;
2350 url
= quote_ref_url(base
, ref
->name
);
2351 if (http_get_strbuf(url
, &buffer
, &options
) == HTTP_OK
) {
2352 strbuf_rtrim(&buffer
);
2353 if (buffer
.len
== the_hash_algo
->hexsz
)
2354 ret
= get_oid_hex(buffer
.buf
, &ref
->old_oid
);
2355 else if (starts_with(buffer
.buf
, "ref: ")) {
2356 ref
->symref
= xstrdup(buffer
.buf
+ 5);
2361 strbuf_release(&buffer
);
2366 /* Helpers for fetching packs */
2367 static char *fetch_pack_index(unsigned char *hash
, const char *base_url
)
2370 struct strbuf buf
= STRBUF_INIT
;
2372 if (http_is_verbose
)
2373 fprintf(stderr
, "Getting index for pack %s\n", hash_to_hex(hash
));
2375 end_url_with_slash(&buf
, base_url
);
2376 strbuf_addf(&buf
, "objects/pack/pack-%s.idx", hash_to_hex(hash
));
2377 url
= strbuf_detach(&buf
, NULL
);
2380 * Don't put this into packs/, since it's just temporary and we don't
2381 * want to confuse it with our local .idx files. We'll generate our
2382 * own index if we choose to download the matching packfile.
2384 * It's tempting to use xmks_tempfile() here, but it's important that
2385 * the file not exist, otherwise http_get_file() complains. So we
2386 * create a filename that should be unique, and then just register it
2387 * as a tempfile so that it will get cleaned up on exit.
2389 * In theory we could hold on to the tempfile and delete these as soon
2390 * as we download the matching pack, but it would take a bit of
2391 * refactoring. Leaving them until the process ends is probably OK.
2393 tmp
= xstrfmt("%s/tmp_pack_%s.idx",
2394 repo_get_object_directory(the_repository
),
2396 register_tempfile(tmp
);
2398 if (http_get_file(url
, tmp
, NULL
) != HTTP_OK
) {
2399 error("Unable to get pack index %s", url
);
2407 static int fetch_and_setup_pack_index(struct packed_git
**packs_head
,
2408 unsigned char *sha1
, const char *base_url
)
2410 struct packed_git
*new_pack
, *p
;
2411 char *tmp_idx
= NULL
;
2415 * If we already have the pack locally, no need to fetch its index or
2416 * even add it to list; we already have all of its objects.
2418 for (p
= get_all_packs(the_repository
); p
; p
= p
->next
) {
2419 if (hasheq(p
->hash
, sha1
, the_repository
->hash_algo
))
2423 tmp_idx
= fetch_pack_index(sha1
, base_url
);
2427 new_pack
= parse_pack_index(the_repository
, sha1
, tmp_idx
);
2432 return -1; /* parse_pack_index() already issued error message */
2435 ret
= verify_pack_index(new_pack
);
2437 close_pack_index(new_pack
);
2442 new_pack
->next
= *packs_head
;
2443 *packs_head
= new_pack
;
2447 int http_get_info_packs(const char *base_url
, struct packed_git
**packs_head
)
2449 struct http_get_options options
= {0};
2453 struct strbuf buf
= STRBUF_INIT
;
2454 struct object_id oid
;
2456 end_url_with_slash(&buf
, base_url
);
2457 strbuf_addstr(&buf
, "objects/info/packs");
2458 url
= strbuf_detach(&buf
, NULL
);
2460 options
.no_cache
= 1;
2461 ret
= http_get_strbuf(url
, &buf
, &options
);
2467 if (skip_prefix(data
, "P pack-", &data
) &&
2468 !parse_oid_hex(data
, &oid
, &data
) &&
2469 skip_prefix(data
, ".pack", &data
) &&
2470 (*data
== '\n' || *data
== '\0')) {
2471 fetch_and_setup_pack_index(packs_head
, oid
.hash
, base_url
);
2473 data
= strchrnul(data
, '\n');
2476 data
++; /* skip past newline */
2481 strbuf_release(&buf
);
2485 void release_http_pack_request(struct http_pack_request
*preq
)
2487 if (preq
->packfile
) {
2488 fclose(preq
->packfile
);
2489 preq
->packfile
= NULL
;
2492 strbuf_release(&preq
->tmpfile
);
2493 curl_slist_free_all(preq
->headers
);
2498 static const char *default_index_pack_args
[] =
2499 {"index-pack", "--stdin", NULL
};
2501 int finish_http_pack_request(struct http_pack_request
*preq
)
2503 struct child_process ip
= CHILD_PROCESS_INIT
;
2507 fclose(preq
->packfile
);
2508 preq
->packfile
= NULL
;
2510 tmpfile_fd
= xopen(preq
->tmpfile
.buf
, O_RDONLY
);
2514 strvec_pushv(&ip
.args
, preq
->index_pack_args
?
2515 preq
->index_pack_args
:
2516 default_index_pack_args
);
2518 if (preq
->preserve_index_pack_stdout
)
2523 if (run_command(&ip
)) {
2530 unlink(preq
->tmpfile
.buf
);
2534 void http_install_packfile(struct packed_git
*p
,
2535 struct packed_git
**list_to_remove_from
)
2537 struct packed_git
**lst
= list_to_remove_from
;
2540 lst
= &((*lst
)->next
);
2541 *lst
= (*lst
)->next
;
2543 install_packed_git(the_repository
, p
);
2546 struct http_pack_request
*new_http_pack_request(
2547 const unsigned char *packed_git_hash
, const char *base_url
) {
2549 struct strbuf buf
= STRBUF_INIT
;
2551 end_url_with_slash(&buf
, base_url
);
2552 strbuf_addf(&buf
, "objects/pack/pack-%s.pack",
2553 hash_to_hex(packed_git_hash
));
2554 return new_direct_http_pack_request(packed_git_hash
,
2555 strbuf_detach(&buf
, NULL
));
2558 struct http_pack_request
*new_direct_http_pack_request(
2559 const unsigned char *packed_git_hash
, char *url
)
2561 off_t prev_posn
= 0;
2562 struct http_pack_request
*preq
;
2564 CALLOC_ARRAY(preq
, 1);
2565 strbuf_init(&preq
->tmpfile
, 0);
2569 odb_pack_name(the_repository
, &preq
->tmpfile
, packed_git_hash
, "pack");
2570 strbuf_addstr(&preq
->tmpfile
, ".temp");
2571 preq
->packfile
= fopen(preq
->tmpfile
.buf
, "a");
2572 if (!preq
->packfile
) {
2573 error("Unable to open local file %s for pack",
2578 preq
->slot
= get_active_slot();
2579 preq
->headers
= object_request_headers();
2580 curl_easy_setopt(preq
->slot
->curl
, CURLOPT_WRITEDATA
, preq
->packfile
);
2581 curl_easy_setopt(preq
->slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite
);
2582 curl_easy_setopt(preq
->slot
->curl
, CURLOPT_URL
, preq
->url
);
2583 curl_easy_setopt(preq
->slot
->curl
, CURLOPT_HTTPHEADER
, preq
->headers
);
2586 * If there is data present from a previous transfer attempt,
2587 * resume where it left off
2589 prev_posn
= ftello(preq
->packfile
);
2591 if (http_is_verbose
)
2593 "Resuming fetch of pack %s at byte %"PRIuMAX
"\n",
2594 hash_to_hex(packed_git_hash
),
2595 (uintmax_t)prev_posn
);
2596 http_opt_request_remainder(preq
->slot
->curl
, prev_posn
);
2602 strbuf_release(&preq
->tmpfile
);
2608 /* Helpers for fetching objects (loose) */
2609 static size_t fwrite_sha1_file(char *ptr
, size_t eltsize
, size_t nmemb
,
2612 unsigned char expn
[4096];
2613 size_t size
= eltsize
* nmemb
;
2615 struct http_object_request
*freq
= data
;
2616 struct active_request_slot
*slot
= freq
->slot
;
2619 CURLcode c
= curl_easy_getinfo(slot
->curl
, CURLINFO_HTTP_CODE
,
2622 BUG("curl_easy_getinfo for HTTP code failed: %s",
2623 curl_easy_strerror(c
));
2624 if (slot
->http_code
>= 300)
2629 ssize_t retval
= xwrite(freq
->localfile
,
2630 (char *) ptr
+ posn
, size
- posn
);
2632 return posn
/ eltsize
;
2634 } while (posn
< size
);
2636 freq
->stream
.avail_in
= size
;
2637 freq
->stream
.next_in
= (void *)ptr
;
2639 freq
->stream
.next_out
= expn
;
2640 freq
->stream
.avail_out
= sizeof(expn
);
2641 freq
->zret
= git_inflate(&freq
->stream
, Z_SYNC_FLUSH
);
2642 git_hash_update(&freq
->c
, expn
,
2643 sizeof(expn
) - freq
->stream
.avail_out
);
2644 } while (freq
->stream
.avail_in
&& freq
->zret
== Z_OK
);
2648 struct http_object_request
*new_http_object_request(const char *base_url
,
2649 const struct object_id
*oid
)
2651 char *hex
= oid_to_hex(oid
);
2652 struct strbuf filename
= STRBUF_INIT
;
2653 struct strbuf prevfile
= STRBUF_INIT
;
2655 char prev_buf
[PREV_BUF_SIZE
];
2656 ssize_t prev_read
= 0;
2657 off_t prev_posn
= 0;
2658 struct http_object_request
*freq
;
2660 CALLOC_ARRAY(freq
, 1);
2661 strbuf_init(&freq
->tmpfile
, 0);
2662 oidcpy(&freq
->oid
, oid
);
2663 freq
->localfile
= -1;
2665 odb_loose_path(the_repository
->objects
->odb
, &filename
, oid
);
2666 strbuf_addf(&freq
->tmpfile
, "%s.temp", filename
.buf
);
2668 strbuf_addf(&prevfile
, "%s.prev", filename
.buf
);
2669 unlink_or_warn(prevfile
.buf
);
2670 rename(freq
->tmpfile
.buf
, prevfile
.buf
);
2671 unlink_or_warn(freq
->tmpfile
.buf
);
2672 strbuf_release(&filename
);
2674 if (freq
->localfile
!= -1)
2675 error("fd leakage in start: %d", freq
->localfile
);
2676 freq
->localfile
= open(freq
->tmpfile
.buf
,
2677 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
2679 * This could have failed due to the "lazy directory creation";
2680 * try to mkdir the last path component.
2682 if (freq
->localfile
< 0 && errno
== ENOENT
) {
2683 char *dir
= strrchr(freq
->tmpfile
.buf
, '/');
2686 mkdir(freq
->tmpfile
.buf
, 0777);
2689 freq
->localfile
= open(freq
->tmpfile
.buf
,
2690 O_WRONLY
| O_CREAT
| O_EXCL
, 0666);
2693 if (freq
->localfile
< 0) {
2694 error_errno("Couldn't create temporary file %s",
2699 git_inflate_init(&freq
->stream
);
2701 the_hash_algo
->init_fn(&freq
->c
);
2703 freq
->url
= get_remote_object_url(base_url
, hex
, 0);
2706 * If a previous temp file is present, process what was already
2709 prevlocal
= open(prevfile
.buf
, O_RDONLY
);
2710 if (prevlocal
!= -1) {
2712 prev_read
= xread(prevlocal
, prev_buf
, PREV_BUF_SIZE
);
2714 if (fwrite_sha1_file(prev_buf
,
2717 freq
) == prev_read
) {
2718 prev_posn
+= prev_read
;
2723 } while (prev_read
> 0);
2726 unlink_or_warn(prevfile
.buf
);
2727 strbuf_release(&prevfile
);
2730 * Reset inflate/SHA1 if there was an error reading the previous temp
2731 * file; also rewind to the beginning of the local file.
2733 if (prev_read
== -1) {
2734 git_inflate_end(&freq
->stream
);
2735 memset(&freq
->stream
, 0, sizeof(freq
->stream
));
2736 git_inflate_init(&freq
->stream
);
2737 the_hash_algo
->init_fn(&freq
->c
);
2740 lseek(freq
->localfile
, 0, SEEK_SET
);
2741 if (ftruncate(freq
->localfile
, 0) < 0) {
2742 error_errno("Couldn't truncate temporary file %s",
2749 freq
->slot
= get_active_slot();
2750 freq
->headers
= object_request_headers();
2752 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_WRITEDATA
, freq
);
2753 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_FAILONERROR
, 0L);
2754 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_sha1_file
);
2755 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_ERRORBUFFER
, freq
->errorstr
);
2756 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_URL
, freq
->url
);
2757 curl_easy_setopt(freq
->slot
->curl
, CURLOPT_HTTPHEADER
, freq
->headers
);
2760 * If we have successfully processed data from a previous fetch
2761 * attempt, only fetch the data we don't already have.
2764 if (http_is_verbose
)
2766 "Resuming fetch of object %s at byte %"PRIuMAX
"\n",
2767 hex
, (uintmax_t)prev_posn
);
2768 http_opt_request_remainder(freq
->slot
->curl
, prev_posn
);
2774 strbuf_release(&prevfile
);
2780 void process_http_object_request(struct http_object_request
*freq
)
2784 freq
->curl_result
= freq
->slot
->curl_result
;
2785 freq
->http_code
= freq
->slot
->http_code
;
2789 int finish_http_object_request(struct http_object_request
*freq
)
2792 struct strbuf filename
= STRBUF_INIT
;
2794 close(freq
->localfile
);
2795 freq
->localfile
= -1;
2797 process_http_object_request(freq
);
2799 if (freq
->http_code
== 416) {
2800 warning("requested range invalid; we may already have all the data.");
2801 } else if (freq
->curl_result
!= CURLE_OK
) {
2802 if (stat(freq
->tmpfile
.buf
, &st
) == 0)
2803 if (st
.st_size
== 0)
2804 unlink_or_warn(freq
->tmpfile
.buf
);
2808 git_hash_final_oid(&freq
->real_oid
, &freq
->c
);
2809 if (freq
->zret
!= Z_STREAM_END
) {
2810 unlink_or_warn(freq
->tmpfile
.buf
);
2813 if (!oideq(&freq
->oid
, &freq
->real_oid
)) {
2814 unlink_or_warn(freq
->tmpfile
.buf
);
2817 odb_loose_path(the_repository
->objects
->odb
, &filename
, &freq
->oid
);
2818 freq
->rename
= finalize_object_file(freq
->tmpfile
.buf
, filename
.buf
);
2819 strbuf_release(&filename
);
2821 return freq
->rename
;
2824 void abort_http_object_request(struct http_object_request
**freq_p
)
2826 struct http_object_request
*freq
= *freq_p
;
2827 unlink_or_warn(freq
->tmpfile
.buf
);
2829 release_http_object_request(freq_p
);
2832 void release_http_object_request(struct http_object_request
**freq_p
)
2834 struct http_object_request
*freq
= *freq_p
;
2835 if (freq
->localfile
!= -1) {
2836 close(freq
->localfile
);
2837 freq
->localfile
= -1;
2839 FREE_AND_NULL(freq
->url
);
2841 freq
->slot
->callback_func
= NULL
;
2842 freq
->slot
->callback_data
= NULL
;
2843 release_active_slot(freq
->slot
);
2846 curl_slist_free_all(freq
->headers
);
2847 strbuf_release(&freq
->tmpfile
);
2848 git_inflate_end(&freq
->stream
);