2 #include "repository.h"
9 #include "object-store.h"
14 struct packed_git
*packs
;
15 struct alt_base
*next
;
18 enum object_request_state
{
25 struct object_request
{
26 struct walker
*walker
;
28 struct alt_base
*repo
;
29 enum object_request_state state
;
30 struct http_object_request
*req
;
31 struct list_head node
;
34 struct alternates_request
{
35 struct walker
*walker
;
38 struct strbuf
*buffer
;
39 struct active_request_slot
*slot
;
49 static LIST_HEAD(object_queue_head
);
51 static void fetch_alternates(struct walker
*walker
, const char *base
);
53 static void process_object_response(void *callback_data
);
55 static void start_object_request(struct walker
*walker
,
56 struct object_request
*obj_req
)
58 struct active_request_slot
*slot
;
59 struct http_object_request
*req
;
61 req
= new_http_object_request(obj_req
->repo
->base
, &obj_req
->oid
);
63 obj_req
->state
= ABORTED
;
69 slot
->callback_func
= process_object_response
;
70 slot
->callback_data
= obj_req
;
72 /* Try to get the request started, abort the request on error */
73 obj_req
->state
= ACTIVE
;
74 if (!start_active_slot(slot
)) {
75 obj_req
->state
= ABORTED
;
76 release_http_object_request(req
);
81 static void finish_object_request(struct object_request
*obj_req
)
83 if (finish_http_object_request(obj_req
->req
))
86 if (obj_req
->req
->rename
== 0)
87 walker_say(obj_req
->walker
, "got %s\n", oid_to_hex(&obj_req
->oid
));
90 static void process_object_response(void *callback_data
)
92 struct object_request
*obj_req
=
93 (struct object_request
*)callback_data
;
94 struct walker
*walker
= obj_req
->walker
;
95 struct walker_data
*data
= walker
->data
;
96 struct alt_base
*alt
= data
->alt
;
98 process_http_object_request(obj_req
->req
);
99 obj_req
->state
= COMPLETE
;
101 /* Use alternates if necessary */
102 if (missing_target(obj_req
->req
)) {
103 fetch_alternates(walker
, alt
->base
);
104 if (obj_req
->repo
->next
!= NULL
) {
107 release_http_object_request(obj_req
->req
);
108 start_object_request(walker
, obj_req
);
113 finish_object_request(obj_req
);
116 static void release_object_request(struct object_request
*obj_req
)
118 if (obj_req
->req
!=NULL
&& obj_req
->req
->localfile
!= -1)
119 error("fd leakage in release: %d", obj_req
->req
->localfile
);
121 list_del(&obj_req
->node
);
125 #ifdef USE_CURL_MULTI
126 static int fill_active_slot(struct walker
*walker
)
128 struct object_request
*obj_req
;
129 struct list_head
*pos
, *tmp
, *head
= &object_queue_head
;
131 list_for_each_safe(pos
, tmp
, head
) {
132 obj_req
= list_entry(pos
, struct object_request
, node
);
133 if (obj_req
->state
== WAITING
) {
134 if (has_object_file(&obj_req
->oid
))
135 obj_req
->state
= COMPLETE
;
137 start_object_request(walker
, obj_req
);
146 static void prefetch(struct walker
*walker
, unsigned char *sha1
)
148 struct object_request
*newreq
;
149 struct walker_data
*data
= walker
->data
;
151 newreq
= xmalloc(sizeof(*newreq
));
152 newreq
->walker
= walker
;
153 hashcpy(newreq
->oid
.hash
, sha1
);
154 newreq
->repo
= data
->alt
;
155 newreq
->state
= WAITING
;
158 http_is_verbose
= walker
->get_verbosely
;
159 list_add_tail(&newreq
->node
, &object_queue_head
);
161 #ifdef USE_CURL_MULTI
167 static int is_alternate_allowed(const char *url
)
169 const char *protocols
[] = {
170 "http", "https", "ftp", "ftps"
174 if (http_follow_config
!= HTTP_FOLLOW_ALWAYS
) {
175 warning("alternate disabled by http.followRedirects: %s", url
);
179 for (i
= 0; i
< ARRAY_SIZE(protocols
); i
++) {
181 if (skip_prefix(url
, protocols
[i
], &end
) &&
182 starts_with(end
, "://"))
186 if (i
>= ARRAY_SIZE(protocols
)) {
187 warning("ignoring alternate with unknown protocol: %s", url
);
190 if (!is_transport_allowed(protocols
[i
], 0)) {
191 warning("ignoring alternate with restricted protocol: %s", url
);
198 static void process_alternates_response(void *callback_data
)
200 struct alternates_request
*alt_req
=
201 (struct alternates_request
*)callback_data
;
202 struct walker
*walker
= alt_req
->walker
;
203 struct walker_data
*cdata
= walker
->data
;
204 struct active_request_slot
*slot
= alt_req
->slot
;
205 struct alt_base
*tail
= cdata
->alt
;
206 const char *base
= alt_req
->base
;
207 const char null_byte
= '\0';
211 if (alt_req
->http_specific
) {
212 if (slot
->curl_result
!= CURLE_OK
||
213 !alt_req
->buffer
->len
) {
215 /* Try reusing the slot to get non-http alternates */
216 alt_req
->http_specific
= 0;
217 strbuf_reset(alt_req
->url
);
218 strbuf_addf(alt_req
->url
, "%s/objects/info/alternates",
220 curl_easy_setopt(slot
->curl
, CURLOPT_URL
,
224 if (slot
->finished
!= NULL
)
225 (*slot
->finished
) = 0;
226 if (!start_active_slot(slot
)) {
227 cdata
->got_alternates
= -1;
229 if (slot
->finished
!= NULL
)
230 (*slot
->finished
) = 1;
234 } else if (slot
->curl_result
!= CURLE_OK
) {
235 if (!missing_target(slot
)) {
236 cdata
->got_alternates
= -1;
241 fwrite_buffer((char *)&null_byte
, 1, 1, alt_req
->buffer
);
242 alt_req
->buffer
->len
--;
243 data
= alt_req
->buffer
->buf
;
245 while (i
< alt_req
->buffer
->len
) {
247 while (posn
< alt_req
->buffer
->len
&& data
[posn
] != '\n')
249 if (data
[posn
] == '\n') {
252 struct alt_base
*newalt
;
253 if (data
[i
] == '/') {
256 * http://git.host/pub/scm/linux.git/
258 * so memcpy(dst, base, serverlen) will
259 * copy up to "...git.host".
261 const char *colon_ss
= strstr(base
,"://");
263 serverlen
= (strchr(colon_ss
+ 3, '/')
267 } else if (!memcmp(data
+ i
, "../", 3)) {
269 * Relative URL; chop the corresponding
270 * number of subpath from base (and ../
271 * from data), and concatenate the result.
273 * The code first drops ../ from data, and
274 * then drops one ../ from data and one path
275 * from base. IOW, one extra ../ is dropped
276 * from data than path is dropped from base.
278 * This is not wrong. The alternate in
279 * http://git.host/pub/scm/linux.git/
281 * http://git.host/pub/scm/linus.git/
282 * is ../../linus.git/objects/. You need
283 * two ../../ to borrow from your direct
287 serverlen
= strlen(base
);
288 while (i
+ 2 < posn
&&
289 !memcmp(data
+ i
, "../", 3)) {
292 } while (serverlen
&&
293 base
[serverlen
- 1] != '/');
296 /* If the server got removed, give up. */
297 okay
= strchr(base
, ':') - base
+ 3 <
299 } else if (alt_req
->http_specific
) {
300 char *colon
= strchr(data
+ i
, ':');
301 char *slash
= strchr(data
+ i
, '/');
302 if (colon
&& slash
&& colon
< data
+ posn
&&
303 slash
< data
+ posn
&& colon
< slash
) {
308 struct strbuf target
= STRBUF_INIT
;
309 strbuf_add(&target
, base
, serverlen
);
310 strbuf_add(&target
, data
+ i
, posn
- i
);
311 if (!strbuf_strip_suffix(&target
, "objects")) {
312 warning("ignoring alternate that does"
313 " not end in 'objects': %s",
315 strbuf_release(&target
);
316 } else if (is_alternate_allowed(target
.buf
)) {
317 warning("adding alternate object store: %s",
319 newalt
= xmalloc(sizeof(*newalt
));
321 newalt
->base
= strbuf_detach(&target
, NULL
);
322 newalt
->got_indices
= 0;
323 newalt
->packs
= NULL
;
325 while (tail
->next
!= NULL
)
329 strbuf_release(&target
);
336 cdata
->got_alternates
= 1;
339 static void fetch_alternates(struct walker
*walker
, const char *base
)
341 struct strbuf buffer
= STRBUF_INIT
;
342 struct strbuf url
= STRBUF_INIT
;
343 struct active_request_slot
*slot
;
344 struct alternates_request alt_req
;
345 struct walker_data
*cdata
= walker
->data
;
348 * If another request has already started fetching alternates,
349 * wait for them to arrive and return to processing this request's
352 #ifdef USE_CURL_MULTI
353 while (cdata
->got_alternates
== 0) {
358 /* Nothing to do if they've already been fetched */
359 if (cdata
->got_alternates
== 1)
362 /* Start the fetch */
363 cdata
->got_alternates
= 0;
365 if (walker
->get_verbosely
)
366 fprintf(stderr
, "Getting alternates list for %s\n", base
);
368 strbuf_addf(&url
, "%s/objects/info/http-alternates", base
);
371 * Use a callback to process the result, since another request
372 * may fail and need to have alternates loaded before continuing
374 slot
= get_active_slot();
375 slot
->callback_func
= process_alternates_response
;
376 alt_req
.walker
= walker
;
377 slot
->callback_data
= &alt_req
;
379 curl_easy_setopt(slot
->curl
, CURLOPT_FILE
, &buffer
);
380 curl_easy_setopt(slot
->curl
, CURLOPT_WRITEFUNCTION
, fwrite_buffer
);
381 curl_easy_setopt(slot
->curl
, CURLOPT_URL
, url
.buf
);
385 alt_req
.buffer
= &buffer
;
386 alt_req
.http_specific
= 1;
389 if (start_active_slot(slot
))
390 run_active_slot(slot
);
392 cdata
->got_alternates
= -1;
394 strbuf_release(&buffer
);
395 strbuf_release(&url
);
398 static int fetch_indices(struct walker
*walker
, struct alt_base
*repo
)
402 if (repo
->got_indices
)
405 if (walker
->get_verbosely
)
406 fprintf(stderr
, "Getting pack list for %s\n", repo
->base
);
408 switch (http_get_info_packs(repo
->base
, &repo
->packs
)) {
410 case HTTP_MISSING_TARGET
:
411 repo
->got_indices
= 1;
415 repo
->got_indices
= 0;
422 static int http_fetch_pack(struct walker
*walker
, struct alt_base
*repo
, unsigned char *sha1
)
424 struct packed_git
*target
;
426 struct slot_results results
;
427 struct http_pack_request
*preq
;
429 if (fetch_indices(walker
, repo
))
431 target
= find_sha1_pack(sha1
, repo
->packs
);
435 if (walker
->get_verbosely
) {
436 fprintf(stderr
, "Getting pack %s\n",
437 sha1_to_hex(target
->sha1
));
438 fprintf(stderr
, " which contains %s\n",
442 preq
= new_http_pack_request(target
, repo
->base
);
445 preq
->lst
= &repo
->packs
;
446 preq
->slot
->results
= &results
;
448 if (start_active_slot(preq
->slot
)) {
449 run_active_slot(preq
->slot
);
450 if (results
.curl_result
!= CURLE_OK
) {
451 error("Unable to get pack file %s\n%s", preq
->url
,
456 error("Unable to start request");
460 ret
= finish_http_pack_request(preq
);
461 release_http_pack_request(preq
);
471 static void abort_object_request(struct object_request
*obj_req
)
473 release_object_request(obj_req
);
476 static int fetch_object(struct walker
*walker
, unsigned char *sha1
)
478 char *hex
= sha1_to_hex(sha1
);
480 struct object_request
*obj_req
= NULL
;
481 struct http_object_request
*req
;
482 struct list_head
*pos
, *head
= &object_queue_head
;
484 list_for_each(pos
, head
) {
485 obj_req
= list_entry(pos
, struct object_request
, node
);
486 if (hasheq(obj_req
->oid
.hash
, sha1
))
490 return error("Couldn't find request for %s in the queue", hex
);
492 if (has_object_file(&obj_req
->oid
)) {
493 if (obj_req
->req
!= NULL
)
494 abort_http_object_request(obj_req
->req
);
495 abort_object_request(obj_req
);
499 #ifdef USE_CURL_MULTI
500 while (obj_req
->state
== WAITING
)
503 start_object_request(walker
, obj_req
);
507 * obj_req->req might change when fetching alternates in the callback
508 * process_object_response; therefore, the "shortcut" variable, req,
509 * is used only after we're done with slots.
511 while (obj_req
->state
== ACTIVE
)
512 run_active_slot(obj_req
->req
->slot
);
516 if (req
->localfile
!= -1) {
517 close(req
->localfile
);
522 * we turned off CURLOPT_FAILONERROR to avoid losing a
523 * persistent connection and got CURLE_OK.
525 if (req
->http_code
>= 300 && req
->curl_result
== CURLE_OK
&&
526 (starts_with(req
->url
, "http://") ||
527 starts_with(req
->url
, "https://"))) {
528 req
->curl_result
= CURLE_HTTP_RETURNED_ERROR
;
529 xsnprintf(req
->errorstr
, sizeof(req
->errorstr
),
530 "HTTP request failed");
533 if (obj_req
->state
== ABORTED
) {
534 ret
= error("Request for %s aborted", hex
);
535 } else if (req
->curl_result
!= CURLE_OK
&&
536 req
->http_code
!= 416) {
537 if (missing_target(req
))
538 ret
= -1; /* Be silent, it is probably in a pack. */
540 ret
= error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
541 req
->errorstr
, req
->curl_result
,
542 req
->http_code
, hex
);
543 } else if (req
->zret
!= Z_STREAM_END
) {
544 walker
->corrupt_object_found
++;
545 ret
= error("File %s (%s) corrupt", hex
, req
->url
);
546 } else if (!oideq(&obj_req
->oid
, &req
->real_oid
)) {
547 ret
= error("File %s has bad hash", hex
);
548 } else if (req
->rename
< 0) {
549 struct strbuf buf
= STRBUF_INIT
;
550 loose_object_path(the_repository
, &buf
, &req
->oid
);
551 ret
= error("unable to write sha1 filename %s", buf
.buf
);
552 strbuf_release(&buf
);
555 release_http_object_request(req
);
556 release_object_request(obj_req
);
560 static int fetch(struct walker
*walker
, unsigned char *sha1
)
562 struct walker_data
*data
= walker
->data
;
563 struct alt_base
*altbase
= data
->alt
;
565 if (!fetch_object(walker
, sha1
))
568 if (!http_fetch_pack(walker
, altbase
, sha1
))
570 fetch_alternates(walker
, data
->alt
->base
);
571 altbase
= altbase
->next
;
573 return error("Unable to find %s under %s", sha1_to_hex(sha1
),
577 static int fetch_ref(struct walker
*walker
, struct ref
*ref
)
579 struct walker_data
*data
= walker
->data
;
580 return http_fetch_ref(data
->alt
->base
, ref
);
583 static void cleanup(struct walker
*walker
)
585 struct walker_data
*data
= walker
->data
;
586 struct alt_base
*alt
, *alt_next
;
591 alt_next
= alt
->next
;
603 struct walker
*get_http_walker(const char *url
)
606 struct walker_data
*data
= xmalloc(sizeof(struct walker_data
));
607 struct walker
*walker
= xmalloc(sizeof(struct walker
));
609 data
->alt
= xmalloc(sizeof(*data
->alt
));
610 data
->alt
->base
= xstrdup(url
);
611 for (s
= data
->alt
->base
+ strlen(data
->alt
->base
) - 1; *s
== '/'; --s
)
614 data
->alt
->got_indices
= 0;
615 data
->alt
->packs
= NULL
;
616 data
->alt
->next
= NULL
;
617 data
->got_alternates
= -1;
619 walker
->corrupt_object_found
= 0;
620 walker
->fetch
= fetch
;
621 walker
->fetch_ref
= fetch_ref
;
622 walker
->prefetch
= prefetch
;
623 walker
->cleanup
= cleanup
;
626 #ifdef USE_CURL_MULTI
627 add_fill_function(walker
, (int (*)(void *)) fill_active_slot
);