2 #include "repository.h"
12 #include "fetch-pack.h"
14 #include "run-command.h"
16 #include "transport.h"
18 #include "oid-array.h"
21 #include "object-store.h"
22 #include "connected.h"
23 #include "fetch-negotiator.h"
26 #include "commit-reach.h"
27 #include "commit-graph.h"
30 static int transfer_unpack_limit
= -1;
31 static int fetch_unpack_limit
= -1;
32 static int unpack_limit
= 100;
33 static int prefer_ofs_delta
= 1;
35 static int deepen_since_ok
;
36 static int deepen_not_ok
;
37 static int fetch_fsck_objects
= -1;
38 static int transfer_fsck_objects
= -1;
39 static int agent_supported
;
40 static int server_supports_filtering
;
41 static int advertise_sid
;
42 static struct shallow_lock shallow_lock
;
43 static const char *alternate_shallow_file
;
44 static struct fsck_options fsck_options
= FSCK_OPTIONS_MISSING_GITMODULES
;
45 static struct strbuf fsck_msg_types
= STRBUF_INIT
;
46 static struct string_list uri_protocols
= STRING_LIST_INIT_DUP
;
48 /* Remember to update object flag allocation in object.h */
49 #define COMPLETE (1U << 0)
50 #define ALTERNATE (1U << 1)
51 #define COMMON (1U << 6)
52 #define REACH_SCRATCH (1U << 7)
55 * After sending this many "have"s if we do not get any new ACK , we
56 * give up traversing our history.
58 #define MAX_IN_VAIN 256
60 static int multi_ack
, use_sideband
;
61 /* Allow specifying sha1 if it is a ref tip. */
62 #define ALLOW_TIP_SHA1 01
63 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
64 #define ALLOW_REACHABLE_SHA1 02
65 static unsigned int allow_unadvertised_object_request
;
67 __attribute__((format (printf
, 2, 3)))
68 static inline void print_verbose(const struct fetch_pack_args
*args
,
76 va_start(params
, fmt
);
77 vfprintf(stderr
, fmt
, params
);
82 struct alternate_object_cache
{
83 struct object
**items
;
87 static void cache_one_alternate(const struct object_id
*oid
,
90 struct alternate_object_cache
*cache
= vcache
;
91 struct object
*obj
= parse_object(the_repository
, oid
);
93 if (!obj
|| (obj
->flags
& ALTERNATE
))
96 obj
->flags
|= ALTERNATE
;
97 ALLOC_GROW(cache
->items
, cache
->nr
+ 1, cache
->alloc
);
98 cache
->items
[cache
->nr
++] = obj
;
101 static void for_each_cached_alternate(struct fetch_negotiator
*negotiator
,
102 void (*cb
)(struct fetch_negotiator
*,
105 static int initialized
;
106 static struct alternate_object_cache cache
;
110 for_each_alternate_ref(cache_one_alternate
, &cache
);
114 for (i
= 0; i
< cache
.nr
; i
++)
115 cb(negotiator
, cache
.items
[i
]);
118 static struct commit
*deref_without_lazy_fetch_extended(const struct object_id
*oid
,
119 int mark_tags_complete
,
120 enum object_type
*type
,
121 unsigned int oi_flags
)
123 struct object_info info
= { .typep
= type
};
124 struct commit
*commit
;
126 commit
= lookup_commit_in_graph(the_repository
, oid
);
131 if (oid_object_info_extended(the_repository
, oid
, &info
,
134 if (*type
== OBJ_TAG
) {
135 struct tag
*tag
= (struct tag
*)
136 parse_object(the_repository
, oid
);
140 if (mark_tags_complete
)
141 tag
->object
.flags
|= COMPLETE
;
142 oid
= &tag
->tagged
->oid
;
148 if (*type
== OBJ_COMMIT
) {
149 struct commit
*commit
= lookup_commit(the_repository
, oid
);
150 if (!commit
|| repo_parse_commit(the_repository
, commit
))
159 static struct commit
*deref_without_lazy_fetch(const struct object_id
*oid
,
160 int mark_tags_complete
)
162 enum object_type type
;
163 unsigned flags
= OBJECT_INFO_SKIP_FETCH_OBJECT
| OBJECT_INFO_QUICK
;
164 return deref_without_lazy_fetch_extended(oid
, mark_tags_complete
,
168 static int rev_list_insert_ref(struct fetch_negotiator
*negotiator
,
169 const struct object_id
*oid
)
171 struct commit
*c
= deref_without_lazy_fetch(oid
, 0);
174 negotiator
->add_tip(negotiator
, c
);
178 static int rev_list_insert_ref_oid(const char *refname
, const struct object_id
*oid
,
179 int flag
, void *cb_data
)
181 return rev_list_insert_ref(cb_data
, oid
);
192 static void consume_shallow_list(struct fetch_pack_args
*args
,
193 struct packet_reader
*reader
)
195 if (args
->stateless_rpc
&& args
->deepen
) {
196 /* If we sent a depth we will get back "duplicate"
197 * shallow and unshallow commands every time there
198 * is a block of have lines exchanged.
200 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
201 if (starts_with(reader
->line
, "shallow "))
203 if (starts_with(reader
->line
, "unshallow "))
205 die(_("git fetch-pack: expected shallow list"));
207 if (reader
->status
!= PACKET_READ_FLUSH
)
208 die(_("git fetch-pack: expected a flush packet after shallow list"));
212 static enum ack_type
get_ack(struct packet_reader
*reader
,
213 struct object_id
*result_oid
)
218 if (packet_reader_read(reader
) != PACKET_READ_NORMAL
)
219 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
220 len
= reader
->pktlen
;
222 if (!strcmp(reader
->line
, "NAK"))
224 if (skip_prefix(reader
->line
, "ACK ", &arg
)) {
226 if (!parse_oid_hex(arg
, result_oid
, &p
)) {
227 len
-= p
- reader
->line
;
230 if (strstr(p
, "continue"))
232 if (strstr(p
, "common"))
234 if (strstr(p
, "ready"))
239 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader
->line
);
242 static void send_request(struct fetch_pack_args
*args
,
243 int fd
, struct strbuf
*buf
)
245 if (args
->stateless_rpc
) {
246 send_sideband(fd
, -1, buf
->buf
, buf
->len
, LARGE_PACKET_MAX
);
249 if (write_in_full(fd
, buf
->buf
, buf
->len
) < 0)
250 die_errno(_("unable to write to remote"));
254 static void insert_one_alternate_object(struct fetch_negotiator
*negotiator
,
257 rev_list_insert_ref(negotiator
, &obj
->oid
);
260 #define INITIAL_FLUSH 16
261 #define PIPESAFE_FLUSH 32
262 #define LARGE_FLUSH 16384
264 static int next_flush(int stateless_rpc
, int count
)
267 if (count
< LARGE_FLUSH
)
270 count
= count
* 11 / 10;
272 if (count
< PIPESAFE_FLUSH
)
275 count
+= PIPESAFE_FLUSH
;
280 static void mark_tips(struct fetch_negotiator
*negotiator
,
281 const struct oid_array
*negotiation_tips
)
285 if (!negotiation_tips
) {
286 for_each_rawref(rev_list_insert_ref_oid
, negotiator
);
290 for (i
= 0; i
< negotiation_tips
->nr
; i
++)
291 rev_list_insert_ref(negotiator
, &negotiation_tips
->oid
[i
]);
295 static int find_common(struct fetch_negotiator
*negotiator
,
296 struct fetch_pack_args
*args
,
297 int fd
[2], struct object_id
*result_oid
,
301 int count
= 0, flushes
= 0, flush_at
= INITIAL_FLUSH
, retval
;
302 const struct object_id
*oid
;
303 unsigned in_vain
= 0;
304 int got_continue
= 0;
306 struct strbuf req_buf
= STRBUF_INIT
;
307 size_t state_len
= 0;
308 struct packet_reader reader
;
310 if (args
->stateless_rpc
&& multi_ack
== 1)
311 die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");
313 packet_reader_init(&reader
, fd
[0], NULL
, 0,
314 PACKET_READ_CHOMP_NEWLINE
|
315 PACKET_READ_DIE_ON_ERR_PACKET
);
317 mark_tips(negotiator
, args
->negotiation_tips
);
318 for_each_cached_alternate(negotiator
, insert_one_alternate_object
);
321 for ( ; refs
; refs
= refs
->next
) {
322 struct object_id
*remote
= &refs
->old_oid
;
323 const char *remote_hex
;
326 if (!args
->refetch
) {
328 * If that object is complete (i.e. it is an ancestor of a
329 * local ref), we tell them we have it but do not have to
330 * tell them about its ancestors, which they already know
333 * We use lookup_object here because we are only
334 * interested in the case we *know* the object is
335 * reachable and we have already scanned it.
337 if (((o
= lookup_object(the_repository
, remote
)) != NULL
) &&
338 (o
->flags
& COMPLETE
)) {
343 remote_hex
= oid_to_hex(remote
);
345 struct strbuf c
= STRBUF_INIT
;
346 if (multi_ack
== 2) strbuf_addstr(&c
, " multi_ack_detailed");
347 if (multi_ack
== 1) strbuf_addstr(&c
, " multi_ack");
348 if (no_done
) strbuf_addstr(&c
, " no-done");
349 if (use_sideband
== 2) strbuf_addstr(&c
, " side-band-64k");
350 if (use_sideband
== 1) strbuf_addstr(&c
, " side-band");
351 if (args
->deepen_relative
) strbuf_addstr(&c
, " deepen-relative");
352 if (args
->use_thin_pack
) strbuf_addstr(&c
, " thin-pack");
353 if (args
->no_progress
) strbuf_addstr(&c
, " no-progress");
354 if (args
->include_tag
) strbuf_addstr(&c
, " include-tag");
355 if (prefer_ofs_delta
) strbuf_addstr(&c
, " ofs-delta");
356 if (deepen_since_ok
) strbuf_addstr(&c
, " deepen-since");
357 if (deepen_not_ok
) strbuf_addstr(&c
, " deepen-not");
358 if (agent_supported
) strbuf_addf(&c
, " agent=%s",
359 git_user_agent_sanitized());
361 strbuf_addf(&c
, " session-id=%s", trace2_session_id());
362 if (args
->filter_options
.choice
)
363 strbuf_addstr(&c
, " filter");
364 packet_buf_write(&req_buf
, "want %s%s\n", remote_hex
, c
.buf
);
367 packet_buf_write(&req_buf
, "want %s\n", remote_hex
);
372 strbuf_release(&req_buf
);
377 if (is_repository_shallow(the_repository
))
378 write_shallow_commits(&req_buf
, 1, NULL
);
380 packet_buf_write(&req_buf
, "deepen %d", args
->depth
);
381 if (args
->deepen_since
) {
382 timestamp_t max_age
= approxidate(args
->deepen_since
);
383 packet_buf_write(&req_buf
, "deepen-since %"PRItime
, max_age
);
385 if (args
->deepen_not
) {
387 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
388 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
389 packet_buf_write(&req_buf
, "deepen-not %s", s
->string
);
392 if (server_supports_filtering
&& args
->filter_options
.choice
) {
394 expand_list_objects_filter_spec(&args
->filter_options
);
395 packet_buf_write(&req_buf
, "filter %s", spec
);
397 packet_buf_flush(&req_buf
);
398 state_len
= req_buf
.len
;
402 struct object_id oid
;
404 send_request(args
, fd
[1], &req_buf
);
405 while (packet_reader_read(&reader
) == PACKET_READ_NORMAL
) {
406 if (skip_prefix(reader
.line
, "shallow ", &arg
)) {
407 if (get_oid_hex(arg
, &oid
))
408 die(_("invalid shallow line: %s"), reader
.line
);
409 register_shallow(the_repository
, &oid
);
412 if (skip_prefix(reader
.line
, "unshallow ", &arg
)) {
413 if (get_oid_hex(arg
, &oid
))
414 die(_("invalid unshallow line: %s"), reader
.line
);
415 if (!lookup_object(the_repository
, &oid
))
416 die(_("object not found: %s"), reader
.line
);
417 /* make sure that it is parsed as shallow */
418 if (!parse_object(the_repository
, &oid
))
419 die(_("error in object: %s"), reader
.line
);
420 if (unregister_shallow(&oid
))
421 die(_("no shallow found: %s"), reader
.line
);
424 die(_("expected shallow/unshallow, got %s"), reader
.line
);
426 } else if (!args
->stateless_rpc
)
427 send_request(args
, fd
[1], &req_buf
);
429 if (!args
->stateless_rpc
) {
430 /* If we aren't using the stateless-rpc interface
431 * we don't need to retain the headers.
433 strbuf_setlen(&req_buf
, 0);
437 trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository
);
440 while ((oid
= negotiator
->next(negotiator
))) {
441 packet_buf_write(&req_buf
, "have %s\n", oid_to_hex(oid
));
442 print_verbose(args
, "have %s", oid_to_hex(oid
));
444 if (flush_at
<= ++count
) {
447 packet_buf_flush(&req_buf
);
448 send_request(args
, fd
[1], &req_buf
);
449 strbuf_setlen(&req_buf
, state_len
);
451 flush_at
= next_flush(args
->stateless_rpc
, count
);
454 * We keep one window "ahead" of the other side, and
455 * will wait for an ACK only on the next one
457 if (!args
->stateless_rpc
&& count
== INITIAL_FLUSH
)
460 consume_shallow_list(args
, &reader
);
462 ack
= get_ack(&reader
, result_oid
);
464 print_verbose(args
, _("got %s %d %s"), "ack",
465 ack
, oid_to_hex(result_oid
));
475 struct commit
*commit
=
476 lookup_commit(the_repository
,
481 die(_("invalid commit %s"), oid_to_hex(result_oid
));
482 was_common
= negotiator
->ack(negotiator
, commit
);
483 if (args
->stateless_rpc
486 /* We need to replay the have for this object
487 * on the next RPC request so the peer knows
488 * it is in common with us.
490 const char *hex
= oid_to_hex(result_oid
);
491 packet_buf_write(&req_buf
, "have %s\n", hex
);
492 state_len
= req_buf
.len
;
494 * Reset in_vain because an ack
495 * for this commit has not been
499 } else if (!args
->stateless_rpc
500 || ack
!= ACK_common
)
504 if (ack
== ACK_ready
)
511 if (got_continue
&& MAX_IN_VAIN
< in_vain
) {
512 print_verbose(args
, _("giving up"));
520 trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository
);
521 if (!got_ready
|| !no_done
) {
522 packet_buf_write(&req_buf
, "done\n");
523 send_request(args
, fd
[1], &req_buf
);
525 print_verbose(args
, _("done"));
530 strbuf_release(&req_buf
);
532 if (!got_ready
|| !no_done
)
533 consume_shallow_list(args
, &reader
);
534 while (flushes
|| multi_ack
) {
535 int ack
= get_ack(&reader
, result_oid
);
537 print_verbose(args
, _("got %s (%d) %s"), "ack",
538 ack
, oid_to_hex(result_oid
));
546 /* it is no error to fetch into a completely empty repo */
547 return count
? retval
: 0;
550 static struct commit_list
*complete
;
552 static int mark_complete(const struct object_id
*oid
)
554 struct commit
*commit
= deref_without_lazy_fetch(oid
, 1);
556 if (commit
&& !(commit
->object
.flags
& COMPLETE
)) {
557 commit
->object
.flags
|= COMPLETE
;
558 commit_list_insert(commit
, &complete
);
563 static int mark_complete_oid(const char *refname
, const struct object_id
*oid
,
564 int flag
, void *cb_data
)
566 return mark_complete(oid
);
569 static void mark_recent_complete_commits(struct fetch_pack_args
*args
,
572 while (complete
&& cutoff
<= complete
->item
->date
) {
573 print_verbose(args
, _("Marking %s as complete"),
574 oid_to_hex(&complete
->item
->object
.oid
));
575 pop_most_recent_commit(&complete
, COMPLETE
);
579 static void add_refs_to_oidset(struct oidset
*oids
, struct ref
*refs
)
581 for (; refs
; refs
= refs
->next
)
582 oidset_insert(oids
, &refs
->old_oid
);
585 static int is_unmatched_ref(const struct ref
*ref
)
587 struct object_id oid
;
589 return ref
->match_status
== REF_NOT_MATCHED
&&
590 !parse_oid_hex(ref
->name
, &oid
, &p
) &&
592 oideq(&oid
, &ref
->old_oid
);
595 static void filter_refs(struct fetch_pack_args
*args
,
597 struct ref
**sought
, int nr_sought
)
599 struct ref
*newlist
= NULL
;
600 struct ref
**newtail
= &newlist
;
601 struct ref
*unmatched
= NULL
;
602 struct ref
*ref
, *next
;
603 struct oidset tip_oids
= OIDSET_INIT
;
605 int strict
= !(allow_unadvertised_object_request
&
606 (ALLOW_TIP_SHA1
| ALLOW_REACHABLE_SHA1
));
609 for (ref
= *refs
; ref
; ref
= next
) {
613 if (starts_with(ref
->name
, "refs/") &&
614 check_refname_format(ref
->name
, 0)) {
616 * trash or a peeled value; do not even add it to
622 while (i
< nr_sought
) {
623 int cmp
= strcmp(ref
->name
, sought
[i
]->name
);
625 break; /* definitely do not have it */
627 keep
= 1; /* definitely have it */
628 sought
[i
]->match_status
= REF_MATCHED
;
633 if (!keep
&& args
->fetch_all
&&
634 (!args
->deepen
|| !starts_with(ref
->name
, "refs/tags/")))
641 newtail
= &ref
->next
;
643 ref
->next
= unmatched
;
649 for (i
= 0; i
< nr_sought
; i
++) {
651 if (!is_unmatched_ref(ref
))
654 add_refs_to_oidset(&tip_oids
, unmatched
);
655 add_refs_to_oidset(&tip_oids
, newlist
);
660 /* Append unmatched requests to the list */
661 for (i
= 0; i
< nr_sought
; i
++) {
663 if (!is_unmatched_ref(ref
))
666 if (!strict
|| oidset_contains(&tip_oids
, &ref
->old_oid
)) {
667 ref
->match_status
= REF_MATCHED
;
668 *newtail
= copy_ref(ref
);
669 newtail
= &(*newtail
)->next
;
671 ref
->match_status
= REF_UNADVERTISED_NOT_ALLOWED
;
675 oidset_clear(&tip_oids
);
676 free_refs(unmatched
);
681 static void mark_alternate_complete(struct fetch_negotiator
*unused
,
684 mark_complete(&obj
->oid
);
687 struct loose_object_iter
{
688 struct oidset
*loose_object_set
;
693 * Mark recent commits available locally and reachable from a local ref as
696 * The cutoff time for recency is determined by this heuristic: it is the
697 * earliest commit time of the objects in refs that are commits and that we know
698 * the commit time of.
700 static void mark_complete_and_common_ref(struct fetch_negotiator
*negotiator
,
701 struct fetch_pack_args
*args
,
705 int old_save_commit_buffer
= save_commit_buffer
;
706 timestamp_t cutoff
= 0;
711 save_commit_buffer
= 0;
713 trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL
);
714 for (ref
= *refs
; ref
; ref
= ref
->next
) {
715 struct commit
*commit
;
717 commit
= lookup_commit_in_graph(the_repository
, &ref
->old_oid
);
721 if (!has_object_file_with_flags(&ref
->old_oid
,
723 OBJECT_INFO_SKIP_FETCH_OBJECT
))
725 o
= parse_object(the_repository
, &ref
->old_oid
);
726 if (!o
|| o
->type
!= OBJ_COMMIT
)
729 commit
= (struct commit
*)o
;
733 * We already have it -- which may mean that we were
734 * in sync with the other side at some time after
735 * that (it is OK if we guess wrong here).
737 if (!cutoff
|| cutoff
< commit
->date
)
738 cutoff
= commit
->date
;
740 trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL
);
743 * This block marks all local refs as COMPLETE, and then recursively marks all
744 * parents of those refs as COMPLETE.
746 trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL
);
748 for_each_rawref(mark_complete_oid
, NULL
);
749 for_each_cached_alternate(NULL
, mark_alternate_complete
);
750 commit_list_sort_by_date(&complete
);
752 mark_recent_complete_commits(args
, cutoff
);
754 trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL
);
757 * Mark all complete remote refs as common refs.
758 * Don't mark them common yet; the server has to be told so first.
760 trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL
);
761 for (ref
= *refs
; ref
; ref
= ref
->next
) {
762 struct commit
*c
= deref_without_lazy_fetch(&ref
->old_oid
, 0);
764 if (!c
|| !(c
->object
.flags
& COMPLETE
))
767 negotiator
->known_common(negotiator
, c
);
769 trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL
);
771 save_commit_buffer
= old_save_commit_buffer
;
775 * Returns 1 if every object pointed to by the given remote refs is available
776 * locally and reachable from a local ref, and 0 otherwise.
778 static int everything_local(struct fetch_pack_args
*args
,
784 for (retval
= 1, ref
= *refs
; ref
; ref
= ref
->next
) {
785 const struct object_id
*remote
= &ref
->old_oid
;
788 o
= lookup_object(the_repository
, remote
);
789 if (!o
|| !(o
->flags
& COMPLETE
)) {
791 print_verbose(args
, "want %s (%s)", oid_to_hex(remote
),
795 print_verbose(args
, _("already have %s (%s)"), oid_to_hex(remote
),
802 static int sideband_demux(int in
, int out
, void *data
)
807 ret
= recv_sideband("fetch-pack", xd
[0], out
);
812 static void create_promisor_file(const char *keep_name
,
813 struct ref
**sought
, int nr_sought
)
815 struct strbuf promisor_name
= STRBUF_INIT
;
818 strbuf_addstr(&promisor_name
, keep_name
);
819 suffix_stripped
= strbuf_strip_suffix(&promisor_name
, ".keep");
820 if (!suffix_stripped
)
821 BUG("name of pack lockfile should end with .keep (was '%s')",
823 strbuf_addstr(&promisor_name
, ".promisor");
825 write_promisor_file(promisor_name
.buf
, sought
, nr_sought
);
827 strbuf_release(&promisor_name
);
830 static void parse_gitmodules_oids(int fd
, struct oidset
*gitmodules_oids
)
832 int len
= the_hash_algo
->hexsz
+ 1; /* hash + NL */
835 char hex_hash
[GIT_MAX_HEXSZ
+ 1];
836 int read_len
= read_in_full(fd
, hex_hash
, len
);
837 struct object_id oid
;
843 die("invalid length read %d", read_len
);
844 if (parse_oid_hex(hex_hash
, &oid
, &end
) || *end
!= '\n')
846 oidset_insert(gitmodules_oids
, &oid
);
850 static void add_index_pack_keep_option(struct strvec
*args
)
852 char hostname
[HOST_NAME_MAX
+ 1];
854 if (xgethostname(hostname
, sizeof(hostname
)))
855 xsnprintf(hostname
, sizeof(hostname
), "localhost");
856 strvec_pushf(args
, "--keep=fetch-pack %"PRIuMAX
" on %s",
857 (uintmax_t)getpid(), hostname
);
861 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
862 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
863 * stored there. (It must be freed by the caller.)
865 static int get_pack(struct fetch_pack_args
*args
,
866 int xd
[2], struct string_list
*pack_lockfiles
,
867 struct strvec
*index_pack_args
,
868 struct ref
**sought
, int nr_sought
,
869 struct oidset
*gitmodules_oids
)
872 int do_keep
= args
->keep_pack
;
873 const char *cmd_name
;
874 struct pack_header header
;
876 struct child_process cmd
= CHILD_PROCESS_INIT
;
877 int fsck_objects
= 0;
880 memset(&demux
, 0, sizeof(demux
));
882 /* xd[] is talking with upload-pack; subprocess reads from
883 * xd[0], spits out band#2 to stderr, and feeds us band#1
884 * through demux->out.
886 demux
.proc
= sideband_demux
;
889 demux
.isolate_sigpipe
= 1;
890 if (start_async(&demux
))
891 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
896 if (!args
->keep_pack
&& unpack_limit
&& !index_pack_args
) {
898 if (read_pack_header(demux
.out
, &header
))
899 die(_("protocol error: bad pack header"));
901 if (ntohl(header
.hdr_entries
) < unpack_limit
)
907 if (alternate_shallow_file
) {
908 strvec_push(&cmd
.args
, "--shallow-file");
909 strvec_push(&cmd
.args
, alternate_shallow_file
);
912 if (fetch_fsck_objects
>= 0
914 : transfer_fsck_objects
>= 0
915 ? transfer_fsck_objects
919 if (do_keep
|| args
->from_promisor
|| index_pack_args
|| fsck_objects
) {
920 if (pack_lockfiles
|| fsck_objects
)
922 cmd_name
= "index-pack";
923 strvec_push(&cmd
.args
, cmd_name
);
924 strvec_push(&cmd
.args
, "--stdin");
925 if (!args
->quiet
&& !args
->no_progress
)
926 strvec_push(&cmd
.args
, "-v");
927 if (args
->use_thin_pack
)
928 strvec_push(&cmd
.args
, "--fix-thin");
929 if ((do_keep
|| index_pack_args
) && (args
->lock_pack
|| unpack_limit
))
930 add_index_pack_keep_option(&cmd
.args
);
931 if (!index_pack_args
&& args
->check_self_contained_and_connected
)
932 strvec_push(&cmd
.args
, "--check-self-contained-and-connected");
935 * We cannot perform any connectivity checks because
936 * not all packs have been downloaded; let the caller
937 * have this responsibility.
939 args
->check_self_contained_and_connected
= 0;
941 if (args
->from_promisor
)
943 * create_promisor_file() may be called afterwards but
944 * we still need index-pack to know that this is a
945 * promisor pack. For example, if transfer.fsckobjects
946 * is true, index-pack needs to know that .gitmodules
947 * is a promisor object (so that it won't complain if
950 strvec_push(&cmd
.args
, "--promisor");
953 cmd_name
= "unpack-objects";
954 strvec_push(&cmd
.args
, cmd_name
);
955 if (args
->quiet
|| args
->no_progress
)
956 strvec_push(&cmd
.args
, "-q");
957 args
->check_self_contained_and_connected
= 0;
961 strvec_pushf(&cmd
.args
, "--pack_header=%"PRIu32
",%"PRIu32
,
962 ntohl(header
.hdr_version
),
963 ntohl(header
.hdr_entries
));
965 if (args
->from_promisor
|| index_pack_args
)
967 * We cannot use --strict in index-pack because it
968 * checks both broken objects and links, but we only
969 * want to check for broken objects.
971 strvec_push(&cmd
.args
, "--fsck-objects");
973 strvec_pushf(&cmd
.args
, "--strict%s",
977 if (index_pack_args
) {
980 for (i
= 0; i
< cmd
.args
.nr
; i
++)
981 strvec_push(index_pack_args
, cmd
.args
.v
[i
]);
984 sigchain_push(SIGPIPE
, SIG_IGN
);
988 if (start_command(&cmd
))
989 die(_("fetch-pack: unable to fork off %s"), cmd_name
);
990 if (do_keep
&& (pack_lockfiles
|| fsck_objects
)) {
992 char *pack_lockfile
= index_pack_lockfile(cmd
.out
, &is_well_formed
);
995 die(_("fetch-pack: invalid index-pack output"));
997 string_list_append_nodup(pack_lockfiles
, pack_lockfile
);
998 parse_gitmodules_oids(cmd
.out
, gitmodules_oids
);
1003 /* Closed by start_command() */
1006 ret
= finish_command(&cmd
);
1007 if (!ret
|| (args
->check_self_contained_and_connected
&& ret
== 1))
1008 args
->self_contained_and_connected
=
1009 args
->check_self_contained_and_connected
&&
1012 die(_("%s failed"), cmd_name
);
1013 if (use_sideband
&& finish_async(&demux
))
1014 die(_("error in sideband demultiplexer"));
1016 sigchain_pop(SIGPIPE
);
1019 * Now that index-pack has succeeded, write the promisor file using the
1020 * obtained .keep filename if necessary
1022 if (do_keep
&& pack_lockfiles
&& pack_lockfiles
->nr
&& args
->from_promisor
)
1023 create_promisor_file(pack_lockfiles
->items
[0].string
, sought
, nr_sought
);
1028 static int cmp_ref_by_name(const void *a_
, const void *b_
)
1030 const struct ref
*a
= *((const struct ref
**)a_
);
1031 const struct ref
*b
= *((const struct ref
**)b_
);
1032 return strcmp(a
->name
, b
->name
);
1035 static struct ref
*do_fetch_pack(struct fetch_pack_args
*args
,
1037 const struct ref
*orig_ref
,
1038 struct ref
**sought
, int nr_sought
,
1039 struct shallow_info
*si
,
1040 struct string_list
*pack_lockfiles
)
1042 struct repository
*r
= the_repository
;
1043 struct ref
*ref
= copy_ref_list(orig_ref
);
1044 struct object_id oid
;
1045 const char *agent_feature
;
1047 struct fetch_negotiator negotiator_alloc
;
1048 struct fetch_negotiator
*negotiator
;
1050 negotiator
= &negotiator_alloc
;
1051 if (args
->refetch
) {
1052 fetch_negotiator_init_noop(negotiator
);
1054 fetch_negotiator_init(r
, negotiator
);
1057 sort_ref_list(&ref
, ref_compare_name
);
1058 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
1060 if ((agent_feature
= server_feature_value("agent", &agent_len
))) {
1061 agent_supported
= 1;
1063 print_verbose(args
, _("Server version is %.*s"),
1064 agent_len
, agent_feature
);
1067 if (!server_supports("session-id"))
1070 if (server_supports("shallow"))
1071 print_verbose(args
, _("Server supports %s"), "shallow");
1072 else if (args
->depth
> 0 || is_repository_shallow(r
))
1073 die(_("Server does not support shallow clients"));
1074 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
1076 if (server_supports("multi_ack_detailed")) {
1077 print_verbose(args
, _("Server supports %s"), "multi_ack_detailed");
1079 if (server_supports("no-done")) {
1080 print_verbose(args
, _("Server supports %s"), "no-done");
1081 if (args
->stateless_rpc
)
1085 else if (server_supports("multi_ack")) {
1086 print_verbose(args
, _("Server supports %s"), "multi_ack");
1089 if (server_supports("side-band-64k")) {
1090 print_verbose(args
, _("Server supports %s"), "side-band-64k");
1093 else if (server_supports("side-band")) {
1094 print_verbose(args
, _("Server supports %s"), "side-band");
1097 if (server_supports("allow-tip-sha1-in-want")) {
1098 print_verbose(args
, _("Server supports %s"), "allow-tip-sha1-in-want");
1099 allow_unadvertised_object_request
|= ALLOW_TIP_SHA1
;
1101 if (server_supports("allow-reachable-sha1-in-want")) {
1102 print_verbose(args
, _("Server supports %s"), "allow-reachable-sha1-in-want");
1103 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1105 if (server_supports("thin-pack"))
1106 print_verbose(args
, _("Server supports %s"), "thin-pack");
1108 args
->use_thin_pack
= 0;
1109 if (server_supports("no-progress"))
1110 print_verbose(args
, _("Server supports %s"), "no-progress");
1112 args
->no_progress
= 0;
1113 if (server_supports("include-tag"))
1114 print_verbose(args
, _("Server supports %s"), "include-tag");
1116 args
->include_tag
= 0;
1117 if (server_supports("ofs-delta"))
1118 print_verbose(args
, _("Server supports %s"), "ofs-delta");
1120 prefer_ofs_delta
= 0;
1122 if (server_supports("filter")) {
1123 server_supports_filtering
= 1;
1124 print_verbose(args
, _("Server supports %s"), "filter");
1125 } else if (args
->filter_options
.choice
) {
1126 warning("filtering not recognized by server, ignoring");
1129 if (server_supports("deepen-since")) {
1130 print_verbose(args
, _("Server supports %s"), "deepen-since");
1131 deepen_since_ok
= 1;
1132 } else if (args
->deepen_since
)
1133 die(_("Server does not support --shallow-since"));
1134 if (server_supports("deepen-not")) {
1135 print_verbose(args
, _("Server supports %s"), "deepen-not");
1137 } else if (args
->deepen_not
)
1138 die(_("Server does not support --shallow-exclude"));
1139 if (server_supports("deepen-relative"))
1140 print_verbose(args
, _("Server supports %s"), "deepen-relative");
1141 else if (args
->deepen_relative
)
1142 die(_("Server does not support --deepen"));
1143 if (!server_supports_hash(the_hash_algo
->name
, NULL
))
1144 die(_("Server does not support this repository's object format"));
1146 mark_complete_and_common_ref(negotiator
, args
, &ref
);
1147 filter_refs(args
, &ref
, sought
, nr_sought
);
1148 if (!args
->refetch
&& everything_local(args
, &ref
)) {
1149 packet_flush(fd
[1]);
1152 if (find_common(negotiator
, args
, fd
, &oid
, ref
) < 0)
1153 if (!args
->keep_pack
)
1154 /* When cloning, it is not unusual to have
1157 warning(_("no common commits"));
1159 if (args
->stateless_rpc
)
1160 packet_flush(fd
[1]);
1162 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
,
1164 else if (si
->nr_ours
|| si
->nr_theirs
) {
1165 if (args
->reject_shallow_remote
)
1166 die(_("source repository is shallow, reject to clone."));
1167 alternate_shallow_file
= setup_temporary_shallow(si
->shallow
);
1169 alternate_shallow_file
= NULL
;
1170 if (get_pack(args
, fd
, pack_lockfiles
, NULL
, sought
, nr_sought
,
1171 &fsck_options
.gitmodules_found
))
1172 die(_("git fetch-pack: fetch failed."));
1173 if (fsck_finish(&fsck_options
))
1178 negotiator
->release(negotiator
);
1182 static void add_shallow_requests(struct strbuf
*req_buf
,
1183 const struct fetch_pack_args
*args
)
1185 if (is_repository_shallow(the_repository
))
1186 write_shallow_commits(req_buf
, 1, NULL
);
1187 if (args
->depth
> 0)
1188 packet_buf_write(req_buf
, "deepen %d", args
->depth
);
1189 if (args
->deepen_since
) {
1190 timestamp_t max_age
= approxidate(args
->deepen_since
);
1191 packet_buf_write(req_buf
, "deepen-since %"PRItime
, max_age
);
1193 if (args
->deepen_not
) {
1195 for (i
= 0; i
< args
->deepen_not
->nr
; i
++) {
1196 struct string_list_item
*s
= args
->deepen_not
->items
+ i
;
1197 packet_buf_write(req_buf
, "deepen-not %s", s
->string
);
1200 if (args
->deepen_relative
)
1201 packet_buf_write(req_buf
, "deepen-relative\n");
1204 static void add_wants(const struct ref
*wants
, struct strbuf
*req_buf
)
1206 int use_ref_in_want
= server_supports_feature("fetch", "ref-in-want", 0);
1208 for ( ; wants
; wants
= wants
->next
) {
1209 const struct object_id
*remote
= &wants
->old_oid
;
1213 * If that object is complete (i.e. it is an ancestor of a
1214 * local ref), we tell them we have it but do not have to
1215 * tell them about its ancestors, which they already know
1218 * We use lookup_object here because we are only
1219 * interested in the case we *know* the object is
1220 * reachable and we have already scanned it.
1222 if (((o
= lookup_object(the_repository
, remote
)) != NULL
) &&
1223 (o
->flags
& COMPLETE
)) {
1227 if (!use_ref_in_want
|| wants
->exact_oid
)
1228 packet_buf_write(req_buf
, "want %s\n", oid_to_hex(remote
));
1230 packet_buf_write(req_buf
, "want-ref %s\n", wants
->name
);
1234 static void add_common(struct strbuf
*req_buf
, struct oidset
*common
)
1236 struct oidset_iter iter
;
1237 const struct object_id
*oid
;
1238 oidset_iter_init(common
, &iter
);
1240 while ((oid
= oidset_iter_next(&iter
))) {
1241 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1245 static int add_haves(struct fetch_negotiator
*negotiator
,
1246 struct strbuf
*req_buf
,
1249 int haves_added
= 0;
1250 const struct object_id
*oid
;
1252 while ((oid
= negotiator
->next(negotiator
))) {
1253 packet_buf_write(req_buf
, "have %s\n", oid_to_hex(oid
));
1254 if (++haves_added
>= *haves_to_send
)
1258 /* Increase haves to send on next round */
1259 *haves_to_send
= next_flush(1, *haves_to_send
);
1264 static void write_fetch_command_and_capabilities(struct strbuf
*req_buf
,
1265 const struct string_list
*server_options
)
1267 const char *hash_name
;
1269 if (server_supports_v2("fetch", 1))
1270 packet_buf_write(req_buf
, "command=fetch");
1271 if (server_supports_v2("agent", 0))
1272 packet_buf_write(req_buf
, "agent=%s", git_user_agent_sanitized());
1273 if (advertise_sid
&& server_supports_v2("session-id", 0))
1274 packet_buf_write(req_buf
, "session-id=%s", trace2_session_id());
1275 if (server_options
&& server_options
->nr
&&
1276 server_supports_v2("server-option", 1)) {
1278 for (i
= 0; i
< server_options
->nr
; i
++)
1279 packet_buf_write(req_buf
, "server-option=%s",
1280 server_options
->items
[i
].string
);
1283 if (server_feature_v2("object-format", &hash_name
)) {
1284 int hash_algo
= hash_algo_by_name(hash_name
);
1285 if (hash_algo_by_ptr(the_hash_algo
) != hash_algo
)
1286 die(_("mismatched algorithms: client %s; server %s"),
1287 the_hash_algo
->name
, hash_name
);
1288 packet_buf_write(req_buf
, "object-format=%s", the_hash_algo
->name
);
1289 } else if (hash_algo_by_ptr(the_hash_algo
) != GIT_HASH_SHA1
) {
1290 die(_("the server does not support algorithm '%s'"),
1291 the_hash_algo
->name
);
1293 packet_buf_delim(req_buf
);
1296 static int send_fetch_request(struct fetch_negotiator
*negotiator
, int fd_out
,
1297 struct fetch_pack_args
*args
,
1298 const struct ref
*wants
, struct oidset
*common
,
1299 int *haves_to_send
, int *in_vain
,
1300 int sideband_all
, int seen_ack
)
1304 struct strbuf req_buf
= STRBUF_INIT
;
1306 write_fetch_command_and_capabilities(&req_buf
, args
->server_options
);
1308 if (args
->use_thin_pack
)
1309 packet_buf_write(&req_buf
, "thin-pack");
1310 if (args
->no_progress
)
1311 packet_buf_write(&req_buf
, "no-progress");
1312 if (args
->include_tag
)
1313 packet_buf_write(&req_buf
, "include-tag");
1314 if (prefer_ofs_delta
)
1315 packet_buf_write(&req_buf
, "ofs-delta");
1317 packet_buf_write(&req_buf
, "sideband-all");
1319 /* Add shallow-info and deepen request */
1320 if (server_supports_feature("fetch", "shallow", 0))
1321 add_shallow_requests(&req_buf
, args
);
1322 else if (is_repository_shallow(the_repository
) || args
->deepen
)
1323 die(_("Server does not support shallow requests"));
1326 if (server_supports_feature("fetch", "filter", 0) &&
1327 args
->filter_options
.choice
) {
1329 expand_list_objects_filter_spec(&args
->filter_options
);
1330 print_verbose(args
, _("Server supports filter"));
1331 packet_buf_write(&req_buf
, "filter %s", spec
);
1332 } else if (args
->filter_options
.choice
) {
1333 warning("filtering not recognized by server, ignoring");
1336 if (server_supports_feature("fetch", "packfile-uris", 0)) {
1338 struct strbuf to_send
= STRBUF_INIT
;
1340 for (i
= 0; i
< uri_protocols
.nr
; i
++) {
1341 const char *s
= uri_protocols
.items
[i
].string
;
1343 if (!strcmp(s
, "https") || !strcmp(s
, "http")) {
1345 strbuf_addch(&to_send
, ',');
1346 strbuf_addstr(&to_send
, s
);
1350 packet_buf_write(&req_buf
, "packfile-uris %s",
1352 strbuf_release(&to_send
);
1357 add_wants(wants
, &req_buf
);
1359 /* Add all of the common commits we've found in previous rounds */
1360 add_common(&req_buf
, common
);
1362 haves_added
= add_haves(negotiator
, &req_buf
, haves_to_send
);
1363 *in_vain
+= haves_added
;
1364 if (!haves_added
|| (seen_ack
&& *in_vain
>= MAX_IN_VAIN
)) {
1366 packet_buf_write(&req_buf
, "done\n");
1371 packet_buf_flush(&req_buf
);
1372 if (write_in_full(fd_out
, req_buf
.buf
, req_buf
.len
) < 0)
1373 die_errno(_("unable to write request to remote"));
1375 strbuf_release(&req_buf
);
1380 * Processes a section header in a server's response and checks if it matches
1381 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1382 * not consumed); if 0, the line will be consumed and the function will die if
1383 * the section header doesn't match what was expected.
1385 static int process_section_header(struct packet_reader
*reader
,
1386 const char *section
, int peek
)
1390 if (packet_reader_peek(reader
) == PACKET_READ_NORMAL
&&
1391 !strcmp(reader
->line
, section
))
1397 die(_("expected '%s', received '%s'"),
1398 section
, reader
->line
);
1400 die(_("expected '%s'"), section
);
1402 packet_reader_read(reader
);
1408 static int process_ack(struct fetch_negotiator
*negotiator
,
1409 struct packet_reader
*reader
,
1410 struct object_id
*common_oid
,
1411 int *received_ready
)
1413 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1416 if (!strcmp(reader
->line
, "NAK"))
1419 if (skip_prefix(reader
->line
, "ACK ", &arg
)) {
1420 if (!get_oid_hex(arg
, common_oid
)) {
1421 struct commit
*commit
;
1422 commit
= lookup_commit(the_repository
, common_oid
);
1424 negotiator
->ack(negotiator
, commit
);
1429 if (!strcmp(reader
->line
, "ready")) {
1430 *received_ready
= 1;
1434 die(_("unexpected acknowledgment line: '%s'"), reader
->line
);
1437 if (reader
->status
!= PACKET_READ_FLUSH
&&
1438 reader
->status
!= PACKET_READ_DELIM
)
1439 die(_("error processing acks: %d"), reader
->status
);
1442 * If an "acknowledgments" section is sent, a packfile is sent if and
1443 * only if "ready" was sent in this section. The other sections
1444 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1445 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1448 if (*received_ready
&& reader
->status
!= PACKET_READ_DELIM
)
1450 * TRANSLATORS: The parameter will be 'ready', a protocol
1453 die(_("expected packfile to be sent after '%s'"), "ready");
1454 if (!*received_ready
&& reader
->status
!= PACKET_READ_FLUSH
)
1456 * TRANSLATORS: The parameter will be 'ready', a protocol
1459 die(_("expected no other sections to be sent after no '%s'"), "ready");
1464 static void receive_shallow_info(struct fetch_pack_args
*args
,
1465 struct packet_reader
*reader
,
1466 struct oid_array
*shallows
,
1467 struct shallow_info
*si
)
1469 int unshallow_received
= 0;
1471 process_section_header(reader
, "shallow-info", 0);
1472 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1474 struct object_id oid
;
1476 if (skip_prefix(reader
->line
, "shallow ", &arg
)) {
1477 if (get_oid_hex(arg
, &oid
))
1478 die(_("invalid shallow line: %s"), reader
->line
);
1479 oid_array_append(shallows
, &oid
);
1482 if (skip_prefix(reader
->line
, "unshallow ", &arg
)) {
1483 if (get_oid_hex(arg
, &oid
))
1484 die(_("invalid unshallow line: %s"), reader
->line
);
1485 if (!lookup_object(the_repository
, &oid
))
1486 die(_("object not found: %s"), reader
->line
);
1487 /* make sure that it is parsed as shallow */
1488 if (!parse_object(the_repository
, &oid
))
1489 die(_("error in object: %s"), reader
->line
);
1490 if (unregister_shallow(&oid
))
1491 die(_("no shallow found: %s"), reader
->line
);
1492 unshallow_received
= 1;
1495 die(_("expected shallow/unshallow, got %s"), reader
->line
);
1498 if (reader
->status
!= PACKET_READ_FLUSH
&&
1499 reader
->status
!= PACKET_READ_DELIM
)
1500 die(_("error processing shallow info: %d"), reader
->status
);
1502 if (args
->deepen
|| unshallow_received
) {
1504 * Treat these as shallow lines caused by our depth settings.
1505 * In v0, these lines cannot cause refs to be rejected; do the
1510 for (i
= 0; i
< shallows
->nr
; i
++)
1511 register_shallow(the_repository
, &shallows
->oid
[i
]);
1512 setup_alternate_shallow(&shallow_lock
, &alternate_shallow_file
,
1515 } else if (shallows
->nr
) {
1517 * Treat these as shallow lines caused by the remote being
1518 * shallow. In v0, remote refs that reach these objects are
1519 * rejected (unless --update-shallow is set); do the same.
1521 prepare_shallow_info(si
, shallows
);
1522 if (si
->nr_ours
|| si
->nr_theirs
) {
1523 if (args
->reject_shallow_remote
)
1524 die(_("source repository is shallow, reject to clone."));
1525 alternate_shallow_file
=
1526 setup_temporary_shallow(si
->shallow
);
1528 alternate_shallow_file
= NULL
;
1530 alternate_shallow_file
= NULL
;
1534 static int cmp_name_ref(const void *name
, const void *ref
)
1536 return strcmp(name
, (*(struct ref
**)ref
)->name
);
1539 static void receive_wanted_refs(struct packet_reader
*reader
,
1540 struct ref
**sought
, int nr_sought
)
1542 process_section_header(reader
, "wanted-refs", 0);
1543 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1544 struct object_id oid
;
1548 if (parse_oid_hex(reader
->line
, &oid
, &end
) || *end
++ != ' ')
1549 die(_("expected wanted-ref, got '%s'"), reader
->line
);
1551 found
= bsearch(end
, sought
, nr_sought
, sizeof(*sought
),
1554 die(_("unexpected wanted-ref: '%s'"), reader
->line
);
1555 oidcpy(&(*found
)->old_oid
, &oid
);
1558 if (reader
->status
!= PACKET_READ_DELIM
)
1559 die(_("error processing wanted refs: %d"), reader
->status
);
1562 static void receive_packfile_uris(struct packet_reader
*reader
,
1563 struct string_list
*uris
)
1565 process_section_header(reader
, "packfile-uris", 0);
1566 while (packet_reader_read(reader
) == PACKET_READ_NORMAL
) {
1567 if (reader
->pktlen
< the_hash_algo
->hexsz
||
1568 reader
->line
[the_hash_algo
->hexsz
] != ' ')
1569 die("expected '<hash> <uri>', got: %s\n", reader
->line
);
1571 string_list_append(uris
, reader
->line
);
1573 if (reader
->status
!= PACKET_READ_DELIM
)
1574 die("expected DELIM");
1578 FETCH_CHECK_LOCAL
= 0,
1585 static void do_check_stateless_delimiter(int stateless_rpc
,
1586 struct packet_reader
*reader
)
1588 check_stateless_delimiter(stateless_rpc
, reader
,
1589 _("git fetch-pack: expected response end packet"));
1592 static struct ref
*do_fetch_pack_v2(struct fetch_pack_args
*args
,
1594 const struct ref
*orig_ref
,
1595 struct ref
**sought
, int nr_sought
,
1596 struct oid_array
*shallows
,
1597 struct shallow_info
*si
,
1598 struct string_list
*pack_lockfiles
)
1600 struct repository
*r
= the_repository
;
1601 struct ref
*ref
= copy_ref_list(orig_ref
);
1602 enum fetch_state state
= FETCH_CHECK_LOCAL
;
1603 struct oidset common
= OIDSET_INIT
;
1604 struct packet_reader reader
;
1605 int in_vain
= 0, negotiation_started
= 0;
1606 int haves_to_send
= INITIAL_FLUSH
;
1607 struct fetch_negotiator negotiator_alloc
;
1608 struct fetch_negotiator
*negotiator
;
1610 struct object_id common_oid
;
1611 int received_ready
= 0;
1612 struct string_list packfile_uris
= STRING_LIST_INIT_DUP
;
1614 struct strvec index_pack_args
= STRVEC_INIT
;
1616 negotiator
= &negotiator_alloc
;
1618 fetch_negotiator_init_noop(negotiator
);
1620 fetch_negotiator_init(r
, negotiator
);
1622 packet_reader_init(&reader
, fd
[0], NULL
, 0,
1623 PACKET_READ_CHOMP_NEWLINE
|
1624 PACKET_READ_DIE_ON_ERR_PACKET
);
1625 if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1626 server_supports_feature("fetch", "sideband-all", 0)) {
1627 reader
.use_sideband
= 1;
1628 reader
.me
= "fetch-pack";
1631 while (state
!= FETCH_DONE
) {
1633 case FETCH_CHECK_LOCAL
:
1634 sort_ref_list(&ref
, ref_compare_name
);
1635 QSORT(sought
, nr_sought
, cmp_ref_by_name
);
1637 /* v2 supports these by default */
1638 allow_unadvertised_object_request
|= ALLOW_REACHABLE_SHA1
;
1640 if (args
->depth
> 0 || args
->deepen_since
|| args
->deepen_not
)
1643 /* Filter 'ref' by 'sought' and those that aren't local */
1644 mark_complete_and_common_ref(negotiator
, args
, &ref
);
1645 filter_refs(args
, &ref
, sought
, nr_sought
);
1646 if (!args
->refetch
&& everything_local(args
, &ref
))
1649 state
= FETCH_SEND_REQUEST
;
1651 mark_tips(negotiator
, args
->negotiation_tips
);
1652 for_each_cached_alternate(negotiator
,
1653 insert_one_alternate_object
);
1655 case FETCH_SEND_REQUEST
:
1656 if (!negotiation_started
) {
1657 negotiation_started
= 1;
1658 trace2_region_enter("fetch-pack",
1662 if (send_fetch_request(negotiator
, fd
[1], args
, ref
,
1664 &haves_to_send
, &in_vain
,
1665 reader
.use_sideband
,
1667 state
= FETCH_GET_PACK
;
1669 state
= FETCH_PROCESS_ACKS
;
1671 case FETCH_PROCESS_ACKS
:
1672 /* Process ACKs/NAKs */
1673 process_section_header(&reader
, "acknowledgments", 0);
1674 while (process_ack(negotiator
, &reader
, &common_oid
,
1678 oidset_insert(&common
, &common_oid
);
1680 if (received_ready
) {
1682 * Don't check for response delimiter; get_pack() will
1683 * read the rest of this response.
1685 state
= FETCH_GET_PACK
;
1687 do_check_stateless_delimiter(args
->stateless_rpc
, &reader
);
1688 state
= FETCH_SEND_REQUEST
;
1691 case FETCH_GET_PACK
:
1692 trace2_region_leave("fetch-pack",
1695 /* Check for shallow-info section */
1696 if (process_section_header(&reader
, "shallow-info", 1))
1697 receive_shallow_info(args
, &reader
, shallows
, si
);
1699 if (process_section_header(&reader
, "wanted-refs", 1))
1700 receive_wanted_refs(&reader
, sought
, nr_sought
);
1702 /* get the pack(s) */
1703 if (git_env_bool("GIT_TRACE_REDACT", 1))
1704 reader
.options
|= PACKET_READ_REDACT_URI_PATH
;
1705 if (process_section_header(&reader
, "packfile-uris", 1))
1706 receive_packfile_uris(&reader
, &packfile_uris
);
1707 /* We don't expect more URIs. Reset to avoid expensive URI check. */
1708 reader
.options
&= ~PACKET_READ_REDACT_URI_PATH
;
1710 process_section_header(&reader
, "packfile", 0);
1713 * this is the final request we'll make of the server;
1714 * do a half-duplex shutdown to indicate that they can
1715 * hang up as soon as the pack is sent.
1720 if (get_pack(args
, fd
, pack_lockfiles
,
1721 packfile_uris
.nr
? &index_pack_args
: NULL
,
1722 sought
, nr_sought
, &fsck_options
.gitmodules_found
))
1723 die(_("git fetch-pack: fetch failed."));
1724 do_check_stateless_delimiter(args
->stateless_rpc
, &reader
);
1733 for (i
= 0; i
< packfile_uris
.nr
; i
++) {
1735 struct child_process cmd
= CHILD_PROCESS_INIT
;
1736 char packname
[GIT_MAX_HEXSZ
+ 1];
1737 const char *uri
= packfile_uris
.items
[i
].string
+
1738 the_hash_algo
->hexsz
+ 1;
1740 strvec_push(&cmd
.args
, "http-fetch");
1741 strvec_pushf(&cmd
.args
, "--packfile=%.*s",
1742 (int) the_hash_algo
->hexsz
,
1743 packfile_uris
.items
[i
].string
);
1744 for (j
= 0; j
< index_pack_args
.nr
; j
++)
1745 strvec_pushf(&cmd
.args
, "--index-pack-arg=%s",
1746 index_pack_args
.v
[j
]);
1747 strvec_push(&cmd
.args
, uri
);
1751 if (start_command(&cmd
))
1752 die("fetch-pack: unable to spawn http-fetch");
1754 if (read_in_full(cmd
.out
, packname
, 5) < 0 ||
1755 memcmp(packname
, "keep\t", 5))
1756 die("fetch-pack: expected keep then TAB at start of http-fetch output");
1758 if (read_in_full(cmd
.out
, packname
,
1759 the_hash_algo
->hexsz
+ 1) < 0 ||
1760 packname
[the_hash_algo
->hexsz
] != '\n')
1761 die("fetch-pack: expected hash then LF at end of http-fetch output");
1763 packname
[the_hash_algo
->hexsz
] = '\0';
1765 parse_gitmodules_oids(cmd
.out
, &fsck_options
.gitmodules_found
);
1769 if (finish_command(&cmd
))
1770 die("fetch-pack: unable to finish http-fetch");
1772 if (memcmp(packfile_uris
.items
[i
].string
, packname
,
1773 the_hash_algo
->hexsz
))
1774 die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1775 uri
, (int) the_hash_algo
->hexsz
,
1776 packfile_uris
.items
[i
].string
);
1778 string_list_append_nodup(pack_lockfiles
,
1779 xstrfmt("%s/pack/pack-%s.keep",
1780 get_object_directory(),
1783 string_list_clear(&packfile_uris
, 0);
1784 strvec_clear(&index_pack_args
);
1786 if (fsck_finish(&fsck_options
))
1790 negotiator
->release(negotiator
);
1792 oidset_clear(&common
);
1796 static int fetch_pack_config_cb(const char *var
, const char *value
, void *cb
)
1798 if (strcmp(var
, "fetch.fsck.skiplist") == 0) {
1801 if (git_config_pathname(&path
, var
, value
))
1803 strbuf_addf(&fsck_msg_types
, "%cskiplist=%s",
1804 fsck_msg_types
.len
? ',' : '=', path
);
1809 if (skip_prefix(var
, "fetch.fsck.", &var
)) {
1810 if (is_valid_msg_type(var
, value
))
1811 strbuf_addf(&fsck_msg_types
, "%c%s=%s",
1812 fsck_msg_types
.len
? ',' : '=', var
, value
);
1814 warning("Skipping unknown msg id '%s'", var
);
1818 return git_default_config(var
, value
, cb
);
1821 static void fetch_pack_config(void)
1823 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit
);
1824 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit
);
1825 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta
);
1826 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects
);
1827 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects
);
1828 git_config_get_bool("transfer.advertisesid", &advertise_sid
);
1829 if (!uri_protocols
.nr
) {
1832 if (!git_config_get_string("fetch.uriprotocols", &str
) && str
) {
1833 string_list_split(&uri_protocols
, str
, ',', -1);
1838 git_config(fetch_pack_config_cb
, NULL
);
1841 static void fetch_pack_setup(void)
1843 static int did_setup
;
1846 fetch_pack_config();
1847 if (0 <= transfer_unpack_limit
)
1848 unpack_limit
= transfer_unpack_limit
;
1849 else if (0 <= fetch_unpack_limit
)
1850 unpack_limit
= fetch_unpack_limit
;
1854 static int remove_duplicates_in_refs(struct ref
**ref
, int nr
)
1856 struct string_list names
= STRING_LIST_INIT_NODUP
;
1859 for (src
= dst
= 0; src
< nr
; src
++) {
1860 struct string_list_item
*item
;
1861 item
= string_list_insert(&names
, ref
[src
]->name
);
1863 continue; /* already have it */
1864 item
->util
= ref
[src
];
1866 ref
[dst
] = ref
[src
];
1869 for (src
= dst
; src
< nr
; src
++)
1871 string_list_clear(&names
, 0);
1875 static void update_shallow(struct fetch_pack_args
*args
,
1876 struct ref
**sought
, int nr_sought
,
1877 struct shallow_info
*si
)
1879 struct oid_array ref
= OID_ARRAY_INIT
;
1883 if (args
->deepen
&& alternate_shallow_file
) {
1884 if (*alternate_shallow_file
== '\0') { /* --unshallow */
1885 unlink_or_warn(git_path_shallow(the_repository
));
1886 rollback_shallow_file(the_repository
, &shallow_lock
);
1888 commit_shallow_file(the_repository
, &shallow_lock
);
1889 alternate_shallow_file
= NULL
;
1893 if (!si
->shallow
|| !si
->shallow
->nr
)
1896 if (args
->cloning
) {
1898 * remote is shallow, but this is a clone, there are
1899 * no objects in repo to worry about. Accept any
1900 * shallow points that exist in the pack (iow in repo
1901 * after get_pack() and reprepare_packed_git())
1903 struct oid_array extra
= OID_ARRAY_INIT
;
1904 struct object_id
*oid
= si
->shallow
->oid
;
1905 for (i
= 0; i
< si
->shallow
->nr
; i
++)
1906 if (has_object_file(&oid
[i
]))
1907 oid_array_append(&extra
, &oid
[i
]);
1909 setup_alternate_shallow(&shallow_lock
,
1910 &alternate_shallow_file
,
1912 commit_shallow_file(the_repository
, &shallow_lock
);
1913 alternate_shallow_file
= NULL
;
1915 oid_array_clear(&extra
);
1919 if (!si
->nr_ours
&& !si
->nr_theirs
)
1922 remove_nonexistent_theirs_shallow(si
);
1923 if (!si
->nr_ours
&& !si
->nr_theirs
)
1925 for (i
= 0; i
< nr_sought
; i
++)
1926 oid_array_append(&ref
, &sought
[i
]->old_oid
);
1929 if (args
->update_shallow
) {
1931 * remote is also shallow, .git/shallow may be updated
1932 * so all refs can be accepted. Make sure we only add
1933 * shallow roots that are actually reachable from new
1936 struct oid_array extra
= OID_ARRAY_INIT
;
1937 struct object_id
*oid
= si
->shallow
->oid
;
1938 assign_shallow_commits_to_refs(si
, NULL
, NULL
);
1939 if (!si
->nr_ours
&& !si
->nr_theirs
) {
1940 oid_array_clear(&ref
);
1943 for (i
= 0; i
< si
->nr_ours
; i
++)
1944 oid_array_append(&extra
, &oid
[si
->ours
[i
]]);
1945 for (i
= 0; i
< si
->nr_theirs
; i
++)
1946 oid_array_append(&extra
, &oid
[si
->theirs
[i
]]);
1947 setup_alternate_shallow(&shallow_lock
,
1948 &alternate_shallow_file
,
1950 commit_shallow_file(the_repository
, &shallow_lock
);
1951 oid_array_clear(&extra
);
1952 oid_array_clear(&ref
);
1953 alternate_shallow_file
= NULL
;
1958 * remote is also shallow, check what ref is safe to update
1959 * without updating .git/shallow
1961 CALLOC_ARRAY(status
, nr_sought
);
1962 assign_shallow_commits_to_refs(si
, NULL
, status
);
1963 if (si
->nr_ours
|| si
->nr_theirs
) {
1964 for (i
= 0; i
< nr_sought
; i
++)
1966 sought
[i
]->status
= REF_STATUS_REJECT_SHALLOW
;
1969 oid_array_clear(&ref
);
1972 static const struct object_id
*iterate_ref_map(void *cb_data
)
1974 struct ref
**rm
= cb_data
;
1975 struct ref
*ref
= *rm
;
1980 return &ref
->old_oid
;
1983 struct ref
*fetch_pack(struct fetch_pack_args
*args
,
1985 const struct ref
*ref
,
1986 struct ref
**sought
, int nr_sought
,
1987 struct oid_array
*shallow
,
1988 struct string_list
*pack_lockfiles
,
1989 enum protocol_version version
)
1991 struct ref
*ref_cpy
;
1992 struct shallow_info si
;
1993 struct oid_array shallows_scratch
= OID_ARRAY_INIT
;
1997 nr_sought
= remove_duplicates_in_refs(sought
, nr_sought
);
1999 if (version
!= protocol_v2
&& !ref
) {
2000 packet_flush(fd
[1]);
2001 die(_("no matching remote head"));
2003 if (version
== protocol_v2
) {
2005 BUG("Protocol V2 does not provide shallows at this point in the fetch");
2006 memset(&si
, 0, sizeof(si
));
2007 ref_cpy
= do_fetch_pack_v2(args
, fd
, ref
, sought
, nr_sought
,
2008 &shallows_scratch
, &si
,
2011 prepare_shallow_info(&si
, shallow
);
2012 ref_cpy
= do_fetch_pack(args
, fd
, ref
, sought
, nr_sought
,
2013 &si
, pack_lockfiles
);
2015 reprepare_packed_git(the_repository
);
2017 if (!args
->cloning
&& args
->deepen
) {
2018 struct check_connected_options opt
= CHECK_CONNECTED_INIT
;
2019 struct ref
*iterator
= ref_cpy
;
2020 opt
.shallow_file
= alternate_shallow_file
;
2022 opt
.is_deepening_fetch
= 1;
2023 if (check_connected(iterate_ref_map
, &iterator
, &opt
)) {
2024 error(_("remote did not send all necessary objects"));
2027 rollback_shallow_file(the_repository
, &shallow_lock
);
2030 args
->connectivity_checked
= 1;
2033 update_shallow(args
, sought
, nr_sought
, &si
);
2035 clear_shallow_info(&si
);
2036 oid_array_clear(&shallows_scratch
);
2040 static int add_to_object_array(const struct object_id
*oid
, void *data
)
2042 struct object_array
*a
= data
;
2044 add_object_array(lookup_object(the_repository
, oid
), "", a
);
2048 static void clear_common_flag(struct oidset
*s
)
2050 struct oidset_iter iter
;
2051 const struct object_id
*oid
;
2052 oidset_iter_init(s
, &iter
);
2054 while ((oid
= oidset_iter_next(&iter
))) {
2055 struct object
*obj
= lookup_object(the_repository
, oid
);
2056 obj
->flags
&= ~COMMON
;
2060 void negotiate_using_fetch(const struct oid_array
*negotiation_tips
,
2061 const struct string_list
*server_options
,
2064 struct oidset
*acked_commits
)
2066 struct fetch_negotiator negotiator
;
2067 struct packet_reader reader
;
2068 struct object_array nt_object_array
= OBJECT_ARRAY_INIT
;
2069 struct strbuf req_buf
= STRBUF_INIT
;
2070 int haves_to_send
= INITIAL_FLUSH
;
2073 int last_iteration
= 0;
2074 timestamp_t min_generation
= GENERATION_NUMBER_INFINITY
;
2076 fetch_negotiator_init(the_repository
, &negotiator
);
2077 mark_tips(&negotiator
, negotiation_tips
);
2079 packet_reader_init(&reader
, fd
[0], NULL
, 0,
2080 PACKET_READ_CHOMP_NEWLINE
|
2081 PACKET_READ_DIE_ON_ERR_PACKET
);
2083 oid_array_for_each((struct oid_array
*) negotiation_tips
,
2084 add_to_object_array
,
2087 while (!last_iteration
) {
2089 struct object_id common_oid
;
2090 int received_ready
= 0;
2092 strbuf_reset(&req_buf
);
2093 write_fetch_command_and_capabilities(&req_buf
, server_options
);
2095 packet_buf_write(&req_buf
, "wait-for-done");
2097 haves_added
= add_haves(&negotiator
, &req_buf
, &haves_to_send
);
2098 in_vain
+= haves_added
;
2099 if (!haves_added
|| (seen_ack
&& in_vain
>= MAX_IN_VAIN
))
2103 packet_buf_flush(&req_buf
);
2104 if (write_in_full(fd
[1], req_buf
.buf
, req_buf
.len
) < 0)
2105 die_errno(_("unable to write request to remote"));
2107 /* Process ACKs/NAKs */
2108 process_section_header(&reader
, "acknowledgments", 0);
2109 while (process_ack(&negotiator
, &reader
, &common_oid
,
2111 struct commit
*commit
= lookup_commit(the_repository
,
2114 timestamp_t generation
;
2116 parse_commit_or_die(commit
);
2117 commit
->object
.flags
|= COMMON
;
2118 generation
= commit_graph_generation(commit
);
2119 if (generation
< min_generation
)
2120 min_generation
= generation
;
2124 oidset_insert(acked_commits
, &common_oid
);
2127 die(_("unexpected 'ready' from remote"));
2129 do_check_stateless_delimiter(stateless_rpc
, &reader
);
2130 if (can_all_from_reach_with_flag(&nt_object_array
, COMMON
,
2135 clear_common_flag(acked_commits
);
2136 strbuf_release(&req_buf
);
2139 int report_unmatched_refs(struct ref
**sought
, int nr_sought
)
2143 for (i
= 0; i
< nr_sought
; i
++) {
2146 switch (sought
[i
]->match_status
) {
2149 case REF_NOT_MATCHED
:
2150 error(_("no such remote ref %s"), sought
[i
]->name
);
2152 case REF_UNADVERTISED_NOT_ALLOWED
:
2153 error(_("Server does not allow request for unadvertised object %s"),