]> git.ipfire.org Git - thirdparty/git.git/blame - fetch-pack.c
fetch-pack: redact packfile urls in traces
[thirdparty/git.git] / fetch-pack.c
CommitLineData
745f7a8c 1#include "cache.h"
a49d2834 2#include "repository.h"
b2141fc1 3#include "config.h"
697cc8ef 4#include "lockfile.h"
745f7a8c
NTND
5#include "refs.h"
6#include "pkt-line.h"
7#include "commit.h"
8#include "tag.h"
d807c4a0 9#include "exec-cmd.h"
745f7a8c
NTND
10#include "pack.h"
11#include "sideband.h"
12#include "fetch-pack.h"
13#include "remote.h"
14#include "run-command.h"
47a59185 15#include "connect.h"
745f7a8c
NTND
16#include "transport.h"
17#include "version.h"
fe299ec5 18#include "oid-array.h"
fdb69d33 19#include "oidset.h"
0abe14f6 20#include "packfile.h"
cbd53a21 21#include "object-store.h"
cf1e7c07 22#include "connected.h"
ec062838 23#include "fetch-negotiator.h"
1362df0d 24#include "fsck.h"
120ad2b0 25#include "shallow.h"
9c1e657a
JT
26#include "commit-reach.h"
27#include "commit-graph.h"
745f7a8c
NTND
28
29static int transfer_unpack_limit = -1;
30static int fetch_unpack_limit = -1;
31static int unpack_limit = 100;
32static int prefer_ofs_delta = 1;
33static int no_done;
508ea882 34static int deepen_since_ok;
a45a2600 35static int deepen_not_ok;
745f7a8c
NTND
36static int fetch_fsck_objects = -1;
37static int transfer_fsck_objects = -1;
38static int agent_supported;
640d8b72 39static int server_supports_filtering;
1e905bbc 40static int advertise_sid;
cac4b8e2 41static struct shallow_lock shallow_lock;
6035d6aa 42static const char *alternate_shallow_file;
3745e269 43static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
1362df0d 44static struct strbuf fsck_msg_types = STRBUF_INIT;
dd4b732d 45static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
745f7a8c 46
208acbfb 47/* Remember to update object flag allocation in object.h */
745f7a8c 48#define COMPLETE (1U << 0)
ec062838 49#define ALTERNATE (1U << 1)
9c1e657a
JT
50#define COMMON (1U << 6)
51#define REACH_SCRATCH (1U << 7)
745f7a8c
NTND
52
53/*
54 * After sending this many "have"s if we do not get any new ACK , we
55 * give up traversing our history.
56 */
57#define MAX_IN_VAIN 256
58
d30fe89c 59static int multi_ack, use_sideband;
7199c093
FM
60/* Allow specifying sha1 if it is a ref tip. */
61#define ALLOW_TIP_SHA1 01
68ee6289
FM
62/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
63#define ALLOW_REACHABLE_SHA1 02
7199c093 64static unsigned int allow_unadvertised_object_request;
745f7a8c 65
0d789a5b
NTND
66__attribute__((format (printf, 2, 3)))
67static inline void print_verbose(const struct fetch_pack_args *args,
68 const char *fmt, ...)
69{
70 va_list params;
71
72 if (!args->verbose)
73 return;
74
75 va_start(params, fmt);
76 vfprintf(stderr, fmt, params);
77 va_end(params);
78 fputc('\n', stderr);
79}
80
41a078c6
JK
81struct alternate_object_cache {
82 struct object **items;
83 size_t nr, alloc;
84};
85
bdf4276c 86static void cache_one_alternate(const struct object_id *oid,
41a078c6
JK
87 void *vcache)
88{
89 struct alternate_object_cache *cache = vcache;
109cd76d 90 struct object *obj = parse_object(the_repository, oid);
41a078c6
JK
91
92 if (!obj || (obj->flags & ALTERNATE))
93 return;
94
95 obj->flags |= ALTERNATE;
96 ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
97 cache->items[cache->nr++] = obj;
98}
99
ec062838
JT
100static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
101 void (*cb)(struct fetch_negotiator *,
d30fe89c 102 struct object *))
41a078c6
JK
103{
104 static int initialized;
105 static struct alternate_object_cache cache;
106 size_t i;
107
108 if (!initialized) {
109 for_each_alternate_ref(cache_one_alternate, &cache);
110 initialized = 1;
111 }
112
113 for (i = 0; i < cache.nr; i++)
ec062838 114 cb(negotiator, cache.items[i]);
745f7a8c
NTND
115}
116
5c3b801d
JT
117static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
118 int mark_tags_complete)
119{
120 enum object_type type;
121 struct object_info info = { .typep = &type };
62b5a35a
PS
122 struct commit *commit;
123
124 commit = lookup_commit_in_graph(the_repository, oid);
125 if (commit)
126 return commit;
5c3b801d
JT
127
128 while (1) {
129 if (oid_object_info_extended(the_repository, oid, &info,
130 OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK))
131 return NULL;
132 if (type == OBJ_TAG) {
133 struct tag *tag = (struct tag *)
134 parse_object(the_repository, oid);
135
136 if (!tag->tagged)
137 return NULL;
138 if (mark_tags_complete)
139 tag->object.flags |= COMPLETE;
140 oid = &tag->tagged->oid;
141 } else {
142 break;
143 }
144 }
3e5e6c6e
PS
145
146 if (type == OBJ_COMMIT) {
147 struct commit *commit = lookup_commit(the_repository, oid);
148 if (!commit || repo_parse_commit(the_repository, commit))
149 return NULL;
150 return commit;
151 }
152
5c3b801d
JT
153 return NULL;
154}
155
ec062838 156static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
d30fe89c 157 const struct object_id *oid)
745f7a8c 158{
5c3b801d 159 struct commit *c = deref_without_lazy_fetch(oid, 0);
745f7a8c 160
5c3b801d
JT
161 if (c)
162 negotiator->add_tip(negotiator, c);
745f7a8c
NTND
163 return 0;
164}
165
b1b49c6e
MH
166static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
167 int flag, void *cb_data)
745f7a8c 168{
5c3b801d 169 return rev_list_insert_ref(cb_data, oid);
745f7a8c
NTND
170}
171
172enum ack_type {
173 NAK = 0,
174 ACK,
175 ACK_continue,
176 ACK_common,
177 ACK_ready
178};
179
01f9ec64
MS
180static void consume_shallow_list(struct fetch_pack_args *args,
181 struct packet_reader *reader)
745f7a8c 182{
79891cb9 183 if (args->stateless_rpc && args->deepen) {
745f7a8c
NTND
184 /* If we sent a depth we will get back "duplicate"
185 * shallow and unshallow commands every time there
186 * is a block of have lines exchanged.
187 */
01f9ec64
MS
188 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
189 if (starts_with(reader->line, "shallow "))
745f7a8c 190 continue;
01f9ec64 191 if (starts_with(reader->line, "unshallow "))
745f7a8c 192 continue;
1dd73e20 193 die(_("git fetch-pack: expected shallow list"));
745f7a8c 194 }
01f9ec64
MS
195 if (reader->status != PACKET_READ_FLUSH)
196 die(_("git fetch-pack: expected a flush packet after shallow list"));
745f7a8c
NTND
197 }
198}
199
01f9ec64
MS
200static enum ack_type get_ack(struct packet_reader *reader,
201 struct object_id *result_oid)
745f7a8c 202{
74543a04 203 int len;
82e56767 204 const char *arg;
745f7a8c 205
01f9ec64 206 if (packet_reader_read(reader) != PACKET_READ_NORMAL)
bc9d4dc5 207 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
01f9ec64
MS
208 len = reader->pktlen;
209
210 if (!strcmp(reader->line, "NAK"))
745f7a8c 211 return NAK;
01f9ec64 212 if (skip_prefix(reader->line, "ACK ", &arg)) {
f6af19a9 213 const char *p;
214 if (!parse_oid_hex(arg, result_oid, &p)) {
215 len -= p - reader->line;
82e56767 216 if (len < 1)
030e9dd6 217 return ACK;
f6af19a9 218 if (strstr(p, "continue"))
745f7a8c 219 return ACK_continue;
f6af19a9 220 if (strstr(p, "common"))
745f7a8c 221 return ACK_common;
f6af19a9 222 if (strstr(p, "ready"))
745f7a8c
NTND
223 return ACK_ready;
224 return ACK;
225 }
226 }
01f9ec64 227 die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
745f7a8c
NTND
228}
229
230static void send_request(struct fetch_pack_args *args,
231 int fd, struct strbuf *buf)
232{
233 if (args->stateless_rpc) {
234 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
235 packet_flush(fd);
37c80012
JK
236 } else {
237 if (write_in_full(fd, buf->buf, buf->len) < 0)
238 die_errno(_("unable to write to remote"));
239 }
745f7a8c
NTND
240}
241
ec062838 242static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
d30fe89c 243 struct object *obj)
745f7a8c 244{
5c3b801d 245 rev_list_insert_ref(negotiator, &obj->oid);
745f7a8c
NTND
246}
247
248#define INITIAL_FLUSH 16
249#define PIPESAFE_FLUSH 32
da470981 250#define LARGE_FLUSH 16384
745f7a8c 251
685fbd32 252static int next_flush(int stateless_rpc, int count)
745f7a8c 253{
685fbd32 254 if (stateless_rpc) {
da470981
JT
255 if (count < LARGE_FLUSH)
256 count <<= 1;
257 else
258 count = count * 11 / 10;
259 } else {
260 if (count < PIPESAFE_FLUSH)
261 count <<= 1;
262 else
263 count += PIPESAFE_FLUSH;
264 }
745f7a8c
NTND
265 return count;
266}
267
3390e42a
JT
268static void mark_tips(struct fetch_negotiator *negotiator,
269 const struct oid_array *negotiation_tips)
270{
271 int i;
272
273 if (!negotiation_tips) {
5c3b801d 274 for_each_rawref(rev_list_insert_ref_oid, negotiator);
3390e42a
JT
275 return;
276 }
277
278 for (i = 0; i < negotiation_tips->nr; i++)
5c3b801d 279 rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
3390e42a
JT
280 return;
281}
282
ec062838 283static int find_common(struct fetch_negotiator *negotiator,
d30fe89c 284 struct fetch_pack_args *args,
1b283377 285 int fd[2], struct object_id *result_oid,
745f7a8c
NTND
286 struct ref *refs)
287{
288 int fetching;
289 int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
1b283377 290 const struct object_id *oid;
745f7a8c
NTND
291 unsigned in_vain = 0;
292 int got_continue = 0;
293 int got_ready = 0;
294 struct strbuf req_buf = STRBUF_INIT;
295 size_t state_len = 0;
01f9ec64 296 struct packet_reader reader;
745f7a8c
NTND
297
298 if (args->stateless_rpc && multi_ack == 1)
1dd73e20 299 die(_("--stateless-rpc requires multi_ack_detailed"));
745f7a8c 300
01f9ec64 301 packet_reader_init(&reader, fd[0], NULL, 0,
2d103c31
MS
302 PACKET_READ_CHOMP_NEWLINE |
303 PACKET_READ_DIE_ON_ERR_PACKET);
01f9ec64 304
9dfa8dbe
JT
305 mark_tips(negotiator, args->negotiation_tips);
306 for_each_cached_alternate(negotiator, insert_one_alternate_object);
745f7a8c
NTND
307
308 fetching = 0;
309 for ( ; refs ; refs = refs->next) {
1b283377 310 struct object_id *remote = &refs->old_oid;
745f7a8c
NTND
311 const char *remote_hex;
312 struct object *o;
313
314 /*
315 * If that object is complete (i.e. it is an ancestor of a
316 * local ref), we tell them we have it but do not have to
317 * tell them about its ancestors, which they already know
318 * about.
319 *
320 * We use lookup_object here because we are only
321 * interested in the case we *know* the object is
322 * reachable and we have already scanned it.
323 */
9dfa8dbe 324 if (((o = lookup_object(the_repository, remote)) != NULL) &&
745f7a8c
NTND
325 (o->flags & COMPLETE)) {
326 continue;
327 }
328
1b283377 329 remote_hex = oid_to_hex(remote);
745f7a8c
NTND
330 if (!fetching) {
331 struct strbuf c = STRBUF_INIT;
332 if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed");
333 if (multi_ack == 1) strbuf_addstr(&c, " multi_ack");
334 if (no_done) strbuf_addstr(&c, " no-done");
335 if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k");
336 if (use_sideband == 1) strbuf_addstr(&c, " side-band");
cccf74e2 337 if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
745f7a8c
NTND
338 if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
339 if (args->no_progress) strbuf_addstr(&c, " no-progress");
340 if (args->include_tag) strbuf_addstr(&c, " include-tag");
341 if (prefer_ofs_delta) strbuf_addstr(&c, " ofs-delta");
508ea882 342 if (deepen_since_ok) strbuf_addstr(&c, " deepen-since");
a45a2600 343 if (deepen_not_ok) strbuf_addstr(&c, " deepen-not");
745f7a8c
NTND
344 if (agent_supported) strbuf_addf(&c, " agent=%s",
345 git_user_agent_sanitized());
1e905bbc
JS
346 if (advertise_sid)
347 strbuf_addf(&c, " session-id=%s", trace2_session_id());
640d8b72
JH
348 if (args->filter_options.choice)
349 strbuf_addstr(&c, " filter");
745f7a8c
NTND
350 packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
351 strbuf_release(&c);
352 } else
353 packet_buf_write(&req_buf, "want %s\n", remote_hex);
354 fetching++;
355 }
356
357 if (!fetching) {
358 strbuf_release(&req_buf);
359 packet_flush(fd[1]);
360 return 1;
361 }
362
c8813487 363 if (is_repository_shallow(the_repository))
1a30f5a2 364 write_shallow_commits(&req_buf, 1, NULL);
745f7a8c
NTND
365 if (args->depth > 0)
366 packet_buf_write(&req_buf, "deepen %d", args->depth);
508ea882 367 if (args->deepen_since) {
dddbad72 368 timestamp_t max_age = approxidate(args->deepen_since);
cb71f8bd 369 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
508ea882 370 }
a45a2600
NTND
371 if (args->deepen_not) {
372 int i;
373 for (i = 0; i < args->deepen_not->nr; i++) {
374 struct string_list_item *s = args->deepen_not->items + i;
375 packet_buf_write(&req_buf, "deepen-not %s", s->string);
376 }
377 }
87c2d9d3 378 if (server_supports_filtering && args->filter_options.choice) {
cf9ceb5a
MD
379 const char *spec =
380 expand_list_objects_filter_spec(&args->filter_options);
381 packet_buf_write(&req_buf, "filter %s", spec);
87c2d9d3 382 }
745f7a8c
NTND
383 packet_buf_flush(&req_buf);
384 state_len = req_buf.len;
385
79891cb9 386 if (args->deepen) {
ae021d87 387 const char *arg;
1b283377 388 struct object_id oid;
745f7a8c
NTND
389
390 send_request(args, fd[1], &req_buf);
01f9ec64
MS
391 while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
392 if (skip_prefix(reader.line, "shallow ", &arg)) {
1b283377 393 if (get_oid_hex(arg, &oid))
01f9ec64 394 die(_("invalid shallow line: %s"), reader.line);
19143f13 395 register_shallow(the_repository, &oid);
745f7a8c
NTND
396 continue;
397 }
01f9ec64 398 if (skip_prefix(reader.line, "unshallow ", &arg)) {
1b283377 399 if (get_oid_hex(arg, &oid))
01f9ec64 400 die(_("invalid unshallow line: %s"), reader.line);
d0229abd 401 if (!lookup_object(the_repository, &oid))
01f9ec64 402 die(_("object not found: %s"), reader.line);
745f7a8c 403 /* make sure that it is parsed as shallow */
109cd76d 404 if (!parse_object(the_repository, &oid))
01f9ec64 405 die(_("error in object: %s"), reader.line);
e92b848c 406 if (unregister_shallow(&oid))
01f9ec64 407 die(_("no shallow found: %s"), reader.line);
745f7a8c
NTND
408 continue;
409 }
01f9ec64 410 die(_("expected shallow/unshallow, got %s"), reader.line);
745f7a8c
NTND
411 }
412 } else if (!args->stateless_rpc)
413 send_request(args, fd[1], &req_buf);
414
415 if (!args->stateless_rpc) {
416 /* If we aren't using the stateless-rpc interface
417 * we don't need to retain the headers.
418 */
419 strbuf_setlen(&req_buf, 0);
420 state_len = 0;
421 }
422
5fc31180 423 trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
745f7a8c
NTND
424 flushes = 0;
425 retval = -1;
ec062838 426 while ((oid = negotiator->next(negotiator))) {
1b283377 427 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
428 print_verbose(args, "have %s", oid_to_hex(oid));
745f7a8c
NTND
429 in_vain++;
430 if (flush_at <= ++count) {
431 int ack;
432
433 packet_buf_flush(&req_buf);
434 send_request(args, fd[1], &req_buf);
435 strbuf_setlen(&req_buf, state_len);
436 flushes++;
685fbd32 437 flush_at = next_flush(args->stateless_rpc, count);
745f7a8c
NTND
438
439 /*
440 * We keep one window "ahead" of the other side, and
441 * will wait for an ACK only on the next one
442 */
443 if (!args->stateless_rpc && count == INITIAL_FLUSH)
444 continue;
445
01f9ec64 446 consume_shallow_list(args, &reader);
745f7a8c 447 do {
01f9ec64 448 ack = get_ack(&reader, result_oid);
0d789a5b 449 if (ack)
1dd73e20 450 print_verbose(args, _("got %s %d %s"), "ack",
1b283377 451 ack, oid_to_hex(result_oid));
745f7a8c
NTND
452 switch (ack) {
453 case ACK:
454 flushes = 0;
455 multi_ack = 0;
456 retval = 0;
457 goto done;
458 case ACK_common:
459 case ACK_ready:
460 case ACK_continue: {
461 struct commit *commit =
c1f5eb49
SB
462 lookup_commit(the_repository,
463 result_oid);
d093bc75 464 int was_common;
3a2a1dc1 465
745f7a8c 466 if (!commit)
1b283377 467 die(_("invalid commit %s"), oid_to_hex(result_oid));
ec062838 468 was_common = negotiator->ack(negotiator, commit);
745f7a8c
NTND
469 if (args->stateless_rpc
470 && ack == ACK_common
d093bc75 471 && !was_common) {
745f7a8c
NTND
472 /* We need to replay the have for this object
473 * on the next RPC request so the peer knows
474 * it is in common with us.
475 */
1b283377 476 const char *hex = oid_to_hex(result_oid);
745f7a8c
NTND
477 packet_buf_write(&req_buf, "have %s\n", hex);
478 state_len = req_buf.len;
06b3d386
JT
479 /*
480 * Reset in_vain because an ack
481 * for this commit has not been
482 * seen.
483 */
484 in_vain = 0;
485 } else if (!args->stateless_rpc
486 || ack != ACK_common)
487 in_vain = 0;
745f7a8c 488 retval = 0;
745f7a8c 489 got_continue = 1;
21bcf6e4 490 if (ack == ACK_ready)
745f7a8c 491 got_ready = 1;
745f7a8c
NTND
492 break;
493 }
494 }
495 } while (ack);
496 flushes--;
497 if (got_continue && MAX_IN_VAIN < in_vain) {
1dd73e20 498 print_verbose(args, _("giving up"));
745f7a8c
NTND
499 break; /* give up */
500 }
21bcf6e4
JT
501 if (got_ready)
502 break;
745f7a8c
NTND
503 }
504 }
505done:
5fc31180 506 trace2_region_leave("fetch-pack", "negotiation_v0_v1", the_repository);
745f7a8c
NTND
507 if (!got_ready || !no_done) {
508 packet_buf_write(&req_buf, "done\n");
509 send_request(args, fd[1], &req_buf);
510 }
1dd73e20 511 print_verbose(args, _("done"));
745f7a8c
NTND
512 if (retval != 0) {
513 multi_ack = 0;
514 flushes++;
515 }
516 strbuf_release(&req_buf);
517
ff62eca7 518 if (!got_ready || !no_done)
01f9ec64 519 consume_shallow_list(args, &reader);
745f7a8c 520 while (flushes || multi_ack) {
01f9ec64 521 int ack = get_ack(&reader, result_oid);
745f7a8c 522 if (ack) {
1dd73e20 523 print_verbose(args, _("got %s (%d) %s"), "ack",
1b283377 524 ack, oid_to_hex(result_oid));
745f7a8c
NTND
525 if (ack == ACK)
526 return 0;
527 multi_ack = 1;
528 continue;
529 }
530 flushes--;
531 }
532 /* it is no error to fetch into a completely empty repo */
533 return count ? retval : 0;
534}
535
536static struct commit_list *complete;
537
1b283377 538static int mark_complete(const struct object_id *oid)
745f7a8c 539{
5c3b801d
JT
540 struct commit *commit = deref_without_lazy_fetch(oid, 1);
541
542 if (commit && !(commit->object.flags & COMPLETE)) {
543 commit->object.flags |= COMPLETE;
544 commit_list_insert(commit, &complete);
745f7a8c
NTND
545 }
546 return 0;
547}
548
f8ee4d85
MH
549static int mark_complete_oid(const char *refname, const struct object_id *oid,
550 int flag, void *cb_data)
551{
1b283377 552 return mark_complete(oid);
f8ee4d85
MH
553}
554
745f7a8c 555static void mark_recent_complete_commits(struct fetch_pack_args *args,
dddbad72 556 timestamp_t cutoff)
745f7a8c
NTND
557{
558 while (complete && cutoff <= complete->item->date) {
1dd73e20 559 print_verbose(args, _("Marking %s as complete"),
0d789a5b 560 oid_to_hex(&complete->item->object.oid));
745f7a8c
NTND
561 pop_most_recent_commit(&complete, COMPLETE);
562 }
563}
564
fdb69d33
JT
565static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
566{
567 for (; refs; refs = refs->next)
568 oidset_insert(oids, &refs->old_oid);
569}
570
bf73282c
RS
571static int is_unmatched_ref(const struct ref *ref)
572{
573 struct object_id oid;
574 const char *p;
575 return ref->match_status == REF_NOT_MATCHED &&
576 !parse_oid_hex(ref->name, &oid, &p) &&
577 *p == '\0' &&
578 oideq(&oid, &ref->old_oid);
579}
580
745f7a8c 581static void filter_refs(struct fetch_pack_args *args,
f2db854d
JH
582 struct ref **refs,
583 struct ref **sought, int nr_sought)
745f7a8c
NTND
584{
585 struct ref *newlist = NULL;
586 struct ref **newtail = &newlist;
fdb69d33 587 struct ref *unmatched = NULL;
745f7a8c 588 struct ref *ref, *next;
fdb69d33 589 struct oidset tip_oids = OIDSET_INIT;
f2db854d 590 int i;
22a16465
RS
591 int strict = !(allow_unadvertised_object_request &
592 (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
745f7a8c 593
f2db854d 594 i = 0;
745f7a8c
NTND
595 for (ref = *refs; ref; ref = next) {
596 int keep = 0;
597 next = ref->next;
f2db854d 598
50e19a83 599 if (starts_with(ref->name, "refs/") &&
34066f06
JK
600 check_refname_format(ref->name, 0)) {
601 /*
602 * trash or a peeled value; do not even add it to
603 * unmatched list
604 */
605 free_one_ref(ref);
606 continue;
607 } else {
f2db854d
JH
608 while (i < nr_sought) {
609 int cmp = strcmp(ref->name, sought[i]->name);
745f7a8c
NTND
610 if (cmp < 0)
611 break; /* definitely do not have it */
612 else if (cmp == 0) {
613 keep = 1; /* definitely have it */
d56583de 614 sought[i]->match_status = REF_MATCHED;
745f7a8c 615 }
f2db854d 616 i++;
745f7a8c 617 }
745f7a8c 618
e9502c0a
JK
619 if (!keep && args->fetch_all &&
620 (!args->deepen || !starts_with(ref->name, "refs/tags/")))
621 keep = 1;
622 }
745f7a8c
NTND
623
624 if (keep) {
625 *newtail = ref;
626 ref->next = NULL;
627 newtail = &ref->next;
628 } else {
fdb69d33
JT
629 ref->next = unmatched;
630 unmatched = ref;
745f7a8c
NTND
631 }
632 }
633
22a16465
RS
634 if (strict) {
635 for (i = 0; i < nr_sought; i++) {
636 ref = sought[i];
637 if (!is_unmatched_ref(ref))
638 continue;
639
640 add_refs_to_oidset(&tip_oids, unmatched);
641 add_refs_to_oidset(&tip_oids, newlist);
642 break;
643 }
644 }
645
6e7b66ee 646 /* Append unmatched requests to the list */
d56583de 647 for (i = 0; i < nr_sought; i++) {
d56583de 648 ref = sought[i];
bf73282c 649 if (!is_unmatched_ref(ref))
d56583de 650 continue;
6e7b66ee 651
22a16465 652 if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
d56583de 653 ref->match_status = REF_MATCHED;
c3c17bf1
JK
654 *newtail = copy_ref(ref);
655 newtail = &(*newtail)->next;
d56583de
MM
656 } else {
657 ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
6e7b66ee
JH
658 }
659 }
fdb69d33
JT
660
661 oidset_clear(&tip_oids);
259eddde 662 free_refs(unmatched);
fdb69d33 663
745f7a8c
NTND
664 *refs = newlist;
665}
666
ec062838 667static void mark_alternate_complete(struct fetch_negotiator *unused,
d30fe89c 668 struct object *obj)
745f7a8c 669{
1b283377 670 mark_complete(&obj->oid);
745f7a8c
NTND
671}
672
024aa469
TI
673struct loose_object_iter {
674 struct oidset *loose_object_set;
675 struct ref *refs;
676};
677
34c29034
JT
678/*
679 * Mark recent commits available locally and reachable from a local ref as
9dfa8dbe 680 * COMPLETE.
34c29034
JT
681 *
682 * The cutoff time for recency is determined by this heuristic: it is the
683 * earliest commit time of the objects in refs that are commits and that we know
684 * the commit time of.
685 */
ec062838 686static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
d30fe89c 687 struct fetch_pack_args *args,
34c29034 688 struct ref **refs)
745f7a8c
NTND
689{
690 struct ref *ref;
a1c6d7c1 691 int old_save_commit_buffer = save_commit_buffer;
dddbad72 692 timestamp_t cutoff = 0;
745f7a8c
NTND
693
694 save_commit_buffer = 0;
695
9e5afdf9 696 trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
745f7a8c
NTND
697 for (ref = *refs; ref; ref = ref->next) {
698 struct object *o;
699
97b2fa08 700 if (!has_object_file_with_flags(&ref->old_oid,
6462d5eb
JT
701 OBJECT_INFO_QUICK |
702 OBJECT_INFO_SKIP_FETCH_OBJECT))
024aa469 703 continue;
109cd76d 704 o = parse_object(the_repository, &ref->old_oid);
745f7a8c
NTND
705 if (!o)
706 continue;
707
9e5afdf9
EC
708 /*
709 * We already have it -- which may mean that we were
745f7a8c
NTND
710 * in sync with the other side at some time after
711 * that (it is OK if we guess wrong here).
712 */
713 if (o->type == OBJ_COMMIT) {
714 struct commit *commit = (struct commit *)o;
715 if (!cutoff || cutoff < commit->date)
716 cutoff = commit->date;
717 }
718 }
9e5afdf9 719 trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
745f7a8c 720
9e5afdf9
EC
721 /*
722 * This block marks all local refs as COMPLETE, and then recursively marks all
723 * parents of those refs as COMPLETE.
724 */
725 trace2_region_enter("fetch-pack", "mark_complete_local_refs", NULL);
12f19a98 726 if (!args->deepen) {
5c3b801d 727 for_each_rawref(mark_complete_oid, NULL);
12f19a98
JT
728 for_each_cached_alternate(NULL, mark_alternate_complete);
729 commit_list_sort_by_date(&complete);
730 if (cutoff)
731 mark_recent_complete_commits(args, cutoff);
732 }
9e5afdf9 733 trace2_region_leave("fetch-pack", "mark_complete_local_refs", NULL);
745f7a8c 734
12f19a98
JT
735 /*
736 * Mark all complete remote refs as common refs.
737 * Don't mark them common yet; the server has to be told so first.
738 */
9e5afdf9 739 trace2_region_enter("fetch-pack", "mark_common_remote_refs", NULL);
12f19a98 740 for (ref = *refs; ref; ref = ref->next) {
5c3b801d 741 struct commit *c = deref_without_lazy_fetch(&ref->old_oid, 0);
745f7a8c 742
5c3b801d 743 if (!c || !(c->object.flags & COMPLETE))
12f19a98 744 continue;
745f7a8c 745
5c3b801d 746 negotiator->known_common(negotiator, c);
745f7a8c 747 }
9e5afdf9 748 trace2_region_leave("fetch-pack", "mark_common_remote_refs", NULL);
745f7a8c 749
34c29034
JT
750 save_commit_buffer = old_save_commit_buffer;
751}
752
753/*
754 * Returns 1 if every object pointed to by the given remote refs is available
755 * locally and reachable from a local ref, and 0 otherwise.
756 */
757static int everything_local(struct fetch_pack_args *args,
758 struct ref **refs)
759{
760 struct ref *ref;
761 int retval;
745f7a8c
NTND
762
763 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
1b283377 764 const struct object_id *remote = &ref->old_oid;
745f7a8c
NTND
765 struct object *o;
766
d0229abd 767 o = lookup_object(the_repository, remote);
745f7a8c
NTND
768 if (!o || !(o->flags & COMPLETE)) {
769 retval = 0;
1b283377 770 print_verbose(args, "want %s (%s)", oid_to_hex(remote),
0d789a5b 771 ref->name);
745f7a8c
NTND
772 continue;
773 }
1b283377 774 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
0d789a5b 775 ref->name);
745f7a8c 776 }
a1c6d7c1 777
745f7a8c
NTND
778 return retval;
779}
780
781static int sideband_demux(int in, int out, void *data)
782{
783 int *xd = data;
9ff18faf 784 int ret;
745f7a8c 785
9ff18faf 786 ret = recv_sideband("fetch-pack", xd[0], out);
745f7a8c
NTND
787 close(out);
788 return ret;
789}
790
9d7fa3be
CC
791static void create_promisor_file(const char *keep_name,
792 struct ref **sought, int nr_sought)
5374a290
JT
793{
794 struct strbuf promisor_name = STRBUF_INIT;
795 int suffix_stripped;
5374a290
JT
796
797 strbuf_addstr(&promisor_name, keep_name);
798 suffix_stripped = strbuf_strip_suffix(&promisor_name, ".keep");
799 if (!suffix_stripped)
800 BUG("name of pack lockfile should end with .keep (was '%s')",
801 keep_name);
802 strbuf_addstr(&promisor_name, ".promisor");
803
33add2ad 804 write_promisor_file(promisor_name.buf, sought, nr_sought);
5374a290
JT
805
806 strbuf_release(&promisor_name);
807}
808
5476e1ef
JT
809static void parse_gitmodules_oids(int fd, struct oidset *gitmodules_oids)
810{
811 int len = the_hash_algo->hexsz + 1; /* hash + NL */
812
813 do {
814 char hex_hash[GIT_MAX_HEXSZ + 1];
815 int read_len = read_in_full(fd, hex_hash, len);
816 struct object_id oid;
817 const char *end;
818
819 if (!read_len)
820 return;
821 if (read_len != len)
822 die("invalid length read %d", read_len);
823 if (parse_oid_hex(hex_hash, &oid, &end) || *end != '\n')
824 die("invalid hash");
825 oidset_insert(gitmodules_oids, &oid);
826 } while (1);
827}
828
ece9aea2 829/*
b664e9ff
JT
830 * If packfile URIs were provided, pass a non-NULL pointer to index_pack_args.
831 * The strings to pass as the --index-pack-arg arguments to http-fetch will be
832 * stored there. (It must be freed by the caller.)
ece9aea2 833 */
745f7a8c 834static int get_pack(struct fetch_pack_args *args,
9da69a65 835 int xd[2], struct string_list *pack_lockfiles,
b664e9ff 836 struct strvec *index_pack_args,
5476e1ef
JT
837 struct ref **sought, int nr_sought,
838 struct oidset *gitmodules_oids)
745f7a8c
NTND
839{
840 struct async demux;
745f7a8c 841 int do_keep = args->keep_pack;
984a43b9
JK
842 const char *cmd_name;
843 struct pack_header header;
844 int pass_header = 0;
d3180279 845 struct child_process cmd = CHILD_PROCESS_INIT;
5476e1ef 846 int fsck_objects = 0;
c6807a40 847 int ret;
745f7a8c
NTND
848
849 memset(&demux, 0, sizeof(demux));
850 if (use_sideband) {
851 /* xd[] is talking with upload-pack; subprocess reads from
852 * xd[0], spits out band#2 to stderr, and feeds us band#1
853 * through demux->out.
854 */
855 demux.proc = sideband_demux;
856 demux.data = xd;
857 demux.out = -1;
df857572 858 demux.isolate_sigpipe = 1;
745f7a8c 859 if (start_async(&demux))
1dd73e20 860 die(_("fetch-pack: unable to fork off sideband demultiplexer"));
745f7a8c
NTND
861 }
862 else
863 demux.out = xd[0];
864
2aec3bc4 865 if (!args->keep_pack && unpack_limit && !index_pack_args) {
745f7a8c
NTND
866
867 if (read_pack_header(demux.out, &header))
1dd73e20 868 die(_("protocol error: bad pack header"));
984a43b9 869 pass_header = 1;
745f7a8c
NTND
870 if (ntohl(header.hdr_entries) < unpack_limit)
871 do_keep = 0;
872 else
873 do_keep = 1;
874 }
875
6035d6aa 876 if (alternate_shallow_file) {
ef8d7ac4
JK
877 strvec_push(&cmd.args, "--shallow-file");
878 strvec_push(&cmd.args, alternate_shallow_file);
6035d6aa
NTND
879 }
880
5476e1ef
JT
881 if (fetch_fsck_objects >= 0
882 ? fetch_fsck_objects
883 : transfer_fsck_objects >= 0
884 ? transfer_fsck_objects
885 : 0)
886 fsck_objects = 1;
887
888 if (do_keep || args->from_promisor || index_pack_args || fsck_objects) {
889 if (pack_lockfiles || fsck_objects)
745f7a8c 890 cmd.out = -1;
984a43b9 891 cmd_name = "index-pack";
ef8d7ac4
JK
892 strvec_push(&cmd.args, cmd_name);
893 strvec_push(&cmd.args, "--stdin");
745f7a8c 894 if (!args->quiet && !args->no_progress)
ef8d7ac4 895 strvec_push(&cmd.args, "-v");
745f7a8c 896 if (args->use_thin_pack)
ef8d7ac4 897 strvec_push(&cmd.args, "--fix-thin");
2aec3bc4 898 if ((do_keep || index_pack_args) && (args->lock_pack || unpack_limit)) {
da25bdb7 899 char hostname[HOST_NAME_MAX + 1];
5781a9a2 900 if (xgethostname(hostname, sizeof(hostname)))
984a43b9 901 xsnprintf(hostname, sizeof(hostname), "localhost");
ef8d7ac4 902 strvec_pushf(&cmd.args,
f6d8942b
JK
903 "--keep=fetch-pack %"PRIuMAX " on %s",
904 (uintmax_t)getpid(), hostname);
745f7a8c 905 }
b664e9ff 906 if (!index_pack_args && args->check_self_contained_and_connected)
ef8d7ac4 907 strvec_push(&cmd.args, "--check-self-contained-and-connected");
dd4b732d
JT
908 else
909 /*
910 * We cannot perform any connectivity checks because
911 * not all packs have been downloaded; let the caller
912 * have this responsibility.
913 */
914 args->check_self_contained_and_connected = 0;
1b03df5f
JT
915
916 if (args->from_promisor)
917 /*
9d7fa3be 918 * create_promisor_file() may be called afterwards but
1b03df5f
JT
919 * we still need index-pack to know that this is a
920 * promisor pack. For example, if transfer.fsckobjects
921 * is true, index-pack needs to know that .gitmodules
922 * is a promisor object (so that it won't complain if
923 * it is missing).
924 */
ef8d7ac4 925 strvec_push(&cmd.args, "--promisor");
745f7a8c
NTND
926 }
927 else {
984a43b9 928 cmd_name = "unpack-objects";
ef8d7ac4 929 strvec_push(&cmd.args, cmd_name);
745f7a8c 930 if (args->quiet || args->no_progress)
ef8d7ac4 931 strvec_push(&cmd.args, "-q");
c6807a40 932 args->check_self_contained_and_connected = 0;
745f7a8c 933 }
984a43b9
JK
934
935 if (pass_header)
ef8d7ac4 936 strvec_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
f6d8942b 937 ntohl(header.hdr_version),
984a43b9 938 ntohl(header.hdr_entries));
5476e1ef 939 if (fsck_objects) {
b664e9ff 940 if (args->from_promisor || index_pack_args)
98a2ea46
JT
941 /*
942 * We cannot use --strict in index-pack because it
943 * checks both broken objects and links, but we only
944 * want to check for broken objects.
945 */
ef8d7ac4 946 strvec_push(&cmd.args, "--fsck-objects");
98a2ea46 947 else
ef8d7ac4 948 strvec_pushf(&cmd.args, "--strict%s",
f6d8942b 949 fsck_msg_types.buf);
98a2ea46 950 }
745f7a8c 951
b664e9ff
JT
952 if (index_pack_args) {
953 int i;
954
955 for (i = 0; i < cmd.args.nr; i++)
956 strvec_push(index_pack_args, cmd.args.v[i]);
957 }
958
745f7a8c
NTND
959 cmd.in = demux.out;
960 cmd.git_cmd = 1;
961 if (start_command(&cmd))
1dd73e20 962 die(_("fetch-pack: unable to fork off %s"), cmd_name);
5476e1ef
JT
963 if (do_keep && (pack_lockfiles || fsck_objects)) {
964 int is_well_formed;
965 char *pack_lockfile = index_pack_lockfile(cmd.out, &is_well_formed);
966
967 if (!is_well_formed)
968 die(_("fetch-pack: invalid index-pack output"));
6031af38
RS
969 if (pack_lockfile)
970 string_list_append_nodup(pack_lockfiles, pack_lockfile);
5476e1ef 971 parse_gitmodules_oids(cmd.out, gitmodules_oids);
745f7a8c
NTND
972 close(cmd.out);
973 }
974
37cb1dd6
JL
975 if (!use_sideband)
976 /* Closed by start_command() */
977 xd[0] = -1;
978
c6807a40
NTND
979 ret = finish_command(&cmd);
980 if (!ret || (args->check_self_contained_and_connected && ret == 1))
981 args->self_contained_and_connected =
982 args->check_self_contained_and_connected &&
983 ret == 0;
984 else
1dd73e20 985 die(_("%s failed"), cmd_name);
745f7a8c 986 if (use_sideband && finish_async(&demux))
1dd73e20 987 die(_("error in sideband demultiplexer"));
5374a290
JT
988
989 /*
990 * Now that index-pack has succeeded, write the promisor file using the
991 * obtained .keep filename if necessary
992 */
9da69a65 993 if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
9d7fa3be 994 create_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
5374a290 995
745f7a8c
NTND
996 return 0;
997}
998
f2db854d
JH
999static int cmp_ref_by_name(const void *a_, const void *b_)
1000{
1001 const struct ref *a = *((const struct ref **)a_);
1002 const struct ref *b = *((const struct ref **)b_);
1003 return strcmp(a->name, b->name);
1004}
1005
745f7a8c
NTND
1006static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1007 int fd[2],
1008 const struct ref *orig_ref,
f2db854d 1009 struct ref **sought, int nr_sought,
beea4152 1010 struct shallow_info *si,
9da69a65 1011 struct string_list *pack_lockfiles)
745f7a8c 1012{
aaf633c2 1013 struct repository *r = the_repository;
745f7a8c 1014 struct ref *ref = copy_ref_list(orig_ref);
1b283377 1015 struct object_id oid;
745f7a8c
NTND
1016 const char *agent_feature;
1017 int agent_len;
603960b5
JT
1018 struct fetch_negotiator negotiator_alloc;
1019 struct fetch_negotiator *negotiator;
1020
9dfa8dbe
JT
1021 negotiator = &negotiator_alloc;
1022 fetch_negotiator_init(r, negotiator);
745f7a8c
NTND
1023
1024 sort_ref_list(&ref, ref_compare_name);
9ed0d8d6 1025 QSORT(sought, nr_sought, cmp_ref_by_name);
745f7a8c 1026
0e042971
NTND
1027 if ((agent_feature = server_feature_value("agent", &agent_len))) {
1028 agent_supported = 1;
1029 if (agent_len)
1030 print_verbose(args, _("Server version is %.*s"),
1031 agent_len, agent_feature);
1032 }
1033
1e905bbc
JS
1034 if (!server_supports("session-id"))
1035 advertise_sid = 0;
1036
5a88583b
NTND
1037 if (server_supports("shallow"))
1038 print_verbose(args, _("Server supports %s"), "shallow");
aaf633c2 1039 else if (args->depth > 0 || is_repository_shallow(r))
1dd73e20 1040 die(_("Server does not support shallow clients"));
a45a2600 1041 if (args->depth > 0 || args->deepen_since || args->deepen_not)
79891cb9 1042 args->deepen = 1;
745f7a8c 1043 if (server_supports("multi_ack_detailed")) {
0778b293 1044 print_verbose(args, _("Server supports %s"), "multi_ack_detailed");
745f7a8c
NTND
1045 multi_ack = 2;
1046 if (server_supports("no-done")) {
0778b293 1047 print_verbose(args, _("Server supports %s"), "no-done");
745f7a8c
NTND
1048 if (args->stateless_rpc)
1049 no_done = 1;
1050 }
1051 }
1052 else if (server_supports("multi_ack")) {
0778b293 1053 print_verbose(args, _("Server supports %s"), "multi_ack");
745f7a8c
NTND
1054 multi_ack = 1;
1055 }
1056 if (server_supports("side-band-64k")) {
0778b293 1057 print_verbose(args, _("Server supports %s"), "side-band-64k");
745f7a8c
NTND
1058 use_sideband = 2;
1059 }
1060 else if (server_supports("side-band")) {
0778b293 1061 print_verbose(args, _("Server supports %s"), "side-band");
745f7a8c
NTND
1062 use_sideband = 1;
1063 }
6e7b66ee 1064 if (server_supports("allow-tip-sha1-in-want")) {
0778b293 1065 print_verbose(args, _("Server supports %s"), "allow-tip-sha1-in-want");
7199c093 1066 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
6e7b66ee 1067 }
68ee6289 1068 if (server_supports("allow-reachable-sha1-in-want")) {
0778b293 1069 print_verbose(args, _("Server supports %s"), "allow-reachable-sha1-in-want");
68ee6289
FM
1070 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1071 }
5a88583b
NTND
1072 if (server_supports("thin-pack"))
1073 print_verbose(args, _("Server supports %s"), "thin-pack");
1074 else
745f7a8c 1075 args->use_thin_pack = 0;
5a88583b
NTND
1076 if (server_supports("no-progress"))
1077 print_verbose(args, _("Server supports %s"), "no-progress");
1078 else
745f7a8c 1079 args->no_progress = 0;
5a88583b
NTND
1080 if (server_supports("include-tag"))
1081 print_verbose(args, _("Server supports %s"), "include-tag");
1082 else
745f7a8c 1083 args->include_tag = 0;
0d789a5b 1084 if (server_supports("ofs-delta"))
0778b293 1085 print_verbose(args, _("Server supports %s"), "ofs-delta");
0d789a5b 1086 else
745f7a8c
NTND
1087 prefer_ofs_delta = 0;
1088
640d8b72
JH
1089 if (server_supports("filter")) {
1090 server_supports_filtering = 1;
0778b293 1091 print_verbose(args, _("Server supports %s"), "filter");
640d8b72
JH
1092 } else if (args->filter_options.choice) {
1093 warning("filtering not recognized by server, ignoring");
1094 }
1095
5a88583b
NTND
1096 if (server_supports("deepen-since")) {
1097 print_verbose(args, _("Server supports %s"), "deepen-since");
508ea882 1098 deepen_since_ok = 1;
5a88583b 1099 } else if (args->deepen_since)
508ea882 1100 die(_("Server does not support --shallow-since"));
5a88583b
NTND
1101 if (server_supports("deepen-not")) {
1102 print_verbose(args, _("Server supports %s"), "deepen-not");
a45a2600 1103 deepen_not_ok = 1;
5a88583b 1104 } else if (args->deepen_not)
a45a2600 1105 die(_("Server does not support --shallow-exclude"));
5a88583b
NTND
1106 if (server_supports("deepen-relative"))
1107 print_verbose(args, _("Server supports %s"), "deepen-relative");
1108 else if (args->deepen_relative)
cccf74e2 1109 die(_("Server does not support --deepen"));
48bf1415 1110 if (!server_supports_hash(the_hash_algo->name, NULL))
1111 die(_("Server does not support this repository's object format"));
745f7a8c 1112
9dfa8dbe
JT
1113 mark_complete_and_common_ref(negotiator, args, &ref);
1114 filter_refs(args, &ref, sought, nr_sought);
1115 if (everything_local(args, &ref)) {
1116 packet_flush(fd[1]);
1117 goto all_done;
745f7a8c 1118 }
603960b5 1119 if (find_common(negotiator, args, fd, &oid, ref) < 0)
745f7a8c
NTND
1120 if (!args->keep_pack)
1121 /* When cloning, it is not unusual to have
1122 * no common commit.
1123 */
1dd73e20 1124 warning(_("no common commits"));
745f7a8c
NTND
1125
1126 if (args->stateless_rpc)
1127 packet_flush(fd[1]);
79891cb9 1128 if (args->deepen)
1a30f5a2
NTND
1129 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1130 NULL);
4fe788b1
LL
1131 else if (si->nr_ours || si->nr_theirs) {
1132 if (args->reject_shallow_remote)
1133 die(_("source repository is shallow, reject to clone."));
beea4152 1134 alternate_shallow_file = setup_temporary_shallow(si->shallow);
4fe788b1 1135 } else
6da8bdcb 1136 alternate_shallow_file = NULL;
5476e1ef 1137 if (get_pack(args, fd, pack_lockfiles, NULL, sought, nr_sought,
3745e269 1138 &fsck_options.gitmodules_found))
1dd73e20 1139 die(_("git fetch-pack: fetch failed."));
3745e269
ÆAB
1140 if (fsck_finish(&fsck_options))
1141 die("fsck failed");
745f7a8c
NTND
1142
1143 all_done:
603960b5
JT
1144 if (negotiator)
1145 negotiator->release(negotiator);
745f7a8c
NTND
1146 return ref;
1147}
1148
f7e20501
BW
1149static void add_shallow_requests(struct strbuf *req_buf,
1150 const struct fetch_pack_args *args)
1151{
00624d60 1152 if (is_repository_shallow(the_repository))
f7e20501
BW
1153 write_shallow_commits(req_buf, 1, NULL);
1154 if (args->depth > 0)
1155 packet_buf_write(req_buf, "deepen %d", args->depth);
1156 if (args->deepen_since) {
1157 timestamp_t max_age = approxidate(args->deepen_since);
1158 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1159 }
1160 if (args->deepen_not) {
1161 int i;
1162 for (i = 0; i < args->deepen_not->nr; i++) {
1163 struct string_list_item *s = args->deepen_not->items + i;
1164 packet_buf_write(req_buf, "deepen-not %s", s->string);
1165 }
1166 }
5056cf4a
JT
1167 if (args->deepen_relative)
1168 packet_buf_write(req_buf, "deepen-relative\n");
f7e20501
BW
1169}
1170
9dfa8dbe 1171static void add_wants(const struct ref *wants, struct strbuf *req_buf)
685fbd32 1172{
73302051
BW
1173 int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1174
685fbd32
BW
1175 for ( ; wants ; wants = wants->next) {
1176 const struct object_id *remote = &wants->old_oid;
685fbd32
BW
1177 struct object *o;
1178
1179 /*
1180 * If that object is complete (i.e. it is an ancestor of a
1181 * local ref), we tell them we have it but do not have to
1182 * tell them about its ancestors, which they already know
1183 * about.
1184 *
1185 * We use lookup_object here because we are only
1186 * interested in the case we *know* the object is
1187 * reachable and we have already scanned it.
1188 */
9dfa8dbe 1189 if (((o = lookup_object(the_repository, remote)) != NULL) &&
685fbd32
BW
1190 (o->flags & COMPLETE)) {
1191 continue;
1192 }
1193
73302051
BW
1194 if (!use_ref_in_want || wants->exact_oid)
1195 packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1196 else
1197 packet_buf_write(req_buf, "want-ref %s\n", wants->name);
685fbd32
BW
1198 }
1199}
1200
1201static void add_common(struct strbuf *req_buf, struct oidset *common)
1202{
1203 struct oidset_iter iter;
1204 const struct object_id *oid;
1205 oidset_iter_init(common, &iter);
1206
1207 while ((oid = oidset_iter_next(&iter))) {
1208 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1209 }
1210}
1211
ec062838
JT
1212static int add_haves(struct fetch_negotiator *negotiator,
1213 struct strbuf *req_buf,
57c3451b 1214 int *haves_to_send)
685fbd32 1215{
685fbd32
BW
1216 int haves_added = 0;
1217 const struct object_id *oid;
1218
ec062838 1219 while ((oid = negotiator->next(negotiator))) {
685fbd32
BW
1220 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1221 if (++haves_added >= *haves_to_send)
1222 break;
1223 }
1224
685fbd32
BW
1225 /* Increase haves to send on next round */
1226 *haves_to_send = next_flush(1, *haves_to_send);
1227
57c3451b 1228 return haves_added;
685fbd32
BW
1229}
1230
6871d0ce
JT
1231static void write_fetch_command_and_capabilities(struct strbuf *req_buf,
1232 const struct string_list *server_options)
685fbd32 1233{
4b831208 1234 const char *hash_name;
685fbd32
BW
1235
1236 if (server_supports_v2("fetch", 1))
6871d0ce 1237 packet_buf_write(req_buf, "command=fetch");
685fbd32 1238 if (server_supports_v2("agent", 0))
6871d0ce 1239 packet_buf_write(req_buf, "agent=%s", git_user_agent_sanitized());
1e905bbc 1240 if (advertise_sid && server_supports_v2("session-id", 0))
6871d0ce
JT
1241 packet_buf_write(req_buf, "session-id=%s", trace2_session_id());
1242 if (server_options && server_options->nr &&
5e3548ef
BW
1243 server_supports_v2("server-option", 1)) {
1244 int i;
6871d0ce
JT
1245 for (i = 0; i < server_options->nr; i++)
1246 packet_buf_write(req_buf, "server-option=%s",
1247 server_options->items[i].string);
5e3548ef 1248 }
685fbd32 1249
4b831208 1250 if (server_feature_v2("object-format", &hash_name)) {
1251 int hash_algo = hash_algo_by_name(hash_name);
1252 if (hash_algo_by_ptr(the_hash_algo) != hash_algo)
1253 die(_("mismatched algorithms: client %s; server %s"),
1254 the_hash_algo->name, hash_name);
6871d0ce 1255 packet_buf_write(req_buf, "object-format=%s", the_hash_algo->name);
4b831208 1256 } else if (hash_algo_by_ptr(the_hash_algo) != GIT_HASH_SHA1) {
1257 die(_("the server does not support algorithm '%s'"),
1258 the_hash_algo->name);
1259 }
6871d0ce
JT
1260 packet_buf_delim(req_buf);
1261}
1262
1263static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1264 struct fetch_pack_args *args,
1265 const struct ref *wants, struct oidset *common,
1266 int *haves_to_send, int *in_vain,
1267 int sideband_all, int seen_ack)
1268{
1269 int haves_added;
1270 int done_sent = 0;
1271 struct strbuf req_buf = STRBUF_INIT;
1272
1273 write_fetch_command_and_capabilities(&req_buf, args->server_options);
4b831208 1274
685fbd32
BW
1275 if (args->use_thin_pack)
1276 packet_buf_write(&req_buf, "thin-pack");
1277 if (args->no_progress)
1278 packet_buf_write(&req_buf, "no-progress");
1279 if (args->include_tag)
1280 packet_buf_write(&req_buf, "include-tag");
1281 if (prefer_ofs_delta)
1282 packet_buf_write(&req_buf, "ofs-delta");
0bbc0bc5
JT
1283 if (sideband_all)
1284 packet_buf_write(&req_buf, "sideband-all");
685fbd32 1285
f7e20501
BW
1286 /* Add shallow-info and deepen request */
1287 if (server_supports_feature("fetch", "shallow", 0))
1288 add_shallow_requests(&req_buf, args);
00624d60 1289 else if (is_repository_shallow(the_repository) || args->deepen)
f7e20501
BW
1290 die(_("Server does not support shallow requests"));
1291
ba95710a
JT
1292 /* Add filter */
1293 if (server_supports_feature("fetch", "filter", 0) &&
1294 args->filter_options.choice) {
cf9ceb5a
MD
1295 const char *spec =
1296 expand_list_objects_filter_spec(&args->filter_options);
ba95710a 1297 print_verbose(args, _("Server supports filter"));
cf9ceb5a 1298 packet_buf_write(&req_buf, "filter %s", spec);
ba95710a
JT
1299 } else if (args->filter_options.choice) {
1300 warning("filtering not recognized by server, ignoring");
1301 }
1302
dd4b732d
JT
1303 if (server_supports_feature("fetch", "packfile-uris", 0)) {
1304 int i;
1305 struct strbuf to_send = STRBUF_INIT;
1306
1307 for (i = 0; i < uri_protocols.nr; i++) {
1308 const char *s = uri_protocols.items[i].string;
1309
1310 if (!strcmp(s, "https") || !strcmp(s, "http")) {
1311 if (to_send.len)
1312 strbuf_addch(&to_send, ',');
1313 strbuf_addstr(&to_send, s);
1314 }
1315 }
1316 if (to_send.len) {
1317 packet_buf_write(&req_buf, "packfile-uris %s",
1318 to_send.buf);
1319 strbuf_release(&to_send);
1320 }
1321 }
1322
685fbd32 1323 /* add wants */
9dfa8dbe 1324 add_wants(wants, &req_buf);
685fbd32 1325
9dfa8dbe
JT
1326 /* Add all of the common commits we've found in previous rounds */
1327 add_common(&req_buf, common);
685fbd32 1328
57c3451b
JT
1329 haves_added = add_haves(negotiator, &req_buf, haves_to_send);
1330 *in_vain += haves_added;
1331 if (!haves_added || (seen_ack && *in_vain >= MAX_IN_VAIN)) {
1332 /* Send Done */
1333 packet_buf_write(&req_buf, "done\n");
1334 done_sent = 1;
1335 }
685fbd32
BW
1336
1337 /* Send request */
1338 packet_buf_flush(&req_buf);
37c80012
JK
1339 if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
1340 die_errno(_("unable to write request to remote"));
685fbd32
BW
1341
1342 strbuf_release(&req_buf);
57c3451b 1343 return done_sent;
685fbd32
BW
1344}
1345
1346/*
1347 * Processes a section header in a server's response and checks if it matches
1348 * `section`. If the value of `peek` is 1, the header line will be peeked (and
1349 * not consumed); if 0, the line will be consumed and the function will die if
1350 * the section header doesn't match what was expected.
1351 */
1352static int process_section_header(struct packet_reader *reader,
1353 const char *section, int peek)
1354{
1355 int ret;
1356
1357 if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
bbb19a8b 1358 die(_("error reading section header '%s'"), section);
685fbd32
BW
1359
1360 ret = !strcmp(reader->line, section);
1361
1362 if (!peek) {
1363 if (!ret)
bbb19a8b 1364 die(_("expected '%s', received '%s'"),
685fbd32
BW
1365 section, reader->line);
1366 packet_reader_read(reader);
1367 }
1368
1369 return ret;
1370}
1371
81025703
JT
1372static int process_ack(struct fetch_negotiator *negotiator,
1373 struct packet_reader *reader,
1374 struct object_id *common_oid,
1375 int *received_ready)
685fbd32 1376{
685fbd32
BW
1377 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1378 const char *arg;
1379
1380 if (!strcmp(reader->line, "NAK"))
1381 continue;
1382
1383 if (skip_prefix(reader->line, "ACK ", &arg)) {
81025703 1384 if (!get_oid_hex(arg, common_oid)) {
685fbd32 1385 struct commit *commit;
81025703 1386 commit = lookup_commit(the_repository, common_oid);
603960b5
JT
1387 if (negotiator)
1388 negotiator->ack(negotiator, commit);
685fbd32 1389 }
81025703 1390 return 1;
685fbd32
BW
1391 }
1392
1393 if (!strcmp(reader->line, "ready")) {
81025703 1394 *received_ready = 1;
685fbd32
BW
1395 continue;
1396 }
1397
bbb19a8b 1398 die(_("unexpected acknowledgment line: '%s'"), reader->line);
685fbd32
BW
1399 }
1400
1401 if (reader->status != PACKET_READ_FLUSH &&
1402 reader->status != PACKET_READ_DELIM)
bbb19a8b 1403 die(_("error processing acks: %d"), reader->status);
685fbd32 1404
5400b2a2
JT
1405 /*
1406 * If an "acknowledgments" section is sent, a packfile is sent if and
1407 * only if "ready" was sent in this section. The other sections
1408 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1409 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1410 * otherwise.
1411 */
81025703 1412 if (*received_ready && reader->status != PACKET_READ_DELIM)
5400b2a2 1413 die(_("expected packfile to be sent after 'ready'"));
81025703 1414 if (!*received_ready && reader->status != PACKET_READ_FLUSH)
5400b2a2
JT
1415 die(_("expected no other sections to be sent after no 'ready'"));
1416
81025703 1417 return 0;
685fbd32
BW
1418}
1419
f7e20501 1420static void receive_shallow_info(struct fetch_pack_args *args,
1339078f
JT
1421 struct packet_reader *reader,
1422 struct oid_array *shallows,
1423 struct shallow_info *si)
f7e20501 1424{
1339078f 1425 int unshallow_received = 0;
bd0b42ae 1426
f7e20501
BW
1427 process_section_header(reader, "shallow-info", 0);
1428 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1429 const char *arg;
1430 struct object_id oid;
1431
1432 if (skip_prefix(reader->line, "shallow ", &arg)) {
1433 if (get_oid_hex(arg, &oid))
1434 die(_("invalid shallow line: %s"), reader->line);
1339078f 1435 oid_array_append(shallows, &oid);
f7e20501
BW
1436 continue;
1437 }
1438 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1439 if (get_oid_hex(arg, &oid))
1440 die(_("invalid unshallow line: %s"), reader->line);
d0229abd 1441 if (!lookup_object(the_repository, &oid))
f7e20501
BW
1442 die(_("object not found: %s"), reader->line);
1443 /* make sure that it is parsed as shallow */
109cd76d 1444 if (!parse_object(the_repository, &oid))
f7e20501
BW
1445 die(_("error in object: %s"), reader->line);
1446 if (unregister_shallow(&oid))
1447 die(_("no shallow found: %s"), reader->line);
1339078f 1448 unshallow_received = 1;
f7e20501
BW
1449 continue;
1450 }
1451 die(_("expected shallow/unshallow, got %s"), reader->line);
1452 }
1453
1454 if (reader->status != PACKET_READ_FLUSH &&
1455 reader->status != PACKET_READ_DELIM)
bbb19a8b 1456 die(_("error processing shallow info: %d"), reader->status);
f7e20501 1457
1339078f
JT
1458 if (args->deepen || unshallow_received) {
1459 /*
1460 * Treat these as shallow lines caused by our depth settings.
1461 * In v0, these lines cannot cause refs to be rejected; do the
1462 * same.
1463 */
1464 int i;
1465
1466 for (i = 0; i < shallows->nr; i++)
1467 register_shallow(the_repository, &shallows->oid[i]);
bd0b42ae
JT
1468 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1469 NULL);
1470 args->deepen = 1;
1339078f
JT
1471 } else if (shallows->nr) {
1472 /*
1473 * Treat these as shallow lines caused by the remote being
1474 * shallow. In v0, remote refs that reach these objects are
1475 * rejected (unless --update-shallow is set); do the same.
1476 */
1477 prepare_shallow_info(si, shallows);
4fe788b1
LL
1478 if (si->nr_ours || si->nr_theirs) {
1479 if (args->reject_shallow_remote)
1480 die(_("source repository is shallow, reject to clone."));
1339078f
JT
1481 alternate_shallow_file =
1482 setup_temporary_shallow(si->shallow);
4fe788b1 1483 } else
1339078f 1484 alternate_shallow_file = NULL;
380ebab2 1485 } else {
1486 alternate_shallow_file = NULL;
bd0b42ae 1487 }
f7e20501
BW
1488}
1489
b7643009
JT
1490static int cmp_name_ref(const void *name, const void *ref)
1491{
1492 return strcmp(name, (*(struct ref **)ref)->name);
1493}
1494
e2842b39
JT
1495static void receive_wanted_refs(struct packet_reader *reader,
1496 struct ref **sought, int nr_sought)
73302051
BW
1497{
1498 process_section_header(reader, "wanted-refs", 0);
1499 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1500 struct object_id oid;
1501 const char *end;
b7643009 1502 struct ref **found;
73302051
BW
1503
1504 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
bbb19a8b 1505 die(_("expected wanted-ref, got '%s'"), reader->line);
73302051 1506
b7643009
JT
1507 found = bsearch(end, sought, nr_sought, sizeof(*sought),
1508 cmp_name_ref);
1509 if (!found)
bbb19a8b 1510 die(_("unexpected wanted-ref: '%s'"), reader->line);
b7643009 1511 oidcpy(&(*found)->old_oid, &oid);
73302051
BW
1512 }
1513
1514 if (reader->status != PACKET_READ_DELIM)
bbb19a8b 1515 die(_("error processing wanted refs: %d"), reader->status);
73302051
BW
1516}
1517
dd4b732d
JT
1518static void receive_packfile_uris(struct packet_reader *reader,
1519 struct string_list *uris)
1520{
1521 process_section_header(reader, "packfile-uris", 0);
1522 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1523 if (reader->pktlen < the_hash_algo->hexsz ||
1524 reader->line[the_hash_algo->hexsz] != ' ')
1525 die("expected '<hash> <uri>', got: %s\n", reader->line);
1526
1527 string_list_append(uris, reader->line);
1528 }
1529 if (reader->status != PACKET_READ_DELIM)
1530 die("expected DELIM");
1531}
1532
685fbd32
BW
1533enum fetch_state {
1534 FETCH_CHECK_LOCAL = 0,
1535 FETCH_SEND_REQUEST,
1536 FETCH_PROCESS_ACKS,
1537 FETCH_GET_PACK,
1538 FETCH_DONE,
1539};
1540
9c1e657a 1541static void do_check_stateless_delimiter(int stateless_rpc,
b0df0c16
DL
1542 struct packet_reader *reader)
1543{
9c1e657a 1544 check_stateless_delimiter(stateless_rpc, reader,
b0df0c16
DL
1545 _("git fetch-pack: expected response end packet"));
1546}
1547
685fbd32
BW
1548static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1549 int fd[2],
1550 const struct ref *orig_ref,
1551 struct ref **sought, int nr_sought,
1339078f
JT
1552 struct oid_array *shallows,
1553 struct shallow_info *si,
9da69a65 1554 struct string_list *pack_lockfiles)
685fbd32 1555{
aaf633c2 1556 struct repository *r = the_repository;
685fbd32
BW
1557 struct ref *ref = copy_ref_list(orig_ref);
1558 enum fetch_state state = FETCH_CHECK_LOCAL;
1559 struct oidset common = OIDSET_INIT;
1560 struct packet_reader reader;
5fc31180 1561 int in_vain = 0, negotiation_started = 0;
685fbd32 1562 int haves_to_send = INITIAL_FLUSH;
603960b5
JT
1563 struct fetch_negotiator negotiator_alloc;
1564 struct fetch_negotiator *negotiator;
4fa3f00a 1565 int seen_ack = 0;
81025703
JT
1566 struct object_id common_oid;
1567 int received_ready = 0;
dd4b732d
JT
1568 struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1569 int i;
b664e9ff 1570 struct strvec index_pack_args = STRVEC_INIT;
603960b5 1571
9dfa8dbe
JT
1572 negotiator = &negotiator_alloc;
1573 fetch_negotiator_init(r, negotiator);
603960b5 1574
685fbd32 1575 packet_reader_init(&reader, fd[0], NULL, 0,
2d103c31
MS
1576 PACKET_READ_CHOMP_NEWLINE |
1577 PACKET_READ_DIE_ON_ERR_PACKET);
07c3c2aa
JT
1578 if (git_env_bool("GIT_TEST_SIDEBAND_ALL", 1) &&
1579 server_supports_feature("fetch", "sideband-all", 0)) {
0bbc0bc5
JT
1580 reader.use_sideband = 1;
1581 reader.me = "fetch-pack";
1582 }
685fbd32
BW
1583
1584 while (state != FETCH_DONE) {
1585 switch (state) {
1586 case FETCH_CHECK_LOCAL:
1587 sort_ref_list(&ref, ref_compare_name);
1588 QSORT(sought, nr_sought, cmp_ref_by_name);
1589
1590 /* v2 supports these by default */
1591 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1592 use_sideband = 2;
f7e20501
BW
1593 if (args->depth > 0 || args->deepen_since || args->deepen_not)
1594 args->deepen = 1;
685fbd32 1595
685fbd32 1596 /* Filter 'ref' by 'sought' and those that aren't local */
9dfa8dbe
JT
1597 mark_complete_and_common_ref(negotiator, args, &ref);
1598 filter_refs(args, &ref, sought, nr_sought);
1599 if (everything_local(args, &ref))
1600 state = FETCH_DONE;
1601 else
685fbd32 1602 state = FETCH_SEND_REQUEST;
9dfa8dbe
JT
1603
1604 mark_tips(negotiator, args->negotiation_tips);
1605 for_each_cached_alternate(negotiator,
1606 insert_one_alternate_object);
685fbd32
BW
1607 break;
1608 case FETCH_SEND_REQUEST:
5fc31180
JS
1609 if (!negotiation_started) {
1610 negotiation_started = 1;
1611 trace2_region_enter("fetch-pack",
1612 "negotiation_v2",
1613 the_repository);
1614 }
603960b5 1615 if (send_fetch_request(negotiator, fd[1], args, ref,
ec062838 1616 &common,
0bbc0bc5 1617 &haves_to_send, &in_vain,
4fa3f00a
JT
1618 reader.use_sideband,
1619 seen_ack))
685fbd32
BW
1620 state = FETCH_GET_PACK;
1621 else
1622 state = FETCH_PROCESS_ACKS;
1623 break;
1624 case FETCH_PROCESS_ACKS:
1625 /* Process ACKs/NAKs */
81025703
JT
1626 process_section_header(&reader, "acknowledgments", 0);
1627 while (process_ack(negotiator, &reader, &common_oid,
1628 &received_ready)) {
1629 in_vain = 0;
1630 seen_ack = 1;
1631 oidset_insert(&common, &common_oid);
1632 }
1633 if (received_ready) {
b0df0c16
DL
1634 /*
1635 * Don't check for response delimiter; get_pack() will
1636 * read the rest of this response.
1637 */
685fbd32 1638 state = FETCH_GET_PACK;
81025703 1639 } else {
9c1e657a 1640 do_check_stateless_delimiter(args->stateless_rpc, &reader);
685fbd32 1641 state = FETCH_SEND_REQUEST;
685fbd32
BW
1642 }
1643 break;
1644 case FETCH_GET_PACK:
5fc31180
JS
1645 trace2_region_leave("fetch-pack",
1646 "negotiation_v2",
1647 the_repository);
f7e20501
BW
1648 /* Check for shallow-info section */
1649 if (process_section_header(&reader, "shallow-info", 1))
1339078f 1650 receive_shallow_info(args, &reader, shallows, si);
f7e20501 1651
73302051 1652 if (process_section_header(&reader, "wanted-refs", 1))
e2842b39 1653 receive_wanted_refs(&reader, sought, nr_sought);
73302051 1654
dd4b732d 1655 /* get the pack(s) */
88e9b1e3
IF
1656 if (git_env_bool("GIT_TRACE_REDACT", 1))
1657 reader.options |= PACKET_READ_REDACT_URI_PATH;
dd4b732d
JT
1658 if (process_section_header(&reader, "packfile-uris", 1))
1659 receive_packfile_uris(&reader, &packfile_uris);
88e9b1e3
IF
1660 /* We don't expect more URIs. Reset to avoid expensive URI check. */
1661 reader.options &= ~PACKET_READ_REDACT_URI_PATH;
1662
685fbd32 1663 process_section_header(&reader, "packfile", 0);
ae1a7eef
JK
1664
1665 /*
1666 * this is the final request we'll make of the server;
1667 * do a half-duplex shutdown to indicate that they can
1668 * hang up as soon as the pack is sent.
1669 */
1670 close(fd[1]);
1671 fd[1] = -1;
1672
dd4b732d 1673 if (get_pack(args, fd, pack_lockfiles,
b664e9ff 1674 packfile_uris.nr ? &index_pack_args : NULL,
3745e269 1675 sought, nr_sought, &fsck_options.gitmodules_found))
685fbd32 1676 die(_("git fetch-pack: fetch failed."));
9c1e657a 1677 do_check_stateless_delimiter(args->stateless_rpc, &reader);
685fbd32
BW
1678
1679 state = FETCH_DONE;
1680 break;
1681 case FETCH_DONE:
1682 continue;
1683 }
1684 }
1685
dd4b732d 1686 for (i = 0; i < packfile_uris.nr; i++) {
b664e9ff 1687 int j;
dd4b732d
JT
1688 struct child_process cmd = CHILD_PROCESS_INIT;
1689 char packname[GIT_MAX_HEXSZ + 1];
1690 const char *uri = packfile_uris.items[i].string +
1691 the_hash_algo->hexsz + 1;
1692
ef8d7ac4
JK
1693 strvec_push(&cmd.args, "http-fetch");
1694 strvec_pushf(&cmd.args, "--packfile=%.*s",
f6d8942b
JK
1695 (int) the_hash_algo->hexsz,
1696 packfile_uris.items[i].string);
b664e9ff
JT
1697 for (j = 0; j < index_pack_args.nr; j++)
1698 strvec_pushf(&cmd.args, "--index-pack-arg=%s",
1699 index_pack_args.v[j]);
ef8d7ac4 1700 strvec_push(&cmd.args, uri);
dd4b732d
JT
1701 cmd.git_cmd = 1;
1702 cmd.no_stdin = 1;
1703 cmd.out = -1;
1704 if (start_command(&cmd))
1705 die("fetch-pack: unable to spawn http-fetch");
1706
1707 if (read_in_full(cmd.out, packname, 5) < 0 ||
1708 memcmp(packname, "keep\t", 5))
1709 die("fetch-pack: expected keep then TAB at start of http-fetch output");
1710
1711 if (read_in_full(cmd.out, packname,
1712 the_hash_algo->hexsz + 1) < 0 ||
1713 packname[the_hash_algo->hexsz] != '\n')
1714 die("fetch-pack: expected hash then LF at end of http-fetch output");
1715
1716 packname[the_hash_algo->hexsz] = '\0';
1717
3745e269 1718 parse_gitmodules_oids(cmd.out, &fsck_options.gitmodules_found);
5476e1ef 1719
dd4b732d
JT
1720 close(cmd.out);
1721
1722 if (finish_command(&cmd))
1723 die("fetch-pack: unable to finish http-fetch");
1724
1725 if (memcmp(packfile_uris.items[i].string, packname,
1726 the_hash_algo->hexsz))
1727 die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1728 uri, (int) the_hash_algo->hexsz,
1729 packfile_uris.items[i].string);
1730
1731 string_list_append_nodup(pack_lockfiles,
1732 xstrfmt("%s/pack/pack-%s.keep",
1733 get_object_directory(),
1734 packname));
1735 }
1736 string_list_clear(&packfile_uris, 0);
b664e9ff 1737 strvec_clear(&index_pack_args);
dd4b732d 1738
3745e269
ÆAB
1739 if (fsck_finish(&fsck_options))
1740 die("fsck failed");
dd4b732d 1741
603960b5
JT
1742 if (negotiator)
1743 negotiator->release(negotiator);
dd4b732d 1744
685fbd32
BW
1745 oidset_clear(&common);
1746 return ref;
1747}
1748
1362df0d
ÆAB
1749static int fetch_pack_config_cb(const char *var, const char *value, void *cb)
1750{
1751 if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1752 const char *path;
1753
1754 if (git_config_pathname(&path, var, value))
1755 return 1;
1756 strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
1757 fsck_msg_types.len ? ',' : '=', path);
1758 free((char *)path);
1759 return 0;
1760 }
1761
1762 if (skip_prefix(var, "fetch.fsck.", &var)) {
1763 if (is_valid_msg_type(var, value))
1764 strbuf_addf(&fsck_msg_types, "%c%s=%s",
1765 fsck_msg_types.len ? ',' : '=', var, value);
1766 else
1767 warning("Skipping unknown msg id '%s'", var);
1768 return 0;
1769 }
1770
1771 return git_default_config(var, value, cb);
1772}
1773
f44af51d 1774static void fetch_pack_config(void)
745f7a8c 1775{
f44af51d
TA
1776 git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1777 git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1778 git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1779 git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1780 git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1e905bbc 1781 git_config_get_bool("transfer.advertisesid", &advertise_sid);
dd4b732d
JT
1782 if (!uri_protocols.nr) {
1783 char *str;
1784
1785 if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
1786 string_list_split(&uri_protocols, str, ',', -1);
1787 free(str);
1788 }
1789 }
745f7a8c 1790
1362df0d 1791 git_config(fetch_pack_config_cb, NULL);
745f7a8c
NTND
1792}
1793
745f7a8c
NTND
1794static void fetch_pack_setup(void)
1795{
1796 static int did_setup;
1797 if (did_setup)
1798 return;
f44af51d 1799 fetch_pack_config();
745f7a8c
NTND
1800 if (0 <= transfer_unpack_limit)
1801 unpack_limit = transfer_unpack_limit;
1802 else if (0 <= fetch_unpack_limit)
1803 unpack_limit = fetch_unpack_limit;
1804 did_setup = 1;
1805}
1806
f2db854d
JH
1807static int remove_duplicates_in_refs(struct ref **ref, int nr)
1808{
1809 struct string_list names = STRING_LIST_INIT_NODUP;
1810 int src, dst;
1811
1812 for (src = dst = 0; src < nr; src++) {
1813 struct string_list_item *item;
1814 item = string_list_insert(&names, ref[src]->name);
1815 if (item->util)
1816 continue; /* already have it */
1817 item->util = ref[src];
1818 if (src != dst)
1819 ref[dst] = ref[src];
1820 dst++;
1821 }
1822 for (src = dst; src < nr; src++)
1823 ref[src] = NULL;
1824 string_list_clear(&names, 0);
1825 return dst;
1826}
1827
beea4152 1828static void update_shallow(struct fetch_pack_args *args,
e2842b39 1829 struct ref **sought, int nr_sought,
beea4152 1830 struct shallow_info *si)
a796ccee 1831{
910650d2 1832 struct oid_array ref = OID_ARRAY_INIT;
4820a33b 1833 int *status;
beea4152
NTND
1834 int i;
1835
79891cb9 1836 if (args->deepen && alternate_shallow_file) {
a796ccee 1837 if (*alternate_shallow_file == '\0') { /* --unshallow */
102de880 1838 unlink_or_warn(git_path_shallow(the_repository));
37b9dcab 1839 rollback_shallow_file(the_repository, &shallow_lock);
a796ccee 1840 } else
37b9dcab 1841 commit_shallow_file(the_repository, &shallow_lock);
23311f35 1842 alternate_shallow_file = NULL;
a796ccee
NTND
1843 return;
1844 }
beea4152
NTND
1845
1846 if (!si->shallow || !si->shallow->nr)
1847 return;
1848
beea4152
NTND
1849 if (args->cloning) {
1850 /*
1851 * remote is shallow, but this is a clone, there are
1852 * no objects in repo to worry about. Accept any
1853 * shallow points that exist in the pack (iow in repo
1854 * after get_pack() and reprepare_packed_git())
1855 */
910650d2 1856 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1857 struct object_id *oid = si->shallow->oid;
beea4152 1858 for (i = 0; i < si->shallow->nr; i++)
ee3051bd 1859 if (has_object_file(&oid[i]))
910650d2 1860 oid_array_append(&extra, &oid[i]);
beea4152
NTND
1861 if (extra.nr) {
1862 setup_alternate_shallow(&shallow_lock,
1863 &alternate_shallow_file,
1864 &extra);
37b9dcab 1865 commit_shallow_file(the_repository, &shallow_lock);
23311f35 1866 alternate_shallow_file = NULL;
beea4152 1867 }
910650d2 1868 oid_array_clear(&extra);
beea4152
NTND
1869 return;
1870 }
4820a33b
NTND
1871
1872 if (!si->nr_ours && !si->nr_theirs)
1873 return;
1874
1875 remove_nonexistent_theirs_shallow(si);
4820a33b
NTND
1876 if (!si->nr_ours && !si->nr_theirs)
1877 return;
e2842b39
JT
1878 for (i = 0; i < nr_sought; i++)
1879 oid_array_append(&ref, &sought[i]->old_oid);
4820a33b
NTND
1880 si->ref = &ref;
1881
48d25cae
NTND
1882 if (args->update_shallow) {
1883 /*
1884 * remote is also shallow, .git/shallow may be updated
1885 * so all refs can be accepted. Make sure we only add
1886 * shallow roots that are actually reachable from new
1887 * refs.
1888 */
910650d2 1889 struct oid_array extra = OID_ARRAY_INIT;
ee3051bd 1890 struct object_id *oid = si->shallow->oid;
48d25cae
NTND
1891 assign_shallow_commits_to_refs(si, NULL, NULL);
1892 if (!si->nr_ours && !si->nr_theirs) {
910650d2 1893 oid_array_clear(&ref);
48d25cae
NTND
1894 return;
1895 }
1896 for (i = 0; i < si->nr_ours; i++)
910650d2 1897 oid_array_append(&extra, &oid[si->ours[i]]);
48d25cae 1898 for (i = 0; i < si->nr_theirs; i++)
910650d2 1899 oid_array_append(&extra, &oid[si->theirs[i]]);
48d25cae
NTND
1900 setup_alternate_shallow(&shallow_lock,
1901 &alternate_shallow_file,
1902 &extra);
37b9dcab 1903 commit_shallow_file(the_repository, &shallow_lock);
910650d2 1904 oid_array_clear(&extra);
1905 oid_array_clear(&ref);
23311f35 1906 alternate_shallow_file = NULL;
48d25cae
NTND
1907 return;
1908 }
1909
4820a33b
NTND
1910 /*
1911 * remote is also shallow, check what ref is safe to update
1912 * without updating .git/shallow
1913 */
ca56dadb 1914 CALLOC_ARRAY(status, nr_sought);
4820a33b
NTND
1915 assign_shallow_commits_to_refs(si, NULL, status);
1916 if (si->nr_ours || si->nr_theirs) {
e2842b39 1917 for (i = 0; i < nr_sought; i++)
4820a33b 1918 if (status[i])
e2842b39 1919 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
4820a33b
NTND
1920 }
1921 free(status);
910650d2 1922 oid_array_clear(&ref);
a796ccee
NTND
1923}
1924
9fec7b21 1925static const struct object_id *iterate_ref_map(void *cb_data)
cf1e7c07
JT
1926{
1927 struct ref **rm = cb_data;
1928 struct ref *ref = *rm;
1929
1930 if (!ref)
9fec7b21 1931 return NULL;
cf1e7c07 1932 *rm = ref->next;
9fec7b21 1933 return &ref->old_oid;
cf1e7c07
JT
1934}
1935
745f7a8c 1936struct ref *fetch_pack(struct fetch_pack_args *args,
0f804b0b 1937 int fd[],
745f7a8c 1938 const struct ref *ref,
f2db854d 1939 struct ref **sought, int nr_sought,
910650d2 1940 struct oid_array *shallow,
9da69a65 1941 struct string_list *pack_lockfiles,
685fbd32 1942 enum protocol_version version)
745f7a8c 1943{
745f7a8c 1944 struct ref *ref_cpy;
beea4152 1945 struct shallow_info si;
1339078f 1946 struct oid_array shallows_scratch = OID_ARRAY_INIT;
745f7a8c
NTND
1947
1948 fetch_pack_setup();
f2db854d
JH
1949 if (nr_sought)
1950 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
745f7a8c 1951
01775651 1952 if (version != protocol_v2 && !ref) {
745f7a8c 1953 packet_flush(fd[1]);
1dd73e20 1954 die(_("no matching remote head"));
745f7a8c 1955 }
1e7d440b
JT
1956 if (version == protocol_v2) {
1957 if (shallow->nr)
1958 BUG("Protocol V2 does not provide shallows at this point in the fetch");
1959 memset(&si, 0, sizeof(si));
685fbd32 1960 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1339078f 1961 &shallows_scratch, &si,
9da69a65 1962 pack_lockfiles);
1e7d440b
JT
1963 } else {
1964 prepare_shallow_info(&si, shallow);
685fbd32 1965 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
9da69a65 1966 &si, pack_lockfiles);
1e7d440b 1967 }
a49d2834 1968 reprepare_packed_git(the_repository);
cf1e7c07
JT
1969
1970 if (!args->cloning && args->deepen) {
1971 struct check_connected_options opt = CHECK_CONNECTED_INIT;
1972 struct ref *iterator = ref_cpy;
1973 opt.shallow_file = alternate_shallow_file;
1974 if (args->deepen)
1975 opt.is_deepening_fetch = 1;
1976 if (check_connected(iterate_ref_map, &iterator, &opt)) {
1977 error(_("remote did not send all necessary objects"));
1978 free_refs(ref_cpy);
1979 ref_cpy = NULL;
37b9dcab 1980 rollback_shallow_file(the_repository, &shallow_lock);
cf1e7c07
JT
1981 goto cleanup;
1982 }
1983 args->connectivity_checked = 1;
1984 }
1985
e2842b39 1986 update_shallow(args, sought, nr_sought, &si);
cf1e7c07 1987cleanup:
beea4152 1988 clear_shallow_info(&si);
1339078f 1989 oid_array_clear(&shallows_scratch);
745f7a8c
NTND
1990 return ref_cpy;
1991}
e860d96b 1992
9c1e657a
JT
1993static int add_to_object_array(const struct object_id *oid, void *data)
1994{
1995 struct object_array *a = data;
1996
1997 add_object_array(lookup_object(the_repository, oid), "", a);
1998 return 0;
1999}
2000
2001static void clear_common_flag(struct oidset *s)
2002{
2003 struct oidset_iter iter;
2004 const struct object_id *oid;
2005 oidset_iter_init(s, &iter);
2006
2007 while ((oid = oidset_iter_next(&iter))) {
2008 struct object *obj = lookup_object(the_repository, oid);
2009 obj->flags &= ~COMMON;
2010 }
2011}
2012
2013void negotiate_using_fetch(const struct oid_array *negotiation_tips,
2014 const struct string_list *server_options,
2015 int stateless_rpc,
2016 int fd[],
2017 struct oidset *acked_commits)
2018{
2019 struct fetch_negotiator negotiator;
2020 struct packet_reader reader;
2021 struct object_array nt_object_array = OBJECT_ARRAY_INIT;
2022 struct strbuf req_buf = STRBUF_INIT;
2023 int haves_to_send = INITIAL_FLUSH;
2024 int in_vain = 0;
2025 int seen_ack = 0;
2026 int last_iteration = 0;
2027 timestamp_t min_generation = GENERATION_NUMBER_INFINITY;
2028
2029 fetch_negotiator_init(the_repository, &negotiator);
2030 mark_tips(&negotiator, negotiation_tips);
2031
2032 packet_reader_init(&reader, fd[0], NULL, 0,
2033 PACKET_READ_CHOMP_NEWLINE |
2034 PACKET_READ_DIE_ON_ERR_PACKET);
2035
2036 oid_array_for_each((struct oid_array *) negotiation_tips,
2037 add_to_object_array,
2038 &nt_object_array);
2039
2040 while (!last_iteration) {
2041 int haves_added;
2042 struct object_id common_oid;
2043 int received_ready = 0;
2044
2045 strbuf_reset(&req_buf);
2046 write_fetch_command_and_capabilities(&req_buf, server_options);
2047
2048 packet_buf_write(&req_buf, "wait-for-done");
2049
2050 haves_added = add_haves(&negotiator, &req_buf, &haves_to_send);
2051 in_vain += haves_added;
2052 if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN))
2053 last_iteration = 1;
2054
2055 /* Send request */
2056 packet_buf_flush(&req_buf);
2057 if (write_in_full(fd[1], req_buf.buf, req_buf.len) < 0)
2058 die_errno(_("unable to write request to remote"));
2059
2060 /* Process ACKs/NAKs */
2061 process_section_header(&reader, "acknowledgments", 0);
2062 while (process_ack(&negotiator, &reader, &common_oid,
2063 &received_ready)) {
2064 struct commit *commit = lookup_commit(the_repository,
2065 &common_oid);
2066 if (commit) {
2067 timestamp_t generation;
2068
2069 parse_commit_or_die(commit);
2070 commit->object.flags |= COMMON;
2071 generation = commit_graph_generation(commit);
2072 if (generation < min_generation)
2073 min_generation = generation;
2074 }
2075 in_vain = 0;
2076 seen_ack = 1;
2077 oidset_insert(acked_commits, &common_oid);
2078 }
2079 if (received_ready)
2080 die(_("unexpected 'ready' from remote"));
2081 else
2082 do_check_stateless_delimiter(stateless_rpc, &reader);
2083 if (can_all_from_reach_with_flag(&nt_object_array, COMMON,
2084 REACH_SCRATCH, 0,
2085 min_generation))
2086 last_iteration = 1;
2087 }
2088 clear_common_flag(acked_commits);
2089 strbuf_release(&req_buf);
2090}
2091
e860d96b
MM
2092int report_unmatched_refs(struct ref **sought, int nr_sought)
2093{
2094 int i, ret = 0;
2095
2096 for (i = 0; i < nr_sought; i++) {
d56583de 2097 if (!sought[i])
e860d96b 2098 continue;
d56583de
MM
2099 switch (sought[i]->match_status) {
2100 case REF_MATCHED:
2101 continue;
2102 case REF_NOT_MATCHED:
2103 error(_("no such remote ref %s"), sought[i]->name);
2104 break;
2105 case REF_UNADVERTISED_NOT_ALLOWED:
2106 error(_("Server does not allow request for unadvertised object %s"),
2107 sought[i]->name);
2108 break;
2109 }
e860d96b
MM
2110 ret = 1;
2111 }
2112 return ret;
2113}