[thirdparty/git.git] / http-walker.c

#include "cache.h"
#include "commit.h"
#include "walker.h"
#include "http.h"
#include "list.h"

struct alt_base {
	char *base;
	int got_indices;
	struct packed_git *packs;
	struct alt_base *next;
};

enum object_request_state {
	WAITING,
	ABORTED,
	ACTIVE,
	COMPLETE
};

struct object_request {
	struct walker *walker;
	unsigned char sha1[20];
	struct alt_base *repo;
	enum object_request_state state;
	struct http_object_request *req;
	struct list_head node;
};

struct alternates_request {
	struct walker *walker;
	const char *base;
	struct strbuf *url;
	struct strbuf *buffer;
	struct active_request_slot *slot;
	int http_specific;
};

struct walker_data {
	const char *url;
	int got_alternates;
	struct alt_base *alt;
};

static LIST_HEAD(object_queue_head);

static void fetch_alternates(struct walker *walker, const char *base);

static void process_object_response(void *callback_data);

static void start_object_request(struct walker *walker,
				 struct object_request *obj_req)
{
	struct active_request_slot *slot;
	struct http_object_request *req;

	req = new_http_object_request(obj_req->repo->base, obj_req->sha1);
	if (req == NULL) {
		obj_req->state = ABORTED;
		return;
	}
	obj_req->req = req;

	slot = req->slot;
	slot->callback_func = process_object_response;
	slot->callback_data = obj_req;

	/* Try to get the request started, abort the request on error */
	obj_req->state = ACTIVE;
	if (!start_active_slot(slot)) {
		obj_req->state = ABORTED;
		release_http_object_request(req);
		return;
	}
}

static void finish_object_request(struct object_request *obj_req)
{
	if (finish_http_object_request(obj_req->req))
		return;

	if (obj_req->req->rename == 0)
		walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
}

static void process_object_response(void *callback_data)
{
	struct object_request *obj_req =
		(struct object_request *)callback_data;
	struct walker *walker = obj_req->walker;
	struct walker_data *data = walker->data;
	struct alt_base *alt = data->alt;

	process_http_object_request(obj_req->req);
	obj_req->state = COMPLETE;

	/* Use alternates if necessary */
	if (missing_target(obj_req->req)) {
		fetch_alternates(walker, alt->base);
		if (obj_req->repo->next != NULL) {
			obj_req->repo =
				obj_req->repo->next;
			release_http_object_request(obj_req->req);
			start_object_request(walker, obj_req);
			return;
		}
	}

	finish_object_request(obj_req);
}

static void release_object_request(struct object_request *obj_req)
{
	if (obj_req->req !=NULL && obj_req->req->localfile != -1)
		error("fd leakage in release: %d", obj_req->req->localfile);

	list_del(&obj_req->node);
	free(obj_req);
}

#ifdef USE_CURL_MULTI
static int fill_active_slot(struct walker *walker)
{
	struct object_request *obj_req;
	struct list_head *pos, *tmp, *head = &object_queue_head;

	list_for_each_safe(pos, tmp, head) {
		obj_req = list_entry(pos, struct object_request, node);
		if (obj_req->state == WAITING) {
			if (has_sha1_file(obj_req->sha1))
				obj_req->state = COMPLETE;
			else {
				start_object_request(walker, obj_req);
				return 1;
			}
		}
	}
	return 0;
}
#endif

static void prefetch(struct walker *walker, unsigned char *sha1)
{
	struct object_request *newreq;
	struct walker_data *data = walker->data;

	newreq = xmalloc(sizeof(*newreq));
	newreq->walker = walker;
	hashcpy(newreq->sha1, sha1);
	newreq->repo = data->alt;
	newreq->state = WAITING;
	newreq->req = NULL;

	http_is_verbose = walker->get_verbosely;
	list_add_tail(&newreq->node, &object_queue_head);

#ifdef USE_CURL_MULTI
	fill_active_slots();
	step_active_slots();
#endif
}

static void process_alternates_response(void *callback_data)
{
	struct alternates_request *alt_req =
		(struct alternates_request *)callback_data;
	struct walker *walker = alt_req->walker;
	struct walker_data *cdata = walker->data;
	struct active_request_slot *slot = alt_req->slot;
	struct alt_base *tail = cdata->alt;
	const char *base = alt_req->base;
	const char null_byte = '\0';
	char *data;
	int i = 0;

	if (alt_req->http_specific) {
		if (slot->curl_result != CURLE_OK ||
		    !alt_req->buffer->len) {

			/* Try reusing the slot to get non-http alternates */
			alt_req->http_specific = 0;
			strbuf_reset(alt_req->url);
			strbuf_addf(alt_req->url, "%s/objects/info/alternates",
				    base);
			curl_easy_setopt(slot->curl, CURLOPT_URL,
					 alt_req->url->buf);
			active_requests++;
			slot->in_use = 1;
			if (slot->finished != NULL)
				(*slot->finished) = 0;
			if (!start_active_slot(slot)) {
				cdata->got_alternates = -1;
				slot->in_use = 0;
				if (slot->finished != NULL)
					(*slot->finished) = 1;
			}
			return;
		}
	} else if (slot->curl_result != CURLE_OK) {
		if (!missing_target(slot)) {
			cdata->got_alternates = -1;
			return;
		}
	}

	fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
	alt_req->buffer->len--;
	data = alt_req->buffer->buf;

	while (i < alt_req->buffer->len) {
		int posn = i;
		while (posn < alt_req->buffer->len && data[posn] != '\n')
			posn++;
		if (data[posn] == '\n') {
			int okay = 0;
			int serverlen = 0;
			struct alt_base *newalt;
			if (data[i] == '/') {
				/*
				 * This counts
				 * http://git.host/pub/scm/linux.git/
				 * -----------here^
				 * so memcpy(dst, base, serverlen) will
				 * copy up to "...git.host".
				 */
				const char *colon_ss = strstr(base,"://");
				if (colon_ss) {
					serverlen = (strchr(colon_ss + 3, '/')
						     - base);
					okay = 1;
				}
			} else if (!memcmp(data + i, "../", 3)) {
				/*
				 * Relative URL; chop the corresponding
				 * number of subpath from base (and ../
				 * from data), and concatenate the result.
				 *
				 * The code first drops ../ from data, and
				 * then drops one ../ from data and one path
				 * from base.  IOW, one extra ../ is dropped
				 * from data than path is dropped from base.
				 *
				 * This is not wrong.  The alternate in
				 *     http://git.host/pub/scm/linux.git/
				 * to borrow from
				 *     http://git.host/pub/scm/linus.git/
				 * is ../../linus.git/objects/.  You need
				 * two ../../ to borrow from your direct
				 * neighbour.
				 */
				i += 3;
				serverlen = strlen(base);
				while (i + 2 < posn &&
				       !memcmp(data + i, "../", 3)) {
					do {
						serverlen--;
					} while (serverlen &&
						 base[serverlen - 1] != '/');
					i += 3;
				}
				/* If the server got removed, give up. */
				okay = strchr(base, ':') - base + 3 <
				       serverlen;
			} else if (alt_req->http_specific) {
				char *colon = strchr(data + i, ':');
				char *slash = strchr(data + i, '/');
				if (colon && slash && colon < data + posn &&
				    slash < data + posn && colon < slash) {
					okay = 1;
				}
			}
			/* skip "objects\n" at end */
			if (okay) {
				struct strbuf target = STRBUF_INIT;
				strbuf_add(&target, base, serverlen);
				strbuf_add(&target, data + i, posn - i - 7);
				warning("adding alternate object store: %s",
					target.buf);
				newalt = xmalloc(sizeof(*newalt));
				newalt->next = NULL;
				newalt->base = strbuf_detach(&target, NULL);
				newalt->got_indices = 0;
				newalt->packs = NULL;

				while (tail->next != NULL)
					tail = tail->next;
				tail->next = newalt;
			}
		}
		i = posn + 1;
	}

	cdata->got_alternates = 1;
}

static void fetch_alternates(struct walker *walker, const char *base)
{
	struct strbuf buffer = STRBUF_INIT;
	struct strbuf url = STRBUF_INIT;
	struct active_request_slot *slot;
	struct alternates_request alt_req;
	struct walker_data *cdata = walker->data;

	if (http_follow_config != HTTP_FOLLOW_ALWAYS)
		return;

	/*
	 * If another request has already started fetching alternates,
	 * wait for them to arrive and return to processing this request's
	 * curl message
	 */
#ifdef USE_CURL_MULTI
	while (cdata->got_alternates == 0) {
		step_active_slots();
	}
#endif

	/* Nothing to do if they've already been fetched */
	if (cdata->got_alternates == 1)
		return;

	/* Start the fetch */
	cdata->got_alternates = 0;

	if (walker->get_verbosely)
		fprintf(stderr, "Getting alternates list for %s\n", base);

	strbuf_addf(&url, "%s/objects/info/http-alternates", base);

	/*
	 * Use a callback to process the result, since another request
	 * may fail and need to have alternates loaded before continuing
	 */
	slot = get_active_slot();
	slot->callback_func = process_alternates_response;
	alt_req.walker = walker;
	slot->callback_data = &alt_req;

	curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
	curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);

	alt_req.base = base;
	alt_req.url = &url;
	alt_req.buffer = &buffer;
	alt_req.http_specific = 1;
	alt_req.slot = slot;

	if (start_active_slot(slot))
		run_active_slot(slot);
	else
		cdata->got_alternates = -1;

	strbuf_release(&buffer);
	strbuf_release(&url);
}

static int fetch_indices(struct walker *walker, struct alt_base *repo)
{
	int ret;

	if (repo->got_indices)
		return 0;

	if (walker->get_verbosely)
		fprintf(stderr, "Getting pack list for %s\n", repo->base);

	switch (http_get_info_packs(repo->base, &repo->packs)) {
	case HTTP_OK:
	case HTTP_MISSING_TARGET:
		repo->got_indices = 1;
		ret = 0;
		break;
	default:
		repo->got_indices = 0;
		ret = -1;
	}

	return ret;
}

static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
{
	struct packed_git *target;
	int ret;
	struct slot_results results;
	struct http_pack_request *preq;

	if (fetch_indices(walker, repo))
		return -1;
	target = find_sha1_pack(sha1, repo->packs);
	if (!target)
		return -1;

	if (walker->get_verbosely) {
		fprintf(stderr, "Getting pack %s\n",
			sha1_to_hex(target->sha1));
		fprintf(stderr, " which contains %s\n",
			sha1_to_hex(sha1));
	}

	preq = new_http_pack_request(target, repo->base);
	if (preq == NULL)
		goto abort;
	preq->lst = &repo->packs;
	preq->slot->results = &results;

	if (start_active_slot(preq->slot)) {
		run_active_slot(preq->slot);
		if (results.curl_result != CURLE_OK) {
			error("Unable to get pack file %s\n%s", preq->url,
			      curl_errorstr);
			goto abort;
		}
	} else {
		error("Unable to start request");
		goto abort;
	}

	ret = finish_http_pack_request(preq);
	release_http_pack_request(preq);
	if (ret)
		return ret;

	return 0;

abort:
	return -1;
}

static void abort_object_request(struct object_request *obj_req)
{
	release_object_request(obj_req);
}

static int fetch_object(struct walker *walker, unsigned char *sha1)
{
	char *hex = sha1_to_hex(sha1);
	int ret = 0;
	struct object_request *obj_req = NULL;
	struct http_object_request *req;
	struct list_head *pos, *head = &object_queue_head;

	list_for_each(pos, head) {
		obj_req = list_entry(pos, struct object_request, node);
		if (!hashcmp(obj_req->sha1, sha1))
			break;
	}
	if (obj_req == NULL)
		return error("Couldn't find request for %s in the queue", hex);

	if (has_sha1_file(obj_req->sha1)) {
		if (obj_req->req != NULL)
			abort_http_object_request(obj_req->req);
		abort_object_request(obj_req);
		return 0;
	}

#ifdef USE_CURL_MULTI
	while (obj_req->state == WAITING)
		step_active_slots();
#else
	start_object_request(walker, obj_req);
#endif

	/*
	 * obj_req->req might change when fetching alternates in the callback
	 * process_object_response; therefore, the "shortcut" variable, req,
	 * is used only after we're done with slots.
	 */
	while (obj_req->state == ACTIVE)
		run_active_slot(obj_req->req->slot);

	req = obj_req->req;

	if (req->localfile != -1) {
		close(req->localfile);
		req->localfile = -1;
	}

	/*
	 * we turned off CURLOPT_FAILONERROR to avoid losing a
	 * persistent connection and got CURLE_OK.
	 */
	if (req->http_code == 404 && req->curl_result == CURLE_OK &&
			(starts_with(req->url, "http://") ||
			 starts_with(req->url, "https://")))
		req->curl_result = CURLE_HTTP_RETURNED_ERROR;

	if (obj_req->state == ABORTED) {
		ret = error("Request for %s aborted", hex);
	} else if (req->curl_result != CURLE_OK &&
		   req->http_code != 416) {
		if (missing_target(req))
			ret = -1; /* Be silent, it is probably in a pack. */
		else
			ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
				    req->errorstr, req->curl_result,
				    req->http_code, hex);
	} else if (req->zret != Z_STREAM_END) {
		walker->corrupt_object_found++;
		ret = error("File %s (%s) corrupt", hex, req->url);
	} else if (hashcmp(obj_req->sha1, req->real_sha1)) {
		ret = error("File %s has bad hash", hex);
	} else if (req->rename < 0) {
		ret = error("unable to write sha1 filename %s",
			    sha1_file_name(req->sha1));
	}

	release_http_object_request(req);
	release_object_request(obj_req);
	return ret;
}

static int fetch(struct walker *walker, unsigned char *sha1)
{
	struct walker_data *data = walker->data;
	struct alt_base *altbase = data->alt;

	if (!fetch_object(walker, sha1))
		return 0;
	while (altbase) {
		if (!http_fetch_pack(walker, altbase, sha1))
			return 0;
		fetch_alternates(walker, data->alt->base);
		altbase = altbase->next;
	}
	return error("Unable to find %s under %s", sha1_to_hex(sha1),
		     data->alt->base);
}

static int fetch_ref(struct walker *walker, struct ref *ref)
{
	struct walker_data *data = walker->data;
	return http_fetch_ref(data->alt->base, ref);
}

static void cleanup(struct walker *walker)
{
	struct walker_data *data = walker->data;
	struct alt_base *alt, *alt_next;

	if (data) {
		alt = data->alt;
		while (alt) {
			alt_next = alt->next;

			free(alt->base);
			free(alt);

			alt = alt_next;
		}
		free(data);
		walker->data = NULL;
	}
}

struct walker *get_http_walker(const char *url)
{
	char *s;
	struct walker_data *data = xmalloc(sizeof(struct walker_data));
	struct walker *walker = xmalloc(sizeof(struct walker));

	data->alt = xmalloc(sizeof(*data->alt));
	data->alt->base = xstrdup(url);
	for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
		*s = 0;

	data->alt->got_indices = 0;
	data->alt->packs = NULL;
	data->alt->next = NULL;
	data->got_alternates = -1;

	walker->corrupt_object_found = 0;
	walker->fetch = fetch;
	walker->fetch_ref = fetch_ref;
	walker->prefetch = prefetch;
	walker->cleanup = cleanup;
	walker->data = data;

#ifdef USE_CURL_MULTI
	add_fill_function(walker, (int (*)(void *)) fill_active_slot);
#endif

	return walker;
}
Commit	Line	Data
6eb7ed54 DB	1	#include "cache.h"
6eb7ed54 DB	2	#include "commit.h"
30ae764b	3	#include "walker.h"
29508e1e	4	#include "http.h"
94e99012	5	#include "list.h"
7baa3e86	6
9cba13ca	7	struct alt_base {
2afea3bc	8	char *base;
b3661567 DB	9	int got_indices;
	10	struct packed_git *packs;
	11	struct alt_base *next;
	12	};
	13
e388ab74	14	enum object_request_state {
1d389ab6 NH	15	WAITING,
	16	ABORTED,
	17	ACTIVE,
4b05548f	18	COMPLETE
1d389ab6	19	};
6eb7ed54	20
9cba13ca	21	struct object_request {
30ae764b	22	struct walker *walker;
1d389ab6 NH	23	unsigned char sha1[20];
1d389ab6 NH	24	struct alt_base *repo;
e388ab74	25	enum object_request_state state;
5424bc55	26	struct http_object_request *req;
94e99012	27	struct list_head node;
1d389ab6 NH	28	};
1d389ab6 NH	29
e388ab74	30	struct alternates_request {
30ae764b	31	struct walker *walker;
8e29f6a0	32	const char *base;
54ba4c5f	33	struct strbuf *url;
028c2976	34	struct strbuf *buffer;
acc075a8 NH	35	struct active_request_slot *slot;
	36	int http_specific;
	37	};
	38
30ae764b DB	39	struct walker_data {
	40	const char *url;
	41	int got_alternates;
	42	struct alt_base *alt;
30ae764b DB	43	};
30ae764b DB	44
94e99012	45	static LIST_HEAD(object_queue_head);
bc8f2652	46
30ae764b	47	static void fetch_alternates(struct walker walker, const char base);
1d389ab6	48
29508e1e	49	static void process_object_response(void *callback_data);
1d389ab6	50
30ae764b DB	51	static void start_object_request(struct walker *walker,
30ae764b DB	52	struct object_request *obj_req)
1d389ab6	53	{
1d389ab6	54	struct active_request_slot *slot;
5424bc55	55	struct http_object_request *req;
1d389ab6	56
5424bc55 TRC	57	req = new_http_object_request(obj_req->repo->base, obj_req->sha1);
5424bc55 TRC	58	if (req == NULL) {
e388ab74	59	obj_req->state = ABORTED;
1d389ab6 NH	60	return;
1d389ab6 NH	61	}
5424bc55	62	obj_req->req = req;
1d389ab6	63
5424bc55	64	slot = req->slot;
29508e1e	65	slot->callback_func = process_object_response;
e388ab74	66	slot->callback_data = obj_req;
1d389ab6	67
a7a8d378	68	/* Try to get the request started, abort the request on error */
e388ab74	69	obj_req->state = ACTIVE;
1d389ab6	70	if (!start_active_slot(slot)) {
e388ab74	71	obj_req->state = ABORTED;
5424bc55	72	release_http_object_request(req);
e388ab74	73	return;
1d389ab6	74	}
1d389ab6 NH	75	}
1d389ab6 NH	76
e388ab74	77	static void finish_object_request(struct object_request *obj_req)
1d389ab6	78	{
5424bc55	79	if (finish_http_object_request(obj_req->req))
1d389ab6	80	return;
1d389ab6	81
5424bc55	82	if (obj_req->req->rename == 0)
30ae764b	83	walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
1d389ab6 NH	84	}
1d389ab6 NH	85
29508e1e NH	86	static void process_object_response(void *callback_data)
29508e1e NH	87	{
e388ab74 NH	88	struct object_request *obj_req =
e388ab74 NH	89	(struct object_request *)callback_data;
30ae764b DB	90	struct walker *walker = obj_req->walker;
	91	struct walker_data *data = walker->data;
	92	struct alt_base *alt = data->alt;
29508e1e	93
5424bc55	94	process_http_object_request(obj_req->req);
e388ab74	95	obj_req->state = COMPLETE;
29508e1e NH	96
29508e1e NH	97	/* Use alternates if necessary */
5424bc55	98	if (missing_target(obj_req->req)) {
30ae764b	99	fetch_alternates(walker, alt->base);
e388ab74 NH	100	if (obj_req->repo->next != NULL) {
	101	obj_req->repo =
	102	obj_req->repo->next;
5424bc55	103	release_http_object_request(obj_req->req);
30ae764b	104	start_object_request(walker, obj_req);
29508e1e NH	105	return;
	106	}
	107	}
	108
e388ab74	109	finish_object_request(obj_req);
29508e1e NH	110	}
29508e1e NH	111
e388ab74	112	static void release_object_request(struct object_request *obj_req)
1d389ab6	113	{
5424bc55 TRC	114	if (obj_req->req !=NULL && obj_req->req->localfile != -1)
5424bc55 TRC	115	error("fd leakage in release: %d", obj_req->req->localfile);
1d389ab6	116
94e99012	117	list_del(&obj_req->node);
e388ab74	118	free(obj_req);
1d389ab6 NH	119	}
1d389ab6 NH	120
a7a8d378	121	#ifdef USE_CURL_MULTI
30ae764b	122	static int fill_active_slot(struct walker *walker)
1d389ab6	123	{
45c17412	124	struct object_request *obj_req;
94e99012	125	struct list_head pos, tmp, *head = &object_queue_head;
1d389ab6	126
94e99012 EW	127	list_for_each_safe(pos, tmp, head) {
94e99012 EW	128	obj_req = list_entry(pos, struct object_request, node);
e388ab74 NH	129	if (obj_req->state == WAITING) {
e388ab74 NH	130	if (has_sha1_file(obj_req->sha1))
09db444f	131	obj_req->state = COMPLETE;
45c17412	132	else {
30ae764b	133	start_object_request(walker, obj_req);
45c17412 DB	134	return 1;
45c17412 DB	135	}
f1a906a3	136	}
8fcf7f9a	137	}
45c17412	138	return 0;
1d389ab6	139	}
a7a8d378	140	#endif
1d389ab6	141
30ae764b	142	static void prefetch(struct walker walker, unsigned char sha1)
1d389ab6	143	{
e388ab74	144	struct object_request *newreq;
30ae764b	145	struct walker_data *data = walker->data;
1d389ab6 NH	146
1d389ab6 NH	147	newreq = xmalloc(sizeof(*newreq));
30ae764b	148	newreq->walker = walker;
e702496e	149	hashcpy(newreq->sha1, sha1);
30ae764b	150	newreq->repo = data->alt;
1d389ab6	151	newreq->state = WAITING;
5424bc55	152	newreq->req = NULL;
1d389ab6	153
e9176745	154	http_is_verbose = walker->get_verbosely;
94e99012	155	list_add_tail(&newreq->node, &object_queue_head);
29508e1e	156
a7a8d378	157	#ifdef USE_CURL_MULTI
29508e1e NH	158	fill_active_slots();
29508e1e NH	159	step_active_slots();
a7a8d378	160	#endif
1d389ab6 NH	161	}
1d389ab6 NH	162
e388ab74	163	static void process_alternates_response(void *callback_data)
b3661567	164	{
e388ab74 NH	165	struct alternates_request *alt_req =
e388ab74 NH	166	(struct alternates_request *)callback_data;
30ae764b DB	167	struct walker *walker = alt_req->walker;
30ae764b DB	168	struct walker_data *cdata = walker->data;
acc075a8	169	struct active_request_slot *slot = alt_req->slot;
30ae764b	170	struct alt_base *tail = cdata->alt;
8e29f6a0	171	const char *base = alt_req->base;
a04ff3ec	172	const char null_byte = '\0';
acc075a8 NH	173	char *data;
acc075a8 NH	174	int i = 0;
1d389ab6	175
acc075a8 NH	176	if (alt_req->http_specific) {
acc075a8 NH	177	if (slot->curl_result != CURLE_OK \|\|
028c2976	178	!alt_req->buffer->len) {
acc075a8 NH	179
	180	/* Try reusing the slot to get non-http alternates */
	181	alt_req->http_specific = 0;
54ba4c5f JK	182	strbuf_reset(alt_req->url);
	183	strbuf_addf(alt_req->url, "%s/objects/info/alternates",
	184	base);
acc075a8	185	curl_easy_setopt(slot->curl, CURLOPT_URL,
54ba4c5f	186	alt_req->url->buf);
acc075a8 NH	187	active_requests++;
acc075a8 NH	188	slot->in_use = 1;
c9826473 NH	189	if (slot->finished != NULL)
c9826473 NH	190	(*slot->finished) = 0;
a3f583cb	191	if (!start_active_slot(slot)) {
30ae764b	192	cdata->got_alternates = -1;
29508e1e	193	slot->in_use = 0;
c9826473 NH	194	if (slot->finished != NULL)
c9826473 NH	195	(*slot->finished) = 1;
1d389ab6	196	}
a3f583cb	197	return;
b3661567	198	}
acc075a8	199	} else if (slot->curl_result != CURLE_OK) {
be4a015b	200	if (!missing_target(slot)) {
30ae764b	201	cdata->got_alternates = -1;
acc075a8 NH	202	return;
acc075a8 NH	203	}
b3661567 DB	204	}
b3661567 DB	205
a04ff3ec	206	fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
028c2976 MH	207	alt_req->buffer->len--;
028c2976 MH	208	data = alt_req->buffer->buf;
1b0c1e67	209
028c2976	210	while (i < alt_req->buffer->len) {
b3661567	211	int posn = i;
028c2976	212	while (posn < alt_req->buffer->len && data[posn] != '\n')
b3661567 DB	213	posn++;
b3661567 DB	214	if (data[posn] == '\n') {
1b0c1e67 DB	215	int okay = 0;
	216	int serverlen = 0;
	217	struct alt_base *newalt;
b3661567	218	if (data[i] == '/') {
4c42aa1a TRC	219	/*
4c42aa1a TRC	220	* This counts
5df1e0d0 JH	221	* http://git.host/pub/scm/linux.git/
	222	* -----------here^
	223	* so memcpy(dst, base, serverlen) will
	224	* copy up to "...git.host".
	225	*/
	226	const char *colon_ss = strstr(base,"://");
	227	if (colon_ss) {
	228	serverlen = (strchr(colon_ss + 3, '/')
	229	- base);
	230	okay = 1;
	231	}
1b0c1e67	232	} else if (!memcmp(data + i, "../", 3)) {
4c42aa1a TRC	233	/*
4c42aa1a TRC	234	* Relative URL; chop the corresponding
5df1e0d0 JH	235	* number of subpath from base (and ../
	236	* from data), and concatenate the result.
	237	*
	238	* The code first drops ../ from data, and
	239	* then drops one ../ from data and one path
	240	* from base. IOW, one extra ../ is dropped
	241	* from data than path is dropped from base.
	242	*
	243	* This is not wrong. The alternate in
	244	* http://git.host/pub/scm/linux.git/
	245	* to borrow from
	246	* http://git.host/pub/scm/linus.git/
	247	* is ../../linus.git/objects/. You need
	248	* two ../../ to borrow from your direct
	249	* neighbour.
	250	*/
1b0c1e67 DB	251	i += 3;
1b0c1e67 DB	252	serverlen = strlen(base);
8fcf7f9a	253	while (i + 2 < posn &&
1b0c1e67 DB	254	!memcmp(data + i, "../", 3)) {
	255	do {
	256	serverlen--;
	257	} while (serverlen &&
	258	base[serverlen - 1] != '/');
	259	i += 3;
	260	}
a9486b02	261	/* If the server got removed, give up. */
8fcf7f9a	262	okay = strchr(base, ':') - base + 3 <
4c42aa1a	263	serverlen;
acc075a8	264	} else if (alt_req->http_specific) {
1b0c1e67 DB	265	char *colon = strchr(data + i, ':');
	266	char *slash = strchr(data + i, '/');
	267	if (colon && slash && colon < data + posn &&
	268	slash < data + posn && colon < slash) {
	269	okay = 1;
	270	}
	271	}
5df1e0d0	272	/* skip "objects\n" at end */
1b0c1e67	273	if (okay) {
59b8263a RS	274	struct strbuf target = STRBUF_INIT;
	275	strbuf_add(&target, base, serverlen);
	276	strbuf_add(&target, data + i, posn - i - 7);
cb4d2d35 JK	277	warning("adding alternate object store: %s",
cb4d2d35 JK	278	target.buf);
b3661567	279	newalt = xmalloc(sizeof(*newalt));
1d389ab6	280	newalt->next = NULL;
59b8263a	281	newalt->base = strbuf_detach(&target, NULL);
b3661567 DB	282	newalt->got_indices = 0;
b3661567 DB	283	newalt->packs = NULL;
8d9fbe57	284
1d389ab6 NH	285	while (tail->next != NULL)
	286	tail = tail->next;
	287	tail->next = newalt;
b3661567 DB	288	}
	289	}
	290	i = posn + 1;
	291	}
bc8f2652	292
30ae764b	293	cdata->got_alternates = 1;
acc075a8 NH	294	}
acc075a8 NH	295
30ae764b	296	static void fetch_alternates(struct walker walker, const char base)
acc075a8	297	{
028c2976	298	struct strbuf buffer = STRBUF_INIT;
54ba4c5f	299	struct strbuf url = STRBUF_INIT;
acc075a8	300	struct active_request_slot *slot;
cb754fdf	301	struct alternates_request alt_req;
30ae764b	302	struct walker_data *cdata = walker->data;
acc075a8	303
cb4d2d35 JK	304	if (http_follow_config != HTTP_FOLLOW_ALWAYS)
	305	return;
	306
4c42aa1a TRC	307	/*
	308	* If another request has already started fetching alternates,
	309	* wait for them to arrive and return to processing this request's
	310	* curl message
	311	*/
29508e1e	312	#ifdef USE_CURL_MULTI
30ae764b	313	while (cdata->got_alternates == 0) {
29508e1e	314	step_active_slots();
acc075a8	315	}
29508e1e	316	#endif
acc075a8 NH	317
acc075a8 NH	318	/* Nothing to do if they've already been fetched */
30ae764b	319	if (cdata->got_alternates == 1)
acc075a8 NH	320	return;
	321
	322	/* Start the fetch */
30ae764b	323	cdata->got_alternates = 0;
acc075a8	324
30ae764b	325	if (walker->get_verbosely)
acc075a8	326	fprintf(stderr, "Getting alternates list for %s\n", base);
8fcf7f9a	327
54ba4c5f	328	strbuf_addf(&url, "%s/objects/info/http-alternates", base);
acc075a8	329
4c42aa1a TRC	330	/*
	331	* Use a callback to process the result, since another request
	332	* may fail and need to have alternates loaded before continuing
	333	*/
acc075a8	334	slot = get_active_slot();
e388ab74	335	slot->callback_func = process_alternates_response;
30ae764b	336	alt_req.walker = walker;
acc075a8 NH	337	slot->callback_data = &alt_req;
	338
	339	curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
29508e1e	340	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
54ba4c5f	341	curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
acc075a8 NH	342
acc075a8 NH	343	alt_req.base = base;
54ba4c5f	344	alt_req.url = &url;
acc075a8 NH	345	alt_req.buffer = &buffer;
	346	alt_req.http_specific = 1;
	347	alt_req.slot = slot;
	348
	349	if (start_active_slot(slot))
	350	run_active_slot(slot);
	351	else
30ae764b	352	cdata->got_alternates = -1;
acc075a8	353
028c2976	354	strbuf_release(&buffer);
54ba4c5f	355	strbuf_release(&url);
b3661567 DB	356	}
b3661567 DB	357
30ae764b	358	static int fetch_indices(struct walker walker, struct alt_base repo)
182005b9	359	{
b8caac2b	360	int ret;
1d389ab6	361
b3661567	362	if (repo->got_indices)
182005b9 DB	363	return 0;
182005b9 DB	364
30ae764b	365	if (walker->get_verbosely)
6fd72e39	366	fprintf(stderr, "Getting pack list for %s\n", repo->base);
8fcf7f9a	367
b8caac2b TRC	368	switch (http_get_info_packs(repo->base, &repo->packs)) {
	369	case HTTP_OK:
	370	case HTTP_MISSING_TARGET:
	371	repo->got_indices = 1;
	372	ret = 0;
	373	break;
	374	default:
5e3a7691	375	repo->got_indices = 0;
b8caac2b	376	ret = -1;
b3661567	377	}
182005b9	378
3a462bc9	379	return ret;
182005b9 DB	380	}
182005b9 DB	381
07c19e72	382	static int http_fetch_pack(struct walker walker, struct alt_base repo, unsigned char *sha1)
182005b9	383	{
182005b9	384	struct packed_git *target;
49a0f240	385	int ret;
cb754fdf	386	struct slot_results results;
2264dfa5	387	struct http_pack_request *preq;
182005b9	388
30ae764b	389	if (fetch_indices(walker, repo))
182005b9	390	return -1;
b3661567	391	target = find_sha1_pack(sha1, repo->packs);
182005b9	392	if (!target)
b3661567	393	return -1;
182005b9	394
30ae764b	395	if (walker->get_verbosely) {
182005b9 DB	396	fprintf(stderr, "Getting pack %s\n",
	397	sha1_to_hex(target->sha1));
	398	fprintf(stderr, " which contains %s\n",
	399	sha1_to_hex(sha1));
	400	}
	401
2264dfa5 TRC	402	preq = new_http_pack_request(target, repo->base);
	403	if (preq == NULL)
	404	goto abort;
	405	preq->lst = &repo->packs;
	406	preq->slot->results = &results;
182005b9	407
2264dfa5 TRC	408	if (start_active_slot(preq->slot)) {
2264dfa5 TRC	409	run_active_slot(preq->slot);
c8568e13	410	if (results.curl_result != CURLE_OK) {
2264dfa5 TRC	411	error("Unable to get pack file %s\n%s", preq->url,
	412	curl_errorstr);
	413	goto abort;
1d389ab6 NH	414	}
1d389ab6 NH	415	} else {
2264dfa5 TRC	416	error("Unable to start request");
2264dfa5 TRC	417	goto abort;
182005b9 DB	418	}
182005b9 DB	419
2264dfa5 TRC	420	ret = finish_http_pack_request(preq);
2264dfa5 TRC	421	release_http_pack_request(preq);
49a0f240	422	if (ret)
b721e01f	423	return ret;
49a0f240	424
182005b9	425	return 0;
2264dfa5 TRC	426
	427	abort:
	428	return -1;
182005b9 DB	429	}
182005b9 DB	430
53f31389 MW	431	static void abort_object_request(struct object_request *obj_req)
53f31389 MW	432	{
53f31389 MW	433	release_object_request(obj_req);
	434	}
	435
43b8bba6	436	static int fetch_object(struct walker walker, unsigned char sha1)
6eb7ed54 DB	437	{
6eb7ed54 DB	438	char *hex = sha1_to_hex(sha1);
29508e1e	439	int ret = 0;
94e99012	440	struct object_request *obj_req = NULL;
5424bc55	441	struct http_object_request *req;
94e99012	442	struct list_head pos, head = &object_queue_head;
1d389ab6	443
94e99012 EW	444	list_for_each(pos, head) {
	445	obj_req = list_entry(pos, struct object_request, node);
	446	if (!hashcmp(obj_req->sha1, sha1))
	447	break;
	448	}
e388ab74	449	if (obj_req == NULL)
1d389ab6 NH	450	return error("Couldn't find request for %s in the queue", hex);
1d389ab6 NH	451
e388ab74	452	if (has_sha1_file(obj_req->sha1)) {
5424bc55 TRC	453	if (obj_req->req != NULL)
5424bc55 TRC	454	abort_http_object_request(obj_req->req);
53f31389	455	abort_object_request(obj_req);
11f0dafe NH	456	return 0;
	457	}
	458
a7a8d378	459	#ifdef USE_CURL_MULTI
4c42aa1a	460	while (obj_req->state == WAITING)
29508e1e	461	step_active_slots();
a7a8d378	462	#else
30ae764b	463	start_object_request(walker, obj_req);
a7a8d378	464	#endif
6eb7ed54	465
5424bc55 TRC	466	/*
	467	* obj_req->req might change when fetching alternates in the callback
	468	* process_object_response; therefore, the "shortcut" variable, req,
	469	* is used only after we're done with slots.
	470	*/
4c42aa1a	471	while (obj_req->state == ACTIVE)
5424bc55 TRC	472	run_active_slot(obj_req->req->slot);
	473
	474	req = obj_req->req;
4c42aa1a	475
5424bc55 TRC	476	if (req->localfile != -1) {
	477	close(req->localfile);
	478	req->localfile = -1;
313c4714	479	}
6eb7ed54	480
17966c0a EW	481	/*
	482	* we turned off CURLOPT_FAILONERROR to avoid losing a
	483	* persistent connection and got CURLE_OK.
	484	*/
	485	if (req->http_code == 404 && req->curl_result == CURLE_OK &&
	486	(starts_with(req->url, "http://") \|\|
	487	starts_with(req->url, "https://")))
	488	req->curl_result = CURLE_HTTP_RETURNED_ERROR;
	489
e388ab74	490	if (obj_req->state == ABORTED) {
29508e1e	491	ret = error("Request for %s aborted", hex);
5424bc55 TRC	492	} else if (req->curl_result != CURLE_OK &&
	493	req->http_code != 416) {
	494	if (missing_target(req))
e2029eb9 PB	495	ret = -1; /* Be silent, it is probably in a pack. */
	496	else
	497	ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
5424bc55 TRC	498	req->errorstr, req->curl_result,
	499	req->http_code, hex);
	500	} else if (req->zret != Z_STREAM_END) {
30ae764b	501	walker->corrupt_object_found++;
5424bc55 TRC	502	ret = error("File %s (%s) corrupt", hex, req->url);
5424bc55 TRC	503	} else if (hashcmp(obj_req->sha1, req->real_sha1)) {
bd2afde8	504	ret = error("File %s has bad hash", hex);
5424bc55	505	} else if (req->rename < 0) {
7b934ec0	506	ret = error("unable to write sha1 filename %s",
0da8b2e7	507	sha1_file_name(req->sha1));
6eb7ed54	508	}
49a0f240	509
5424bc55	510	release_http_object_request(req);
e388ab74	511	release_object_request(obj_req);
29508e1e	512	return ret;
6eb7ed54 DB	513	}
6eb7ed54 DB	514
30ae764b	515	static int fetch(struct walker walker, unsigned char sha1)
b3661567	516	{
30ae764b DB	517	struct walker_data *data = walker->data;
30ae764b DB	518	struct alt_base *altbase = data->alt;
1d389ab6	519
43b8bba6	520	if (!fetch_object(walker, sha1))
1d389ab6	521	return 0;
b3661567	522	while (altbase) {
07c19e72	523	if (!http_fetch_pack(walker, altbase, sha1))
b3661567	524	return 0;
30ae764b	525	fetch_alternates(walker, data->alt->base);
b3661567 DB	526	altbase = altbase->next;
b3661567 DB	527	}
bd2afde8	528	return error("Unable to find %s under %s", sha1_to_hex(sha1),
30ae764b	529	data->alt->base);
b3661567 DB	530	}
b3661567 DB	531
c13b2633	532	static int fetch_ref(struct walker walker, struct ref ref)
cd541a68	533	{
30ae764b	534	struct walker_data *data = walker->data;
c13b2633	535	return http_fetch_ref(data->alt->base, ref);
cd541a68 DB	536	}
cd541a68 DB	537
30ae764b DB	538	static void cleanup(struct walker *walker)
30ae764b DB	539	{
09ae9aca TRC	540	struct walker_data *data = walker->data;
	541	struct alt_base alt, alt_next;
	542
	543	if (data) {
	544	alt = data->alt;
	545	while (alt) {
	546	alt_next = alt->next;
	547
	548	free(alt->base);
	549	free(alt);
	550
	551	alt = alt_next;
	552	}
	553	free(data);
	554	walker->data = NULL;
	555	}
30ae764b DB	556	}
30ae764b DB	557
888692b7	558	struct walker get_http_walker(const char url)
6eb7ed54	559	{
9c880b3e	560	char *s;
30ae764b DB	561	struct walker_data *data = xmalloc(sizeof(struct walker_data));
30ae764b DB	562	struct walker *walker = xmalloc(sizeof(struct walker));
6eb7ed54	563
30ae764b	564	data->alt = xmalloc(sizeof(*data->alt));
95244ae3	565	data->alt->base = xstrdup(url);
30ae764b	566	for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
9c880b3e	567	*s = 0;
6eb7ed54	568
30ae764b DB	569	data->alt->got_indices = 0;
	570	data->alt->packs = NULL;
	571	data->alt->next = NULL;
	572	data->got_alternates = -1;
fc57b6aa	573
30ae764b DB	574	walker->corrupt_object_found = 0;
	575	walker->fetch = fetch;
	576	walker->fetch_ref = fetch_ref;
	577	walker->prefetch = prefetch;
	578	walker->cleanup = cleanup;
	579	walker->data = data;
6eb7ed54	580
30ae764b DB	581	#ifdef USE_CURL_MULTI
	582	add_fill_function(walker, (int ()(void )) fill_active_slot);
	583	#endif
8e29f6a0	584
30ae764b	585	return walker;
6eb7ed54	586	}