promisor.quiet::
If set to "true" assume `--quiet` when fetching additional
objects for a partial clone.
+
+promisor.advertise::
+ If set to "true", a server will use the "promisor-remote"
+ capability, see linkgit:gitprotocol-v2[5], to advertise the
+ promisor remotes it is using, if it uses some. Default is
+ "false", which means the "promisor-remote" capability is not
+ advertised.
+
+promisor.acceptFromServer::
+ If set to "all", a client will accept all the promisor remotes
+ a server might advertise using the "promisor-remote"
+ capability. Default is "none", which means no promisor remote
+ advertised by a server will be accepted. By accepting a
+ promisor remote, the client agrees that the server might omit
+ objects that are lazily fetchable from this promisor remote
+ from its responses to "fetch" and "clone" requests from the
+ client. See linkgit:gitprotocol-v2[5].
save themselves and the server(s) the request(s) needed to inspect the
headers of that bundle or bundles.
+promisor-remote=<pr-infos>
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The server may advertise some promisor remotes it is using or knows
+about to a client which may want to use them as its promisor remotes,
+instead of this repository. In this case <pr-infos> should be of the
+form:
+
+ pr-infos = pr-info | pr-infos ";" pr-info
+
+ pr-info = "name=" pr-name | "name=" pr-name "," "url=" pr-url
+
+where `pr-name` is the urlencoded name of a promisor remote, and
+`pr-url` the urlencoded URL of that promisor remote.
+
+In this case, if the client decides to use one or more promisor
+remotes the server advertised, it can reply with
+"promisor-remote=<pr-names>" where <pr-names> should be of the form:
+
+ pr-names = pr-name | pr-names ";" pr-name
+
+where `pr-name` is the urlencoded name of a promisor remote the server
+advertised and the client accepts.
+
+Note that, everywhere in this document, `pr-name` MUST be a valid
+remote name, and the ';' and ',' characters MUST be encoded if they
+appear in `pr-name` or `pr-url`.
+
+If the server doesn't know any promisor remote that could be good for
+a client to use, or prefers a client not to use any promisor remote it
+uses or knows about, it shouldn't advertise the "promisor-remote"
+capability at all.
+
+In this case, or if the client doesn't want to use any promisor remote
+the server advertised, the client shouldn't advertise the
+"promisor-remote" capability at all in its reply.
+
+The "promisor.advertise" and "promisor.acceptFromServer" configuration
+options can be used on the server and client side to control what they
+advertise or accept respectively. See the documentation of these
+configuration options for more information.
+
+Note that in the future it would be nice if the "promisor-remote"
+protocol capability could be used by the server, when responding to
+`git fetch` or `git clone`, to advertise better-connected remotes that
+the client can use as promisor remotes, instead of this repository, so
+that the client can lazily fetch objects from these other
+better-connected remotes. This would require the server to omit in its
+response the objects available on the better-connected remotes that
+the client has accepted. This hasn't been implemented yet though. So
+for now this "promisor-remote" capability is useful only when the
+server advertises some promisor remotes it already uses to borrow
+objects from.
+
GIT
---
Part of the linkgit:git[1] suite
#include "protocol.h"
#include "alias.h"
#include "bundle-uri.h"
+#include "promisor-remote.h"
static char *server_capabilities_v1;
static struct strvec server_capabilities_v2 = STRVEC_INIT;
static void send_capabilities(int fd_out, struct packet_reader *reader)
{
const char *hash_name;
+ const char *promisor_remote_info;
if (server_supports_v2("agent"))
packet_write_fmt(fd_out, "agent=%s", git_user_agent_sanitized());
} else {
reader->hash_algo = &hash_algos[GIT_HASH_SHA1];
}
+ if (server_feature_v2("promisor-remote", &promisor_remote_info)) {
+ char *reply = promisor_remote_reply(promisor_remote_info);
+ if (reply) {
+ packet_write_fmt(fd_out, "promisor-remote=%s", reply);
+ free(reply);
+ }
+ }
}
int get_remote_bundle_uri(int fd_out, struct packet_reader *reader,
#include "strvec.h"
#include "packfile.h"
#include "environment.h"
+#include "url.h"
+#include "version.h"
struct promisor_remote_config {
struct promisor_remote *promisors;
return !!repo_promisor_remote_find(r, NULL);
}
+int repo_has_accepted_promisor_remote(struct repository *r)
+{
+ struct promisor_remote *p;
+
+ promisor_remote_init(r);
+
+ for (p = r->promisor_remote_config->promisors; p; p = p->next)
+ if (p->accepted)
+ return 1;
+ return 0;
+}
+
static int remove_fetched_oids(struct repository *repo,
struct object_id **oids,
int oid_nr, int to_free)
if (to_free)
free(remaining_oids);
}
+
+static int allow_unsanitized(char ch)
+{
+ if (ch == ',' || ch == ';' || ch == '%')
+ return 0;
+ return ch > 32 && ch < 127;
+}
+
+static void promisor_info_vecs(struct repository *repo,
+ struct strvec *names,
+ struct strvec *urls)
+{
+ struct promisor_remote *r;
+
+ promisor_remote_init(repo);
+
+ for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
+ char *url;
+ char *url_key = xstrfmt("remote.%s.url", r->name);
+
+ strvec_push(names, r->name);
+ strvec_push(urls, git_config_get_string(url_key, &url) ? NULL : url);
+
+ free(url);
+ free(url_key);
+ }
+}
+
+char *promisor_remote_info(struct repository *repo)
+{
+ struct strbuf sb = STRBUF_INIT;
+ int advertise_promisors = 0;
+ struct strvec names = STRVEC_INIT;
+ struct strvec urls = STRVEC_INIT;
+
+ git_config_get_bool("promisor.advertise", &advertise_promisors);
+
+ if (!advertise_promisors)
+ return NULL;
+
+ promisor_info_vecs(repo, &names, &urls);
+
+ if (!names.nr)
+ return NULL;
+
+ for (size_t i = 0; i < names.nr; i++) {
+ if (i)
+ strbuf_addch(&sb, ';');
+ strbuf_addstr(&sb, "name=");
+ strbuf_addstr_urlencode(&sb, names.v[i], allow_unsanitized);
+ if (urls.v[i]) {
+ strbuf_addstr(&sb, ",url=");
+ strbuf_addstr_urlencode(&sb, urls.v[i], allow_unsanitized);
+ }
+ }
+
+ strvec_clear(&names);
+ strvec_clear(&urls);
+
+ return strbuf_detach(&sb, NULL);
+}
+
+enum accept_promisor {
+ ACCEPT_NONE = 0,
+ ACCEPT_ALL
+};
+
+static int should_accept_remote(enum accept_promisor accept,
+ const char *remote_name UNUSED,
+ const char *remote_url UNUSED)
+{
+ if (accept == ACCEPT_ALL)
+ return 1;
+
+ BUG("Unhandled 'enum accept_promisor' value '%d'", accept);
+}
+
+static void filter_promisor_remote(struct strvec *accepted, const char *info)
+{
+ struct strbuf **remotes;
+ const char *accept_str;
+ enum accept_promisor accept = ACCEPT_NONE;
+
+ if (!git_config_get_string_tmp("promisor.acceptfromserver", &accept_str)) {
+ if (!*accept_str || !strcasecmp("None", accept_str))
+ accept = ACCEPT_NONE;
+ else if (!strcasecmp("All", accept_str))
+ accept = ACCEPT_ALL;
+ else
+ warning(_("unknown '%s' value for '%s' config option"),
+ accept_str, "promisor.acceptfromserver");
+ }
+
+ if (accept == ACCEPT_NONE)
+ return;
+
+ /* Parse remote info received */
+
+ remotes = strbuf_split_str(info, ';', 0);
+
+ for (size_t i = 0; remotes[i]; i++) {
+ struct strbuf **elems;
+ const char *remote_name = NULL;
+ const char *remote_url = NULL;
+ char *decoded_name = NULL;
+ char *decoded_url = NULL;
+
+ strbuf_strip_suffix(remotes[i], ";");
+ elems = strbuf_split(remotes[i], ',');
+
+ for (size_t j = 0; elems[j]; j++) {
+ int res;
+ strbuf_strip_suffix(elems[j], ",");
+ res = skip_prefix(elems[j]->buf, "name=", &remote_name) ||
+ skip_prefix(elems[j]->buf, "url=", &remote_url);
+ if (!res)
+ warning(_("unknown element '%s' from remote info"),
+ elems[j]->buf);
+ }
+
+ if (remote_name)
+ decoded_name = url_percent_decode(remote_name);
+ if (remote_url)
+ decoded_url = url_percent_decode(remote_url);
+
+ if (decoded_name && should_accept_remote(accept, decoded_name, decoded_url))
+ strvec_push(accepted, decoded_name);
+
+ strbuf_list_free(elems);
+ free(decoded_name);
+ free(decoded_url);
+ }
+
+ strbuf_list_free(remotes);
+}
+
+char *promisor_remote_reply(const char *info)
+{
+ struct strvec accepted = STRVEC_INIT;
+ struct strbuf reply = STRBUF_INIT;
+
+ filter_promisor_remote(&accepted, info);
+
+ if (!accepted.nr)
+ return NULL;
+
+ for (size_t i = 0; i < accepted.nr; i++) {
+ if (i)
+ strbuf_addch(&reply, ';');
+ strbuf_addstr_urlencode(&reply, accepted.v[i], allow_unsanitized);
+ }
+
+ strvec_clear(&accepted);
+
+ return strbuf_detach(&reply, NULL);
+}
+
+void mark_promisor_remotes_as_accepted(struct repository *r, const char *remotes)
+{
+ struct strbuf **accepted_remotes = strbuf_split_str(remotes, ';', 0);
+
+ for (size_t i = 0; accepted_remotes[i]; i++) {
+ struct promisor_remote *p;
+ char *decoded_remote;
+
+ strbuf_strip_suffix(accepted_remotes[i], ";");
+ decoded_remote = url_percent_decode(accepted_remotes[i]->buf);
+
+ p = repo_promisor_remote_find(r, decoded_remote);
+ if (p)
+ p->accepted = 1;
+ else
+ warning(_("accepted promisor remote '%s' not found"),
+ decoded_remote);
+
+ free(decoded_remote);
+ }
+
+ strbuf_list_free(accepted_remotes);
+}
* Promisor remote linked list
*
* Information in its fields come from remote.XXX config entries or
- * from extensions.partialclone.
+ * from extensions.partialclone, except for 'accepted' which comes
+ * from protocol v2 capabilities exchange.
*/
struct promisor_remote {
struct promisor_remote *next;
char *partial_clone_filter;
+ unsigned int accepted : 1;
const char name[FLEX_ARRAY];
};
const struct object_id *oids,
int oid_nr);
+/*
+ * Prepare a "promisor-remote" advertisement by a server.
+ * Check the value of "promisor.advertise" and maybe the configured
+ * promisor remotes, if any, to prepare information to send in an
+ * advertisement.
+ * Return value is NULL if no promisor remote advertisement should be
+ * made. Otherwise it contains the names and urls of the advertised
+ * promisor remotes separated by ';'. See gitprotocol-v2(5).
+ */
+char *promisor_remote_info(struct repository *repo);
+
+/*
+ * Prepare a reply to a "promisor-remote" advertisement from a server.
+ * Check the value of "promisor.acceptfromserver" and maybe the
+ * configured promisor remotes, if any, to prepare the reply.
+ * Return value is NULL if no promisor remote from the server
+ * is accepted. Otherwise it contains the names of the accepted promisor
+ * remotes separated by ';'. See gitprotocol-v2(5).
+ */
+char *promisor_remote_reply(const char *info);
+
+/*
+ * Set the 'accepted' flag for some promisor remotes. Useful on the
+ * server side when some promisor remotes have been accepted by the
+ * client.
+ */
+void mark_promisor_remotes_as_accepted(struct repository *repo, const char *remotes);
+
+/*
+ * Has any promisor remote been accepted by the client?
+ */
+int repo_has_accepted_promisor_remote(struct repository *r);
+
#endif /* PROMISOR_REMOTE_H */
#include "upload-pack.h"
#include "bundle-uri.h"
#include "trace2.h"
+#include "promisor-remote.h"
static int advertise_sid = -1;
static int advertise_object_info = -1;
return 1;
}
+static int promisor_remote_advertise(struct repository *r,
+ struct strbuf *value)
+{
+ if (value) {
+ char *info = promisor_remote_info(r);
+ if (!info)
+ return 0;
+ strbuf_addstr(value, info);
+ free(info);
+ }
+ return 1;
+}
+
+static void promisor_remote_receive(struct repository *r,
+ const char *remotes)
+{
+ mark_promisor_remotes_as_accepted(r, remotes);
+}
+
+
static int object_format_advertise(struct repository *r,
struct strbuf *value)
{
.advertise = bundle_uri_advertise,
.command = bundle_uri_command,
},
+ {
+ .name = "promisor-remote",
+ .advertise = promisor_remote_advertise,
+ .receive = promisor_remote_receive,
+ },
};
void protocol_v2_advertise_capabilities(struct repository *r)
't5703-upload-pack-ref-in-want.sh',
't5704-protocol-violations.sh',
't5705-session-id-in-capabilities.sh',
+ 't5710-promisor-remote-capability.sh',
't5730-protocol-v2-bundle-uri-file.sh',
't5731-protocol-v2-bundle-uri-git.sh',
't5732-protocol-v2-bundle-uri-http.sh',
--- /dev/null
+#!/bin/sh
+
+test_description='handling of promisor remote advertisement'
+
+. ./test-lib.sh
+
+GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
+
+# Setup the repository with three commits, this way HEAD is always
+# available and we can hide commit 1 or 2.
+test_expect_success 'setup: create "template" repository' '
+ git init template &&
+ test_commit -C template 1 &&
+ test_commit -C template 2 &&
+ test_commit -C template 3 &&
+ test-tool genrandom foo 10240 >template/foo &&
+ git -C template add foo &&
+ git -C template commit -m foo
+'
+
+# A bare repo will act as a server repo with unpacked objects.
+test_expect_success 'setup: create bare "server" repository' '
+ git clone --bare --no-local template server &&
+ mv server/objects/pack/pack-* . &&
+ packfile=$(ls pack-*.pack) &&
+ git -C server unpack-objects --strict <"$packfile"
+'
+
+check_missing_objects () {
+ git -C "$1" rev-list --objects --all --missing=print > all.txt &&
+ perl -ne 'print if s/^[?]//' all.txt >missing.txt &&
+ test_line_count = "$2" missing.txt &&
+ if test "$2" -lt 2
+ then
+ test "$3" = "$(cat missing.txt)"
+ else
+ test -f "$3" &&
+ sort <"$3" >expected_sorted &&
+ sort <missing.txt >actual_sorted &&
+ test_cmp expected_sorted actual_sorted
+ fi
+}
+
+initialize_server () {
+ count="$1"
+ missing_oids="$2"
+
+ # Repack everything first
+ git -C server -c repack.writebitmaps=false repack -a -d &&
+
+ # Remove promisor file in case they exist, useful when reinitializing
+ rm -rf server/objects/pack/*.promisor &&
+
+ # Repack without the largest object and create a promisor pack on server
+ git -C server -c repack.writebitmaps=false repack -a -d \
+ --filter=blob:limit=5k --filter-to="$(pwd)/pack" &&
+ promisor_file=$(ls server/objects/pack/*.pack | sed "s/\.pack/.promisor/") &&
+ >"$promisor_file" &&
+
+ # Check objects missing on the server
+ check_missing_objects server "$count" "$missing_oids"
+}
+
+copy_to_lop () {
+ oid_path="$(test_oid_to_path $1)" &&
+ path="server/objects/$oid_path" &&
+ path2="lop/objects/$oid_path" &&
+ mkdir -p $(dirname "$path2") &&
+ cp "$path" "$path2"
+}
+
+test_expect_success "setup for testing promisor remote advertisement" '
+ # Create another bare repo called "lop" (for Large Object Promisor)
+ git init --bare lop &&
+
+ # Copy the largest object from server to lop
+ obj="HEAD:foo" &&
+ oid="$(git -C server rev-parse $obj)" &&
+ copy_to_lop "$oid" &&
+
+ initialize_server 1 "$oid" &&
+
+ # Configure lop as promisor remote for server
+ git -C server remote add lop "file://$(pwd)/lop" &&
+ git -C server config remote.lop.promisor true &&
+
+ git -C lop config uploadpack.allowFilter true &&
+ git -C lop config uploadpack.allowAnySHA1InWant true &&
+ git -C server config uploadpack.allowFilter true &&
+ git -C server config uploadpack.allowAnySHA1InWant true
+'
+
+test_expect_success "clone with promisor.advertise set to 'true'" '
+ git -C server config promisor.advertise true &&
+
+ # Clone from server to create a client
+ GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \
+ -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \
+ -c remote.lop.url="file://$(pwd)/lop" \
+ -c promisor.acceptfromserver=All \
+ --no-local --filter="blob:limit=5k" server client &&
+ test_when_finished "rm -rf client" &&
+
+ # Check that the largest object is still missing on the server
+ check_missing_objects server 1 "$oid"
+'
+
+test_expect_success "clone with promisor.advertise set to 'false'" '
+ git -C server config promisor.advertise false &&
+
+ # Clone from server to create a client
+ GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \
+ -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \
+ -c remote.lop.url="file://$(pwd)/lop" \
+ -c promisor.acceptfromserver=All \
+ --no-local --filter="blob:limit=5k" server client &&
+ test_when_finished "rm -rf client" &&
+
+ # Check that the largest object is not missing on the server
+ check_missing_objects server 0 "" &&
+
+ # Reinitialize server so that the largest object is missing again
+ initialize_server 1 "$oid"
+'
+
+test_expect_success "clone with promisor.acceptfromserver set to 'None'" '
+ git -C server config promisor.advertise true &&
+
+ # Clone from server to create a client
+ GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \
+ -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \
+ -c remote.lop.url="file://$(pwd)/lop" \
+ -c promisor.acceptfromserver=None \
+ --no-local --filter="blob:limit=5k" server client &&
+ test_when_finished "rm -rf client" &&
+
+ # Check that the largest object is not missing on the server
+ check_missing_objects server 0 "" &&
+
+ # Reinitialize server so that the largest object is missing again
+ initialize_server 1 "$oid"
+'
+
+test_expect_success "init + fetch with promisor.advertise set to 'true'" '
+ git -C server config promisor.advertise true &&
+
+ test_when_finished "rm -rf client" &&
+ mkdir client &&
+ git -C client init &&
+ git -C client config remote.lop.promisor true &&
+ git -C client config remote.lop.fetch "+refs/heads/*:refs/remotes/lop/*" &&
+ git -C client config remote.lop.url "file://$(pwd)/lop" &&
+ git -C client config remote.server.url "file://$(pwd)/server" &&
+ git -C client config remote.server.fetch "+refs/heads/*:refs/remotes/server/*" &&
+ git -C client config promisor.acceptfromserver All &&
+ GIT_NO_LAZY_FETCH=0 git -C client fetch --filter="blob:limit=5k" server &&
+
+ # Check that the largest object is still missing on the server
+ check_missing_objects server 1 "$oid"
+'
+
+test_expect_success "clone with promisor.advertise set to 'true' but don't delete the client" '
+ git -C server config promisor.advertise true &&
+
+ # Clone from server to create a client
+ GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \
+ -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \
+ -c remote.lop.url="file://$(pwd)/lop" \
+ -c promisor.acceptfromserver=All \
+ --no-local --filter="blob:limit=5k" server client &&
+
+ # Check that the largest object is still missing on the server
+ check_missing_objects server 1 "$oid"
+'
+
+test_expect_success "setup for subsequent fetches" '
+ # Generate new commit with large blob
+ test-tool genrandom bar 10240 >template/bar &&
+ git -C template add bar &&
+ git -C template commit -m bar &&
+
+ # Fetch new commit with large blob
+ git -C server fetch origin &&
+ git -C server update-ref HEAD FETCH_HEAD &&
+ git -C server rev-parse HEAD >expected_head &&
+
+ # Repack everything twice and remove .promisor files before
+ # each repack. This makes sure everything gets repacked
+ # into a single packfile. The second repack is necessary
+ # because the first one fetches from lop and creates a new
+ # packfile and its associated .promisor file.
+
+ rm -f server/objects/pack/*.promisor &&
+ git -C server -c repack.writebitmaps=false repack -a -d &&
+ rm -f server/objects/pack/*.promisor &&
+ git -C server -c repack.writebitmaps=false repack -a -d &&
+
+ # Unpack everything
+ rm pack-* &&
+ mv server/objects/pack/pack-* . &&
+ packfile=$(ls pack-*.pack) &&
+ git -C server unpack-objects --strict <"$packfile" &&
+
+ # Copy new large object to lop
+ obj_bar="HEAD:bar" &&
+ oid_bar="$(git -C server rev-parse $obj_bar)" &&
+ copy_to_lop "$oid_bar" &&
+
+ # Reinitialize server so that the 2 largest objects are missing
+ printf "%s\n" "$oid" "$oid_bar" >expected_missing.txt &&
+ initialize_server 2 expected_missing.txt &&
+
+ # Create one more client
+ cp -r client client2
+'
+
+test_expect_success "subsequent fetch from a client when promisor.advertise is true" '
+ git -C server config promisor.advertise true &&
+
+ GIT_NO_LAZY_FETCH=0 git -C client pull origin &&
+
+ git -C client rev-parse HEAD >actual &&
+ test_cmp expected_head actual &&
+
+ cat client/bar >/dev/null &&
+
+ check_missing_objects server 2 expected_missing.txt
+'
+
+test_expect_success "subsequent fetch from a client when promisor.advertise is false" '
+ git -C server config promisor.advertise false &&
+
+ GIT_NO_LAZY_FETCH=0 git -C client2 pull origin &&
+
+ git -C client2 rev-parse HEAD >actual &&
+ test_cmp expected_head actual &&
+
+ cat client2/bar >/dev/null &&
+
+ check_missing_objects server 1 "$oid"
+'
+
+test_done
#include "write-or-die.h"
#include "json-writer.h"
#include "strmap.h"
+#include "promisor-remote.h"
/* Remember to update object flag allocation in object.h */
#define THEY_HAVE (1u << 11)
strvec_push(&pack_objects.args, "--delta-base-offset");
if (pack_data->use_include_tag)
strvec_push(&pack_objects.args, "--include-tag");
+ if (repo_has_accepted_promisor_remote(the_repository))
+ strvec_push(&pack_objects.args, "--missing=allow-promisor");
if (pack_data->filter_options.choice) {
const char *spec =
expand_list_objects_filter_spec(&pack_data->filter_options);