fetch-pack: support negotiation tip whitelist

During negotiation, fetch-pack eventually reports as "have" lines all
commits reachable from all refs. Allow the user to restrict the commits
sent in this way by providing a whitelist of tips; only the tips
themselves and their ancestors will be sent.

Both globs and single objects are supported.

This feature is only supported for protocols that support connect or
stateless-connect (such as HTTP with protocol v2).

This will speed up negotiation when the repository has multiple
relatively independent branches (for example, when a repository
interacts with multiple repositories, such as with linux-next [1] and
torvalds/linux [2]), and the user knows which local branch is likely to
have commits in common with the upstream branch they are fetching.

[1] https://kernel.googlesource.com/pub/scm/linux/kernel/git/next/linux-next/
[2] https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/

Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/fetch-options.txt b/Documentation/fetch-options.txt
index 97d3217df9ac3f048073f62a0d5356c4546354ff..2d09f87b4b38dbb98e0e1e5e341e09e5f84ece17 100644 (file)
--- a/Documentation/fetch-options.txt
+++ b/Documentation/fetch-options.txt
@@ -42,6 +42,22 @@ the current repository has the same history as the source repository.
        .git/shallow. This option updates .git/shallow and accept such
        refs.
 
+--negotiation-tip=<commit|glob>::
+       By default, Git will report, to the server, commits reachable
+       from all local refs to find common commits in an attempt to
+       reduce the size of the to-be-received packfile. If specified,
+       Git will only report commits reachable from the given tips.
+       This is useful to speed up fetches when the user knows which
+       local ref is likely to have commits in common with the
+       upstream ref being fetched.
++
+This option may be specified more than once; if so, Git will report
+commits reachable from any of the given commits.
++
+The argument to this option may be a glob on ref names, a ref, or the (possibly
+abbreviated) SHA-1 of a commit. Specifying a glob is equivalent to specifying
+this option multiple times, one for each matching ref name.
+
 ifndef::git-pull[]
 --dry-run::
        Show what would be done, without making any changes.

diff --git a/builtin/fetch.c b/builtin/fetch.c
index ea5b9669ad1f40da56a8565cada56e5991d24206..49ab6ac0643b9ceff51e8eca806612f310f79c5c 100644 (file)
--- a/builtin/fetch.c
+++ b/builtin/fetch.c
@@ -63,6 +63,7 @@ static int shown_url = 0;
 static struct refspec refmap = REFSPEC_INIT_FETCH;
 static struct list_objects_filter_options filter_options;
 static struct string_list server_options = STRING_LIST_INIT_DUP;
+static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
 
 static int git_fetch_config(const char *k, const char *v, void *cb)
 {
@@ -174,6 +175,8 @@ static struct option builtin_fetch_options[] = {
                        TRANSPORT_FAMILY_IPV4),
        OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
                        TRANSPORT_FAMILY_IPV6),
+       OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"),
+                       N_("report that we have only objects reachable from this object")),
        OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
        OPT_END()
 };
@@ -1049,6 +1052,40 @@ static void set_option(struct transport *transport, const char *name, const char
                        name, transport->url);
 }
 
+
+static int add_oid(const char *refname, const struct object_id *oid, int flags,
+                  void *cb_data)
+{
+       struct oid_array *oids = cb_data;
+
+       oid_array_append(oids, oid);
+       return 0;
+}
+
+static void add_negotiation_tips(struct git_transport_options *smart_options)
+{
+       struct oid_array *oids = xcalloc(1, sizeof(*oids));
+       int i;
+
+       for (i = 0; i < negotiation_tip.nr; i++) {
+               const char *s = negotiation_tip.items[i].string;
+               int old_nr;
+               if (!has_glob_specials(s)) {
+                       struct object_id oid;
+                       if (get_oid(s, &oid))
+                               die("%s is not a valid object", s);
+                       oid_array_append(oids, &oid);
+                       continue;
+               }
+               old_nr = oids->nr;
+               for_each_glob_ref(add_oid, s, oids);
+               if (old_nr == oids->nr)
+                       warning("Ignoring --negotiation-tip=%s because it does not match any refs",
+                               s);
+       }
+       smart_options->negotiation_tips = oids;
+}
+
 static struct transport *prepare_transport(struct remote *remote, int deepen)
 {
        struct transport *transport;
@@ -1075,6 +1112,12 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
                           filter_options.filter_spec);
                set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
        }
+       if (negotiation_tip.nr) {
+               if (transport->smart_options)
+                       add_negotiation_tips(transport->smart_options);
+               else
+                       warning("Ignoring --negotiation-tip because the protocol does not support it.");
+       }
        return transport;
 }
 

diff --git a/fetch-pack.c b/fetch-pack.c
index ba12085c4ade41ca6d8eee7374c0daa2d5fa1f7d..1e50d900826e3e31e5bd5a60f922ab5f07889e61 100644 (file)
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -213,6 +213,22 @@ static int next_flush(int stateless_rpc, int count)
        return count;
 }
 
+static void mark_tips(struct fetch_negotiator *negotiator,
+                     const struct oid_array *negotiation_tips)
+{
+       int i;
+
+       if (!negotiation_tips) {
+               for_each_ref(rev_list_insert_ref_oid, negotiator);
+               return;
+       }
+
+       for (i = 0; i < negotiation_tips->nr; i++)
+               rev_list_insert_ref(negotiator, NULL,
+                                   &negotiation_tips->oid[i]);
+       return;
+}
+
 static int find_common(struct fetch_negotiator *negotiator,
                       struct fetch_pack_args *args,
                       int fd[2], struct object_id *result_oid,
@@ -230,7 +246,7 @@ static int find_common(struct fetch_negotiator *negotiator,
        if (args->stateless_rpc && multi_ack == 1)
                die(_("--stateless-rpc requires multi_ack_detailed"));
 
-       for_each_ref(rev_list_insert_ref_oid, negotiator);
+       mark_tips(negotiator, args->negotiation_tips);
        for_each_cached_alternate(negotiator, insert_one_alternate_object);
 
        fetching = 0;
@@ -1295,7 +1311,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                        else
                                state = FETCH_SEND_REQUEST;
 
-                       for_each_ref(rev_list_insert_ref_oid, &negotiator);
+                       mark_tips(&negotiator, args->negotiation_tips);
                        for_each_cached_alternate(&negotiator,
                                                  insert_one_alternate_object);
                        break;

diff --git a/fetch-pack.h b/fetch-pack.h
index bb45a366a82a4a7dae2524e0845ac33128076c57..1859ee927552a98b7d4474e49b43289437c22e5b 100644 (file)
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -16,6 +16,13 @@ struct fetch_pack_args {
        const struct string_list *deepen_not;
        struct list_objects_filter_options filter_options;
        const struct string_list *server_options;
+
+       /*
+        * If not NULL, during packfile negotiation, fetch-pack will send "have"
+        * lines only with these tips and their ancestors.
+        */
+       const struct oid_array *negotiation_tips;
+
        unsigned deepen_relative:1;
        unsigned quiet:1;
        unsigned keep_pack:1;

diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh
index e402aee6a22e64306c9f093ad1b529b3923fd65e..4e6d049b94c686c7d1830c8e3ad0fcea0a00a605 100755 (executable)
--- a/t/t5510-fetch.sh
+++ b/t/t5510-fetch.sh
@@ -865,4 +865,82 @@ test_expect_success C_LOCALE_OUTPUT 'fetch compact output' '
        test_cmp expect actual
 '
 
+setup_negotiation_tip () {
+       SERVER="$1"
+       URL="$2"
+       USE_PROTOCOL_V2="$3"
+
+       rm -rf "$SERVER" client trace &&
+       git init "$SERVER" &&
+       test_commit -C "$SERVER" alpha_1 &&
+       test_commit -C "$SERVER" alpha_2 &&
+       git -C "$SERVER" checkout --orphan beta &&
+       test_commit -C "$SERVER" beta_1 &&
+       test_commit -C "$SERVER" beta_2 &&
+
+       git clone "$URL" client &&
+
+       if test "$USE_PROTOCOL_V2" -eq 1
+       then
+               git -C "$SERVER" config protocol.version 2 &&
+               git -C client config protocol.version 2
+       fi &&
+
+       test_commit -C "$SERVER" beta_s &&
+       git -C "$SERVER" checkout master &&
+       test_commit -C "$SERVER" alpha_s &&
+       git -C "$SERVER" tag -d alpha_1 alpha_2 beta_1 beta_2
+}
+
+check_negotiation_tip () {
+       # Ensure that {alpha,beta}_1 are sent as "have", but not {alpha_beta}_2
+       ALPHA_1=$(git -C client rev-parse alpha_1) &&
+       grep "fetch> have $ALPHA_1" trace &&
+       BETA_1=$(git -C client rev-parse beta_1) &&
+       grep "fetch> have $BETA_1" trace &&
+       ALPHA_2=$(git -C client rev-parse alpha_2) &&
+       ! grep "fetch> have $ALPHA_2" trace &&
+       BETA_2=$(git -C client rev-parse beta_2) &&
+       ! grep "fetch> have $BETA_2" trace
+}
+
+test_expect_success '--negotiation-tip limits "have" lines sent' '
+       setup_negotiation_tip server server 0 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
+               --negotiation-tip=alpha_1 --negotiation-tip=beta_1 \
+               origin alpha_s beta_s &&
+       check_negotiation_tip
+'
+
+test_expect_success '--negotiation-tip understands globs' '
+       setup_negotiation_tip server server 0 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
+               --negotiation-tip=*_1 \
+               origin alpha_s beta_s &&
+       check_negotiation_tip
+'
+
+test_expect_success '--negotiation-tip understands abbreviated SHA-1' '
+       setup_negotiation_tip server server 0 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
+               --negotiation-tip=$(git -C client rev-parse --short alpha_1) \
+               --negotiation-tip=$(git -C client rev-parse --short beta_1) \
+               origin alpha_s beta_s &&
+       check_negotiation_tip
+'
+
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+test_expect_success '--negotiation-tip limits "have" lines sent with HTTP protocol v2' '
+       setup_negotiation_tip "$HTTPD_DOCUMENT_ROOT_PATH/server" \
+               "$HTTPD_URL/smart/server" 1 &&
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \
+               --negotiation-tip=alpha_1 --negotiation-tip=beta_1 \
+               origin alpha_s beta_s &&
+       check_negotiation_tip
+'
+
+stop_httpd
+
 test_done

diff --git a/transport-helper.c b/transport-helper.c
index 1f8ff7e9424e5d780578a6ab00623f814d8f836a..ad8f7c7726542dea6ab933ee22eca931d139fd7a 100644 (file)
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -684,6 +684,9 @@ static int fetch(struct transport *transport,
                        transport, "filter",
                        data->transport_options.filter_options.filter_spec);
 
+       if (data->transport_options.negotiation_tips)
+               warning("Ignoring --negotiation-tip because the protocol does not support it.");
+
        if (data->fetch)
                return fetch_with_fetch(transport, nr_heads, to_fetch);
 

diff --git a/transport.c b/transport.c
index a32da30dee6f4e38433b68d7c9f7e9404aa58858..9f10f8ad9fca8ea2d387141864860adf7b9276c0 100644 (file)
--- a/transport.c
+++ b/transport.c
@@ -318,6 +318,7 @@ static int fetch_refs_via_pack(struct transport *transport,
        args.filter_options = data->options.filter_options;
        args.stateless_rpc = transport->stateless_rpc;
        args.server_options = transport->server_options;
+       args.negotiation_tips = data->options.negotiation_tips;
 
        if (!data->got_remote_heads)
                refs_tmp = get_refs_via_connect(transport, 0, NULL);

diff --git a/transport.h b/transport.h
index 7792b08582c132e18b495a21b700d9113727c7eb..d31be5be63a888364c55841020235cbab0ea61b3 100644 (file)
--- a/transport.h
+++ b/transport.h
@@ -25,6 +25,16 @@ struct git_transport_options {
        const char *receivepack;
        struct push_cas_option *cas;
        struct list_objects_filter_options filter_options;
+
+       /*
+        * This is only used during fetch. See the documentation of
+        * negotiation_tips in struct fetch_pack_args.
+        *
+        * This field is only supported by transports that support connect or
+        * stateless_connect. Set this field directly instead of using
+        * transport_set_option().
+        */
+       struct oid_array *negotiation_tips;
 };
 
 enum transport_family {