fetch-pack: support more than one pack lockfile

Whenever a fetch results in a packfile being downloaded, a .keep file is
generated, so that the packfile can be preserved (from, say, a running
"git repack") until refs are written referring to the contents of the
packfile.

In a subsequent patch, a successful fetch using protocol v2 may result
in more than one .keep file being generated. Therefore, teach
fetch_pack() and the transport mechanism to support multiple .keep
files.

Implementation notes:

 - builtin/fetch-pack.c normally does not generate .keep files, and thus
   is unaffected by this or future changes. However, it has an
   undocumented "--lock-pack" feature, used by remote-curl.c when
   implementing the "fetch" remote helper command. In keeping with the
   remote helper protocol, only one "lock" line will ever be written;
   the rest will result in warnings to stderr. However, in practice,
   warnings will never be written because the remote-curl.c "fetch" is
   only used for protocol v0/v1 (which will not generate multiple .keep
   files). (Protocol v2 uses the "stateless-connect" command, not the
   "fetch" command.)

 - connected.c has an optimization in that connectivity checks on a ref
   need not be done if the target object is in a pack known to be
   self-contained and connected. If there are multiple packfiles, this
   optimization can no longer be done.

Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index 47711000725b4fb7dc504c62cc2d4ac384f7d46b..f66891b010b0f090b552aa0d0170c83f44b2a219 100644 (file)
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -48,8 +48,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
        struct ref **sought = NULL;
        int nr_sought = 0, alloc_sought = 0;
        int fd[2];
-       char *pack_lockfile = NULL;
-       char **pack_lockfile_ptr = NULL;
+       struct string_list pack_lockfiles = STRING_LIST_INIT_DUP;
+       struct string_list *pack_lockfiles_ptr = NULL;
        struct child_process *conn;
        struct fetch_pack_args args;
        struct oid_array shallow = OID_ARRAY_INIT;
@@ -134,7 +134,7 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
                }
                if (!strcmp("--lock-pack", arg)) {
                        args.lock_pack = 1;
-                       pack_lockfile_ptr = &pack_lockfile;
+                       pack_lockfiles_ptr = &pack_lockfiles;
                        continue;
                }
                if (!strcmp("--check-self-contained-and-connected", arg)) {
@@ -235,10 +235,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
        }
 
        ref = fetch_pack(&args, fd, ref, sought, nr_sought,
-                        &shallow, pack_lockfile_ptr, version);
-       if (pack_lockfile) {
-               printf("lock %s\n", pack_lockfile);
+                        &shallow, pack_lockfiles_ptr, version);
+       if (pack_lockfiles.nr) {
+               int i;
+
+               printf("lock %s\n", pack_lockfiles.items[0].string);
                fflush(stdout);
+               for (i = 1; i < pack_lockfiles.nr; i++)
+                       warning(_("Lockfile created but not reported: %s"),
+                               pack_lockfiles.items[i].string);
        }
        if (args.check_self_contained_and_connected &&
            args.self_contained_and_connected) {

diff --git a/connected.c b/connected.c
index 3135b71e1961ae421855ad4b05887c795c437490..937b4bae387a5757123ebb374fe98102be80921a 100644 (file)
--- a/connected.c
+++ b/connected.c
@@ -43,10 +43,12 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
 
        if (transport && transport->smart_options &&
            transport->smart_options->self_contained_and_connected &&
-           transport->pack_lockfile &&
-           strip_suffix(transport->pack_lockfile, ".keep", &base_len)) {
+           transport->pack_lockfiles.nr == 1 &&
+           strip_suffix(transport->pack_lockfiles.items[0].string,
+                        ".keep", &base_len)) {
                struct strbuf idx_file = STRBUF_INIT;
-               strbuf_add(&idx_file, transport->pack_lockfile, base_len);
+               strbuf_add(&idx_file, transport->pack_lockfiles.items[0].string,
+                          base_len);
                strbuf_addstr(&idx_file, ".idx");
                new_pack = add_packed_git(idx_file.buf, idx_file.len, 1);
                strbuf_release(&idx_file);

diff --git a/fetch-pack.c b/fetch-pack.c
index 7eaa19d7c17abeb4d2975d112a79762507a81009..bc3af3c726af158d5baa47c065e2090af00aa668 100644 (file)
--- a/fetch-pack.c
+++ b/fetch-pack.c
@@ -794,7 +794,7 @@ static void write_promisor_file(const char *keep_name,
 }
 
 static int get_pack(struct fetch_pack_args *args,
-                   int xd[2], char **pack_lockfile,
+                   int xd[2], struct string_list *pack_lockfiles,
                    struct ref **sought, int nr_sought)
 {
        struct async demux;
@@ -838,7 +838,7 @@ static int get_pack(struct fetch_pack_args *args,
        }
 
        if (do_keep || args->from_promisor) {
-               if (pack_lockfile)
+               if (pack_lockfiles)
                        cmd.out = -1;
                cmd_name = "index-pack";
                argv_array_push(&cmd.args, cmd_name);
@@ -863,7 +863,7 @@ static int get_pack(struct fetch_pack_args *args,
                 * information below. If not, we need index-pack to do it for
                 * us.
                 */
-               if (!(do_keep && pack_lockfile) && args->from_promisor)
+               if (!(do_keep && pack_lockfiles) && args->from_promisor)
                        argv_array_push(&cmd.args, "--promisor");
        }
        else {
@@ -899,8 +899,9 @@ static int get_pack(struct fetch_pack_args *args,
        cmd.git_cmd = 1;
        if (start_command(&cmd))
                die(_("fetch-pack: unable to fork off %s"), cmd_name);
-       if (do_keep && pack_lockfile) {
-               *pack_lockfile = index_pack_lockfile(cmd.out);
+       if (do_keep && pack_lockfiles) {
+               string_list_append_nodup(pack_lockfiles,
+                                        index_pack_lockfile(cmd.out));
                close(cmd.out);
        }
 
@@ -922,8 +923,8 @@ static int get_pack(struct fetch_pack_args *args,
         * Now that index-pack has succeeded, write the promisor file using the
         * obtained .keep filename if necessary
         */
-       if (do_keep && pack_lockfile && args->from_promisor)
-               write_promisor_file(*pack_lockfile, sought, nr_sought);
+       if (do_keep && pack_lockfiles && pack_lockfiles->nr && args->from_promisor)
+               write_promisor_file(pack_lockfiles->items[0].string, sought, nr_sought);
 
        return 0;
 }
@@ -940,7 +941,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                                 const struct ref *orig_ref,
                                 struct ref **sought, int nr_sought,
                                 struct shallow_info *si,
-                                char **pack_lockfile)
+                                struct string_list *pack_lockfiles)
 {
        struct repository *r = the_repository;
        struct ref *ref = copy_ref_list(orig_ref);
@@ -1067,7 +1068,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
                alternate_shallow_file = setup_temporary_shallow(si->shallow);
        else
                alternate_shallow_file = NULL;
-       if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
+       if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
                die(_("git fetch-pack: fetch failed."));
 
  all_done:
@@ -1457,7 +1458,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
                                    struct ref **sought, int nr_sought,
                                    struct oid_array *shallows,
                                    struct shallow_info *si,
-                                   char **pack_lockfile)
+                                   struct string_list *pack_lockfiles)
 {
        struct repository *r = the_repository;
        struct ref *ref = copy_ref_list(orig_ref);
@@ -1559,7 +1560,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
 
                        /* get the pack */
                        process_section_header(&reader, "packfile", 0);
-                       if (get_pack(args, fd, pack_lockfile, sought, nr_sought))
+                       if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
                                die(_("git fetch-pack: fetch failed."));
 
                        state = FETCH_DONE;
@@ -1759,7 +1760,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                       const struct ref *ref,
                       struct ref **sought, int nr_sought,
                       struct oid_array *shallow,
-                      char **pack_lockfile,
+                      struct string_list *pack_lockfiles,
                       enum protocol_version version)
 {
        struct ref *ref_cpy;
@@ -1794,11 +1795,11 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                memset(&si, 0, sizeof(si));
                ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
                                           &shallows_scratch, &si,
-                                          pack_lockfile);
+                                          pack_lockfiles);
        } else {
                prepare_shallow_info(&si, shallow);
                ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
-                                       &si, pack_lockfile);
+                                       &si, pack_lockfiles);
        }
        reprepare_packed_git(the_repository);
 

diff --git a/fetch-pack.h b/fetch-pack.h
index 67f684229a9ae8c852cc27138ff376d671d80f58..85d1e39fe752ddc625ba7801faa5373db57b796c 100644 (file)
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -83,7 +83,7 @@ struct ref *fetch_pack(struct fetch_pack_args *args,
                       struct ref **sought,
                       int nr_sought,
                       struct oid_array *shallow,
-                      char **pack_lockfile,
+                      struct string_list *pack_lockfiles,
                       enum protocol_version version);
 
 /*

diff --git a/transport-helper.c b/transport-helper.c
index a46afcb69db615d2f8917f852db28c3dca501de0..93a6f507936348e3b2eb43f06663725540646441 100644 (file)
--- a/transport-helper.c
+++ b/transport-helper.c
@@ -410,10 +410,11 @@ static int fetch_with_fetch(struct transport *transport,
                        exit(128);
 
                if (skip_prefix(buf.buf, "lock ", &name)) {
-                       if (transport->pack_lockfile)
+                       if (transport->pack_lockfiles.nr)
                                warning(_("%s also locked %s"), data->name, name);
                        else
-                               transport->pack_lockfile = xstrdup(name);
+                               string_list_append(&transport->pack_lockfiles,
+                                                  name);
                }
                else if (data->check_connectivity &&
                         data->transport_options.check_self_contained_and_connected &&

diff --git a/transport.c b/transport.c
index 15f5ba4e8f22c69959357bc8a7fe073678fe8862..a67e1990bf24eb05d166ba1017c4428d5e6cd5fa 100644 (file)
--- a/transport.c
+++ b/transport.c
@@ -374,7 +374,7 @@ static int fetch_refs_via_pack(struct transport *transport,
                refs = fetch_pack(&args, data->fd,
                                  refs_tmp ? refs_tmp : transport->remote_refs,
                                  to_fetch, nr_heads, &data->shallow,
-                                 &transport->pack_lockfile, data->version);
+                                 &transport->pack_lockfiles, data->version);
                break;
        case protocol_v1:
        case protocol_v0:
@@ -382,7 +382,7 @@ static int fetch_refs_via_pack(struct transport *transport,
                refs = fetch_pack(&args, data->fd,
                                  refs_tmp ? refs_tmp : transport->remote_refs,
                                  to_fetch, nr_heads, &data->shallow,
-                                 &transport->pack_lockfile, data->version);
+                                 &transport->pack_lockfiles, data->version);
                break;
        case protocol_unknown_version:
                BUG("unknown protocol version");
@@ -929,6 +929,7 @@ struct transport *transport_get(struct remote *remote, const char *url)
        struct transport *ret = xcalloc(1, sizeof(*ret));
 
        ret->progress = isatty(2);
+       string_list_init(&ret->pack_lockfiles, 1);
 
        if (!remote)
                BUG("No remote provided to transport_get()");
@@ -1324,10 +1325,11 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs)
 
 void transport_unlock_pack(struct transport *transport)
 {
-       if (transport->pack_lockfile) {
-               unlink_or_warn(transport->pack_lockfile);
-               FREE_AND_NULL(transport->pack_lockfile);
-       }
+       int i;
+
+       for (i = 0; i < transport->pack_lockfiles.nr; i++)
+               unlink_or_warn(transport->pack_lockfiles.items[i].string);
+       string_list_clear(&transport->pack_lockfiles, 0);
 }
 
 int transport_connect(struct transport *transport, const char *name,

diff --git a/transport.h b/transport.h
index 4298c855be66bb41c41053a1ca9fced7b077a389..05efa72db194e7526f8a5887b8ffeb19c86c91d5 100644 (file)
--- a/transport.h
+++ b/transport.h
@@ -5,8 +5,7 @@
 #include "run-command.h"
 #include "remote.h"
 #include "list-objects-filter-options.h"
-
-struct string_list;
+#include "string-list.h"
 
 struct git_transport_options {
        unsigned thin : 1;
@@ -98,7 +97,8 @@ struct transport {
         */
        const struct string_list *server_options;
 
-       char *pack_lockfile;
+       struct string_list pack_lockfiles;
+
        signed verbose : 3;
        /**
         * Transports should not set this directly, and should use this