midx: allow marking a pack as preferred

When multiple packs in the multi-pack index contain the same object, the
MIDX machinery must make a choice about which pack it associates with
that object. Prior to this patch, the lowest-ordered[1] pack was always
selected.

Pack selection for duplicate objects is relatively unimportant today,
but it will become important for multi-pack bitmaps. This is because we
can only invoke the pack-reuse mechanism when all of the bits for reused
objects come from the reuse pack (in order to ensure that all reused
deltas can find their base objects in the same pack).

To encourage the pack selection process to prefer one pack over another
(the pack to be preferred is the one a caller would like to later use as
a reuse pack), introduce the concept of a "preferred pack". When
provided, the MIDX code will always prefer an object found in a
preferred pack over any other.

No format changes are required to store the preferred pack, since it
will be able to be inferred with a corresponding MIDX bitmap, by looking
up the pack associated with the object in the first bit position (this
ordering is described in detail in a subsequent commit).

[1]: the ordering is specified by MIDX internals; for our purposes we
can consider the "lowest ordered" pack to be "the one with the
most-recent mtime.

Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-multi-pack-index.txt b/Documentation/git-multi-pack-index.txt
index eb0caa04393a7136e125c95038b6b740bc215086..ffd601bc17b4dc8f453e6a5b28fb4d6239176d96 100644 (file)
--- a/Documentation/git-multi-pack-index.txt
+++ b/Documentation/git-multi-pack-index.txt
@@ -9,7 +9,8 @@ git-multi-pack-index - Write and verify multi-pack-indexes
 SYNOPSIS
 --------
 [verse]
-'git multi-pack-index' [--object-dir=<dir>] [--[no-]progress] <subcommand>
+'git multi-pack-index' [--object-dir=<dir>] [--[no-]progress]
+       [--preferred-pack=<pack>] <subcommand>
 
 DESCRIPTION
 -----------
@@ -30,7 +31,16 @@ OPTIONS
 The following subcommands are available:
 
 write::
-       Write a new MIDX file.
+       Write a new MIDX file. The following options are available for
+       the `write` sub-command:
++
+--
+       --preferred-pack=<pack>::
+               Optionally specify the tie-breaking pack used when
+               multiple packs contain the same object. If not given,
+               ties are broken in favor of the pack with the lowest
+               mtime.
+--
 
 verify::
        Verify the contents of the MIDX file.

diff --git a/Documentation/technical/multi-pack-index.txt b/Documentation/technical/multi-pack-index.txt
index e8e377a59f0c19a56fb53ed6d96f10f6c211d087..fb688976c4c0337ef6850b9786daf2173f29e66c 100644 (file)
--- a/Documentation/technical/multi-pack-index.txt
+++ b/Documentation/technical/multi-pack-index.txt
@@ -43,8 +43,9 @@ Design Details
   a change in format.
 
 - The MIDX keeps only one record per object ID. If an object appears
-  in multiple packfiles, then the MIDX selects the copy in the most-
-  recently modified packfile.
+  in multiple packfiles, then the MIDX selects the copy in the
+  preferred packfile, otherwise selecting from the most-recently
+  modified packfile.
 
 - If there exist packfiles in the pack directory not registered in
   the MIDX, then those packfiles are loaded into the `packed_git`

diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c
index 8711174faead9836b52ad25360686047b5f46819..5d3ea445fdb3a83baf4e464ff470d5f632b81db2 100644 (file)
--- a/builtin/multi-pack-index.c
+++ b/builtin/multi-pack-index.c
@@ -4,9 +4,10 @@
 #include "parse-options.h"
 #include "midx.h"
 #include "trace2.h"
+#include "object-store.h"
 
 #define BUILTIN_MIDX_WRITE_USAGE \
-       N_("git multi-pack-index [<options>] write")
+       N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]")
 
 #define BUILTIN_MIDX_VERIFY_USAGE \
        N_("git multi-pack-index [<options>] verify")
@@ -43,6 +44,7 @@ static char const * const builtin_multi_pack_index_usage[] = {
 
 static struct opts_multi_pack_index {
        const char *object_dir;
+       const char *preferred_pack;
        unsigned long batch_size;
        unsigned flags;
 } opts;
@@ -61,7 +63,15 @@ static struct option *add_common_options(struct option *prev)
 
 static int cmd_multi_pack_index_write(int argc, const char **argv)
 {
-       struct option *options = common_opts;
+       struct option *options;
+       static struct option builtin_multi_pack_index_write_options[] = {
+               OPT_STRING(0, "preferred-pack", &opts.preferred_pack,
+                          N_("preferred-pack"),
+                          N_("pack for reuse when computing a multi-pack bitmap")),
+               OPT_END(),
+       };
+
+       options = add_common_options(builtin_multi_pack_index_write_options);
 
        trace2_cmd_mode(argv[0]);
 
@@ -72,7 +82,10 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)
                usage_with_options(builtin_multi_pack_index_write_usage,
                                   options);
 
-       return write_midx_file(opts.object_dir, opts.flags);
+       FREE_AND_NULL(options);
+
+       return write_midx_file(opts.object_dir, opts.preferred_pack,
+                              opts.flags);
 }
 
 static int cmd_multi_pack_index_verify(int argc, const char **argv)

diff --git a/builtin/repack.c b/builtin/repack.c
index 01440de2d5a7030a10b391f1d7423b4cbaf2ed81..9f008068059e9fd5ad4e54bdb823bd76407a21ad 100644 (file)
--- a/builtin/repack.c
+++ b/builtin/repack.c
@@ -523,7 +523,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
        remove_temporary_files();
 
        if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
-               write_midx_file(get_object_directory(), 0);
+               write_midx_file(get_object_directory(), NULL, 0);
 
        string_list_clear(&names, 0);
        string_list_clear(&rollback, 0);

diff --git a/midx.c b/midx.c
index 971faa8cfc2f6e956933b26b29bbe76d4168be1f..b8f42bf31e48cfd84b411241446cd6ce40c2fc8a 100644 (file)
--- a/midx.c
+++ b/midx.c
@@ -431,6 +431,14 @@ static int pack_info_compare(const void *_a, const void *_b)
        return strcmp(a->pack_name, b->pack_name);
 }
 
+static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
+{
+       const char *pack_name = _va;
+       const struct pack_info *compar = _vb;
+
+       return cmp_idx_or_pack_name(pack_name, compar->pack_name);
+}
+
 struct write_midx_context {
        struct pack_info *info;
        uint32_t nr;
@@ -445,6 +453,8 @@ struct write_midx_context {
        uint32_t *pack_perm;
        unsigned large_offsets_needed:1;
        uint32_t num_large_offsets;
+
+       int preferred_pack_idx;
 };
 
 static void add_pack_to_midx(const char *full_path, size_t full_path_len,
@@ -489,6 +499,7 @@ struct pack_midx_entry {
        uint32_t pack_int_id;
        time_t pack_mtime;
        uint64_t offset;
+       unsigned preferred : 1;
 };
 
 static int midx_oid_compare(const void *_a, const void *_b)
@@ -500,6 +511,12 @@ static int midx_oid_compare(const void *_a, const void *_b)
        if (cmp)
                return cmp;
 
+       /* Sort objects in a preferred pack first when multiple copies exist. */
+       if (a->preferred > b->preferred)
+               return -1;
+       if (a->preferred < b->preferred)
+               return 1;
+
        if (a->pack_mtime > b->pack_mtime)
                return -1;
        else if (a->pack_mtime < b->pack_mtime)
@@ -527,7 +544,8 @@ static int nth_midxed_pack_midx_entry(struct multi_pack_index *m,
 static void fill_pack_entry(uint32_t pack_int_id,
                            struct packed_git *p,
                            uint32_t cur_object,
-                           struct pack_midx_entry *entry)
+                           struct pack_midx_entry *entry,
+                           int preferred)
 {
        if (nth_packed_object_id(&entry->oid, p, cur_object) < 0)
                die(_("failed to locate object %d in packfile"), cur_object);
@@ -536,6 +554,7 @@ static void fill_pack_entry(uint32_t pack_int_id,
        entry->pack_mtime = p->mtime;
 
        entry->offset = nth_packed_object_offset(p, cur_object);
+       entry->preferred = !!preferred;
 }
 
 /*
@@ -552,7 +571,8 @@ static void fill_pack_entry(uint32_t pack_int_id,
 static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
                                                  struct pack_info *info,
                                                  uint32_t nr_packs,
-                                                 uint32_t *nr_objects)
+                                                 uint32_t *nr_objects,
+                                                 int preferred_pack)
 {
        uint32_t cur_fanout, cur_pack, cur_object;
        uint32_t alloc_fanout, alloc_objects, total_objects = 0;
@@ -589,12 +609,17 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
                                nth_midxed_pack_midx_entry(m,
                                                           &entries_by_fanout[nr_fanout],
                                                           cur_object);
+                               if (nth_midxed_pack_int_id(m, cur_object) == preferred_pack)
+                                       entries_by_fanout[nr_fanout].preferred = 1;
+                               else
+                                       entries_by_fanout[nr_fanout].preferred = 0;
                                nr_fanout++;
                        }
                }
 
                for (cur_pack = start_pack; cur_pack < nr_packs; cur_pack++) {
                        uint32_t start = 0, end;
+                       int preferred = cur_pack == preferred_pack;
 
                        if (cur_fanout)
                                start = get_pack_fanout(info[cur_pack].p, cur_fanout - 1);
@@ -602,7 +627,11 @@ static struct pack_midx_entry *get_sorted_entries(struct multi_pack_index *m,
 
                        for (cur_object = start; cur_object < end; cur_object++) {
                                ALLOC_GROW(entries_by_fanout, nr_fanout + 1, alloc_fanout);
-                               fill_pack_entry(cur_pack, info[cur_pack].p, cur_object, &entries_by_fanout[nr_fanout]);
+                               fill_pack_entry(cur_pack,
+                                               info[cur_pack].p,
+                                               cur_object,
+                                               &entries_by_fanout[nr_fanout],
+                                               preferred);
                                nr_fanout++;
                        }
                }
@@ -777,7 +806,9 @@ static int write_midx_large_offsets(struct hashfile *f,
 }
 
 static int write_midx_internal(const char *object_dir, struct multi_pack_index *m,
-                              struct string_list *packs_to_drop, unsigned flags)
+                              struct string_list *packs_to_drop,
+                              const char *preferred_pack_name,
+                              unsigned flags)
 {
        char *midx_name;
        uint32_t i;
@@ -828,7 +859,19 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
        if (ctx.m && ctx.nr == ctx.m->num_packs && !packs_to_drop)
                goto cleanup;
 
-       ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr);
+       ctx.preferred_pack_idx = -1;
+       if (preferred_pack_name) {
+               for (i = 0; i < ctx.nr; i++) {
+                       if (!cmp_idx_or_pack_name(preferred_pack_name,
+                                                 ctx.info[i].pack_name)) {
+                               ctx.preferred_pack_idx = i;
+                               break;
+                       }
+               }
+       }
+
+       ctx.entries = get_sorted_entries(ctx.m, ctx.info, ctx.nr, &ctx.entries_nr,
+                                        ctx.preferred_pack_idx);
 
        ctx.large_offsets_needed = 0;
        for (i = 0; i < ctx.entries_nr; i++) {
@@ -889,6 +932,24 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
                        pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
        }
 
+       /* Check that the preferred pack wasn't expired (if given). */
+       if (preferred_pack_name) {
+               struct pack_info *preferred = bsearch(preferred_pack_name,
+                                                     ctx.info, ctx.nr,
+                                                     sizeof(*ctx.info),
+                                                     idx_or_pack_name_cmp);
+
+               if (!preferred)
+                       warning(_("unknown preferred pack: '%s'"),
+                               preferred_pack_name);
+               else {
+                       uint32_t perm = ctx.pack_perm[preferred->orig_pack_int_id];
+                       if (perm == PACK_EXPIRED)
+                               warning(_("preferred pack '%s' is expired"),
+                                       preferred_pack_name);
+               }
+       }
+
        if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
                pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
                                        (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
@@ -947,9 +1008,12 @@ cleanup:
        return result;
 }
 
-int write_midx_file(const char *object_dir, unsigned flags)
+int write_midx_file(const char *object_dir,
+                   const char *preferred_pack_name,
+                   unsigned flags)
 {
-       return write_midx_internal(object_dir, NULL, NULL, flags);
+       return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,
+                                  flags);
 }
 
 void clear_midx_file(struct repository *r)
@@ -1184,7 +1248,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla
        free(count);
 
        if (packs_to_drop.nr)
-               result = write_midx_internal(object_dir, m, &packs_to_drop, flags);
+               result = write_midx_internal(object_dir, m, &packs_to_drop, NULL, flags);
 
        string_list_clear(&packs_to_drop, 0);
        return result;
@@ -1373,7 +1437,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size,
                goto cleanup;
        }
 
-       result = write_midx_internal(object_dir, m, NULL, flags);
+       result = write_midx_internal(object_dir, m, NULL, NULL, flags);
        m = NULL;
 
 cleanup:

diff --git a/midx.h b/midx.h
index b18cf53bc4abf9650019aa7b4827696ea32056ac..e7fea611091190667cc1c62a2c87d7854b3f28a6 100644 (file)
--- a/midx.h
+++ b/midx.h
@@ -47,7 +47,7 @@ int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pa
 int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name);
 int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local);
 
-int write_midx_file(const char *object_dir, unsigned flags);
+int write_midx_file(const char *object_dir, const char *preferred_pack_name, unsigned flags);
 void clear_midx_file(struct repository *r);
 int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags);
 int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags);

diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh
index b4afab1dfc2b37ce06fec8cdeeabdc73e5a9fa30..5641d158dfc4c6163474fdd6dbc8c7090ef73603 100755 (executable)
--- a/t/t5319-multi-pack-index.sh
+++ b/t/t5319-multi-pack-index.sh
@@ -234,6 +234,48 @@ test_expect_success 'warn on improper hash version' '
        )
 '
 
+test_expect_success 'midx picks objects from preferred pack' '
+       test_when_finished rm -rf preferred.git &&
+       git init --bare preferred.git &&
+       (
+               cd preferred.git &&
+
+               a=$(echo "a" | git hash-object -w --stdin) &&
+               b=$(echo "b" | git hash-object -w --stdin) &&
+               c=$(echo "c" | git hash-object -w --stdin) &&
+
+               # Set up two packs, duplicating the object "B" at different
+               # offsets.
+               #
+               # Note that the "BC" pack (the one we choose as preferred) sorts
+               # lexically after the "AB" pack, meaning that omitting the
+               # --preferred-pack argument would cause this test to fail (since
+               # the MIDX code would select the copy of "b" in the "AB" pack).
+               git pack-objects objects/pack/test-AB <<-EOF &&
+               $a
+               $b
+               EOF
+               bc=$(git pack-objects objects/pack/test-BC <<-EOF
+               $b
+               $c
+               EOF
+               ) &&
+
+               git multi-pack-index --object-dir=objects \
+                       write --preferred-pack=test-BC-$bc.idx 2>err &&
+               test_must_be_empty err &&
+
+               test-tool read-midx --show-objects objects >out &&
+
+               ofs=$(git show-index <objects/pack/test-BC-$bc.idx | grep $b |
+                       cut -d" " -f1) &&
+               printf "%s %s\tobjects/pack/test-BC-%s.pack\n" \
+                       "$b" "$ofs" "$bc" >expect &&
+               grep ^$b out >actual &&
+
+               test_cmp expect actual
+       )
+'
 
 test_expect_success 'verify multi-pack-index success' '
        git multi-pack-index verify --object-dir=$objdir