midx: enable reachability bitmaps during MIDX compaction

Enable callers to generate reachability bitmaps when performing MIDX
layer compaction by combining all existing bitmaps from the compacted
layers.

Note that because of the object/pack ordering described by the previous
commit, the pseudo-pack order for the compacted MIDX is the same as
concatenating the individual pseudo-pack orderings for each layer in the
compaction range.

As a result, the only non-test or documentation change necessary is to
treat all objects as non-preferred during compaction so as not to
disturb the object ordering.

In the future, we may want to adjust which commit(s) receive
reachability bitmaps when compacting multiple .bitmap files into one, or
even generate new bitmaps (e.g., if the references have moved
significantly since the .bitmap was generated). This commit only
implements combining all existing bitmaps in range together in order to
demonstrate and lay the groundwork for more exotic strategies.

Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-multi-pack-index.adoc b/Documentation/git-multi-pack-index.adoc
index 883a0529741863ebfd0c1d1e4054902867db7c8b..612568301412d6944d5886b080fe250370b1ef7a 100644 (file)
--- a/Documentation/git-multi-pack-index.adoc
+++ b/Documentation/git-multi-pack-index.adoc
@@ -13,7 +13,7 @@ SYNOPSIS
                         [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]
                         [--refs-snapshot=<path>]
 'git multi-pack-index' [<options>] compact [--[no-]incremental]
-                        <from> <to>
+                        [--[no-]bitmap] <from> <to>
 'git multi-pack-index' [<options>] verify
 'git multi-pack-index' [<options>] expire
 'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -94,6 +94,9 @@ compact::
        --incremental::
                Write the result to a MIDX chain instead of writing a
                stand-alone MIDX.
+
+       --[no-]bitmap::
+               Control whether or not a multi-pack bitmap is written.
 --
 
 verify::

diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c
index 043ee8c478a3d66104e6967eb6c7f0954880b49b..2f24c113c8f2610fb388a194a19af833cb51af60 100644 (file)
--- a/builtin/multi-pack-index.c
+++ b/builtin/multi-pack-index.c
@@ -19,7 +19,7 @@
 
 #define BUILTIN_MIDX_COMPACT_USAGE \
        N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
-          "  <from> <to>")
+          "  [--[no-]bitmap] <from> <to>")
 
 #define BUILTIN_MIDX_VERIFY_USAGE \
        N_("git multi-pack-index [<options>] verify")
@@ -216,6 +216,8 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
 
        struct option *options;
        static struct option builtin_multi_pack_index_compact_options[] = {
+               OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
+                       MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
                OPT_BIT(0, "incremental", &opts.flags,
                        N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
                OPT_END(),

diff --git a/midx-write.c b/midx-write.c
index afa077a09cc4699baa9208b4dd6f917a7395f120..4c52843db9b70115a25e5fa9cecb0bbb764feec6 100644 (file)
--- a/midx-write.c
+++ b/midx-write.c
@@ -669,7 +669,7 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx)
                struct pack_midx_entry *e = &ctx->entries[i];
                data[i].nr = i;
                data[i].pack = midx_pack_perm(ctx, e->pack_int_id);
-               if (!e->preferred)
+               if (!e->preferred || ctx->compact)
                        data[i].pack |= (1U << 31);
                data[i].offset = e->offset;
        }

diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh
index 797ae05c3bdfa5aa54b49e69e8450198b61c4ba8..40f3844282f04efce0a7a2630bf71057ccac95f4 100755 (executable)
--- a/t/t5335-compact-multi-pack-index.sh
+++ b/t/t5335-compact-multi-pack-index.sh
@@ -67,7 +67,7 @@ test_expect_success 'MIDX compaction with lex-ordered pack names' '
                write_packs A B C D E &&
                test_line_count = 5 $midx_chain &&
 
-               git multi-pack-index compact --incremental \
+               git multi-pack-index compact --incremental --bitmap \
                        "$(nth_line 2 "$midx_chain")" \
                        "$(nth_line 4 "$midx_chain")" &&
                test_line_count = 3 $midx_chain &&
@@ -90,7 +90,7 @@ test_expect_success 'MIDX compaction with non-lex-ordered pack names' '
                write_packs D C A B E &&
                test_line_count = 5 $midx_chain &&
 
-               git multi-pack-index compact --incremental \
+               git multi-pack-index compact --incremental --bitmap \
                        "$(nth_line 2 "$midx_chain")" \
                        "$(nth_line 4 "$midx_chain")" &&
                test_line_count = 3 $midx_chain &&
@@ -172,4 +172,122 @@ test_expect_success 'MIDX compaction with midx.version=1' '
        )
 '
 
+midx_objs_by_pack () {
+       awk '/\.pack$/ { split($3, a, "-"); print a[2], $1 }' | sort
+}
+
+tag_objs_from_pack () {
+       objs="$(git rev-list --objects --no-object-names "$2")" &&
+       printf "$1 %s\n" $objs | sort
+}
+
+test_expect_success 'MIDX compaction preserves pack object selection' '
+       git init midx-compact-preserve-selection &&
+       (
+               cd midx-compact-preserve-selection &&
+
+               git config maintenance.auto false &&
+
+               test_commit A &&
+               test_commit B &&
+
+               # Create two packs, one containing just the objects from
+               # A, and another containing all objects from the
+               # repository.
+               p1="$(echo A | git pack-objects --revs --delta-base-offset \
+                       $packdir/pack-1)" &&
+               p0="$(echo B | git pack-objects --revs --delta-base-offset \
+                       $packdir/pack-0)" &&
+
+               echo "pack-1-$p1.idx" | git multi-pack-index write \
+                       --incremental --bitmap --stdin-packs &&
+               echo "pack-0-$p0.idx" | git multi-pack-index write \
+                       --incremental --bitmap --stdin-packs &&
+
+               write_packs C &&
+
+               git multi-pack-index compact --incremental --bitmap \
+                       "$(nth_line 1 "$midx_chain")" \
+                       "$(nth_line 2 "$midx_chain")" &&
+
+
+               test-tool read-midx --show-objects $objdir \
+                       "$(nth_line 1 "$midx_chain")" >AB.info &&
+               test-tool read-midx --show-objects $objdir \
+                       "$(nth_line 2 "$midx_chain")" >C.info &&
+
+               midx_objs_by_pack <AB.info >AB.actual &&
+               midx_objs_by_pack <C.info >C.actual &&
+
+               {
+                       tag_objs_from_pack 1 A &&
+                       tag_objs_from_pack 0 A..B
+               } | sort >AB.expect &&
+               tag_objs_from_pack C B..C >C.expect &&
+
+               test_cmp AB.expect AB.actual &&
+               test_cmp C.expect C.actual
+       )
+'
+
+test_expect_success 'MIDX compaction with bitmaps' '
+       git init midx-compact-with-bitmaps &&
+       (
+               cd midx-compact-with-bitmaps &&
+
+               git config maintenance.auto false &&
+
+               write_packs foo bar baz quux woot &&
+
+               test-tool read-midx --bitmap $objdir >bitmap.expect &&
+               git multi-pack-index compact --incremental --bitmap \
+                       "$(nth_line 2 "$midx_chain")" \
+                       "$(nth_line 4 "$midx_chain")" &&
+               test-tool read-midx --bitmap $objdir >bitmap.actual &&
+
+               test_cmp bitmap.expect bitmap.actual &&
+
+               true
+       )
+'
+
+test_expect_success 'MIDX compaction with bitmaps (non-trivial)' '
+       git init midx-compact-with-bitmaps-non-trivial &&
+       (
+               cd midx-compact-with-bitmaps-non-trivial &&
+
+               git config maintenance.auto false &&
+
+               git branch -m main &&
+
+               #               D(4)
+               #              /
+               # A(1) --- B(2) --- C(3) --- G(7)
+               #              \
+               #               E(5) --- F(6)
+               write_packs A B C &&
+               git checkout -b side &&
+               write_packs D &&
+               git checkout -b other B &&
+               write_packs E F &&
+               git checkout main &&
+               write_packs G &&
+
+               # Compact layers 2-4, leaving us with:
+               #
+               #  [A, [B, C, D], E, F, G]
+               git multi-pack-index compact --incremental --bitmap \
+                       "$(nth_line 2 "$midx_chain")" \
+                       "$(nth_line 4 "$midx_chain")" &&
+
+               # Then compact the top two layers, condensing the above
+               # such that the new 4th layer contains F and G.
+               #
+               #  [A, [B, C, D], E, [F, G]]
+               git multi-pack-index compact --incremental --bitmap \
+                       "$(nth_line 4 "$midx_chain")" \
+                       "$(nth_line 5 "$midx_chain")"
+       )
+'
+
 test_done