midx: introduce `--no-write-chain-file` for incremental MIDX writes

When writing an incremental MIDX layer, the MIDX machinery writes the
new layer into the multi-pack-index.d directory and then updates the
multi-pack-index-chain file to include the freshly written layer.

Future callers however may not wish to immediately update the MIDX chain
itself, preferring instead to write out new layer(s) themselves before
atomically updating the chain. Concretely, the new incremental
MIDX-based repacking strategy will want to do exactly this (that is,
assemble the new MIDX chain itself before writing a new chain file and
atomically linking it into place).

Introduce a `--no-write-chain-file` flag that:

 * writes the new MIDX layer into the multi-pack-index.d directory

 * prints its checksum

 * does not update the multi-pack-index-chain file.

The MIDX chain file (and thus, the lock protecting it) remain untouched,
allowing callers to assemble the chain themselves. This flag requires
`--incremental`, since the notion of a separate layer only makes sense
for incremental MIDXs.

Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-multi-pack-index.adoc b/Documentation/git-multi-pack-index.adoc
index 3a5aa227784071994bfeedefaea650613e383c6e..c26196815e218affb101468bfe50f3439b0c99bb 100644 (file)
--- a/Documentation/git-multi-pack-index.adoc
+++ b/Documentation/git-multi-pack-index.adoc
@@ -11,9 +11,9 @@ SYNOPSIS
 [verse]
 'git multi-pack-index' [<options>] write [--preferred-pack=<pack>]
                         [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]
-                        [--refs-snapshot=<path>]
+                        [--refs-snapshot=<path>] [--[no-]write-chain-file]
 'git multi-pack-index' [<options>] compact [--[no-]incremental]
-                        [--[no-]bitmap] <from> <to>
+                        [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>
 'git multi-pack-index' [<options>] verify
 'git multi-pack-index' [<options>] expire
 'git multi-pack-index' [<options>] repack [--batch-size=<size>]
@@ -83,6 +83,13 @@ marker).
                and packs not present in an existing MIDX layer.
                Migrates non-incremental MIDXs to incremental ones when
                necessary.
+
+       --[no-]write-chain-file::
+               When used with `--incremental`, write a new MIDX layer
+               but do not update the multi-pack-index-chain file.
+               The checksum of the new layer is printed to standard
+               output, allowing the caller to assemble and write the
+               chain itself. Requires `--incremental`.
 --
 
 compact::
@@ -97,6 +104,12 @@ compact::
 
        --[no-]bitmap::
                Control whether or not a multi-pack bitmap is written.
+
+       --[no-]write-chain-file::
+               When used with `--incremental`, write a new compacted
+               MIDX layer but do not update the multi-pack-index-chain
+               file. The checksum of the new layer is printed to
+               standard output. Requires `--incremental`.
 --
 +
 Note that the compact command requires writing a version-2 midx that

diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c
index 0f72d96c02da4c472cf4df110402fe6eac2ed295..f861b4b839463b29f02bc627d1df79b219c136c7 100644 (file)
--- a/builtin/multi-pack-index.c
+++ b/builtin/multi-pack-index.c
@@ -16,11 +16,11 @@
 #define BUILTIN_MIDX_WRITE_USAGE \
        N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]\n" \
           "  [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]\n" \
-          "  [--refs-snapshot=<path>]")
+          "  [--refs-snapshot=<path>] [--[no-]write-chain-file]")
 
 #define BUILTIN_MIDX_COMPACT_USAGE \
        N_("git multi-pack-index [<options>] compact [--[no-]incremental]\n" \
-          "  [--[no-]bitmap] <from> <to>")
+          "  [--[no-]bitmap] [--[no-]write-chain-file] <from> <to>")
 
 #define BUILTIN_MIDX_VERIFY_USAGE \
        N_("git multi-pack-index [<options>] verify")
@@ -153,6 +153,9 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
                        MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
                OPT_BIT(0, "incremental", &opts.flags,
                        N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
+               OPT_NEGBIT(0, "write-chain-file", &opts.flags,
+                       N_("write the multi-pack-index chain file"),
+                       MIDX_WRITE_NO_CHAIN),
                OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
                         N_("write multi-pack index containing only given indexes")),
                OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,
@@ -178,6 +181,15 @@ static int cmd_multi_pack_index_write(int argc, const char **argv,
        if (argc)
                usage_with_options(builtin_multi_pack_index_write_usage,
                                   options);
+
+       if (opts.flags & MIDX_WRITE_NO_CHAIN &&
+           !(opts.flags & MIDX_WRITE_INCREMENTAL)) {
+               error(_("cannot use %s without %s"),
+                     "--no-write-chain-file", "--incremental");
+               usage_with_options(builtin_multi_pack_index_write_usage,
+                                  options);
+       }
+
        source = handle_object_dir_option(repo);
 
        FREE_AND_NULL(options);
@@ -221,6 +233,9 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
                        MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
                OPT_BIT(0, "incremental", &opts.flags,
                        N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
+               OPT_NEGBIT(0, "write-chain-file", &opts.flags,
+                       N_("write the multi-pack-index chain file"),
+                       MIDX_WRITE_NO_CHAIN),
                OPT_END(),
        };
 
@@ -239,6 +254,15 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv,
        if (argc != 2)
                usage_with_options(builtin_multi_pack_index_compact_usage,
                                   options);
+
+       if (opts.flags & MIDX_WRITE_NO_CHAIN &&
+           !(opts.flags & MIDX_WRITE_INCREMENTAL)) {
+               error(_("cannot use %s without %s"),
+                     "--no-write-chain-file", "--incremental");
+               usage_with_options(builtin_multi_pack_index_compact_usage,
+                                  options);
+       }
+
        source = handle_object_dir_option(the_repository);
 
        FREE_AND_NULL(options);

diff --git a/midx-write.c b/midx-write.c
index 5d9409a9741eee34e268415aae44cff3fe02bee6..38c898e5ff5ef16d8e37c411076fda9f94d8e8e8 100644 (file)
--- a/midx-write.c
+++ b/midx-write.c
@@ -1257,7 +1257,7 @@ static int write_midx_internal(struct write_midx_opts *opts)
        unsigned char midx_hash[GIT_MAX_RAWSZ];
        uint32_t start_pack;
        struct hashfile *f = NULL;
-       struct lock_file lk;
+       struct lock_file lk = LOCK_INIT;
        struct tempfile *incr;
        struct write_midx_context ctx = {
                .preferred_pack_idx = NO_PREFERRED_PACK,
@@ -1601,11 +1601,14 @@ static int write_midx_internal(struct write_midx_opts *opts)
        }
 
        if (ctx.incremental) {
-               struct strbuf lock_name = STRBUF_INIT;
+               if (!(opts->flags & MIDX_WRITE_NO_CHAIN)) {
+                       struct strbuf lock_name = STRBUF_INIT;
 
-               get_midx_chain_filename(opts->source, &lock_name);
-               hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR);
-               strbuf_release(&lock_name);
+                       get_midx_chain_filename(opts->source, &lock_name);
+                       hold_lock_file_for_update(&lk, lock_name.buf,
+                                                 LOCK_DIE_ON_ERROR);
+                       strbuf_release(&lock_name);
+               }
 
                incr = mks_tempfile_m(midx_name.buf, 0444);
                if (!incr) {
@@ -1707,16 +1710,23 @@ static int write_midx_internal(struct write_midx_opts *opts)
        if (ctx.num_multi_pack_indexes_before == UINT32_MAX)
                die(_("too many multi-pack-indexes"));
 
+       if (!is_lock_file_locked(&lk))
+               printf("%s\n", hash_to_hex_algop(midx_hash, r->hash_algo));
+       else if (opts->flags & MIDX_WRITE_NO_CHAIN)
+               BUG("lockfile held with MIDX_WRITE_NO_CHAIN set?");
+
        if (ctx.incremental) {
-               FILE *chainf = fdopen_lock_file(&lk, "w");
                struct strbuf final_midx_name = STRBUF_INIT;
                struct multi_pack_index *m = ctx.base_midx;
                struct multi_pack_index **layers = NULL;
                size_t layers_nr = 0, layers_alloc = 0;
 
-               if (!chainf) {
-                       error_errno(_("unable to open multi-pack-index chain file"));
-                       goto cleanup;
+               if (is_lock_file_locked(&lk)){
+                       FILE *chainf = fdopen_lock_file(&lk, "w");
+                       if (!chainf) {
+                               error_errno(_("unable to open multi-pack-index chain file"));
+                               goto cleanup;
+                       }
                }
 
                if (link_midx_to_chain(ctx.base_midx) < 0)
@@ -1773,8 +1783,10 @@ static int write_midx_internal(struct write_midx_opts *opts)
 
                free(layers);
 
-               for (size_t i = 0; i < keep_hashes.nr; i++)
-                       fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes.v[i]);
+               if (is_lock_file_locked(&lk))
+                       for (size_t i = 0; i < keep_hashes.nr; i++)
+                               fprintf(get_lock_file_fp(&lk), "%s\n",
+                                       keep_hashes.v[i]);
        } else {
                strvec_push(&keep_hashes,
                            hash_to_hex_algop(midx_hash, r->hash_algo));
@@ -1783,10 +1795,12 @@ static int write_midx_internal(struct write_midx_opts *opts)
        if (ctx.m || ctx.base_midx)
                odb_close(ctx.repo->objects);
 
-       if (commit_lock_file(&lk) < 0)
-               die_errno(_("could not write multi-pack-index"));
+       if (is_lock_file_locked(&lk)) {
+               if (commit_lock_file(&lk) < 0)
+                       die_errno(_("could not write multi-pack-index"));
 
-       clear_midx_files(opts->source, &keep_hashes, ctx.incremental);
+               clear_midx_files(opts->source, &keep_hashes, ctx.incremental);
+       }
        result = 0;
 
 cleanup:

diff --git a/midx.h b/midx.h
index 08f3728e5204b82fa79f69aef310a0fe346e53f7..5b193882dcf0f1e804c8828383677c7b456d695b 100644 (file)
--- a/midx.h
+++ b/midx.h
@@ -83,6 +83,7 @@ struct multi_pack_index {
 #define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
 #define MIDX_WRITE_INCREMENTAL (1 << 5)
 #define MIDX_WRITE_COMPACT (1 << 6)
+#define MIDX_WRITE_NO_CHAIN (1 << 7)
 
 #define MIDX_EXT_REV "rev"
 #define MIDX_EXT_BITMAP "bitmap"

diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh
index c9f5b4e87aa035d2a568365fd14e59693a725ce3..66d6894761b4a381ccf173dd00ec9e08a343d432 100755 (executable)
--- a/t/t5334-incremental-multi-pack-index.sh
+++ b/t/t5334-incremental-multi-pack-index.sh
@@ -96,6 +96,23 @@ test_expect_success 'show object from second pack' '
        git cat-file -p 2.2
 '
 
+test_expect_success 'write MIDX layer with --no-write-chain-file' '
+       test_commit no-write-chain-file &&
+       git repack -d &&
+
+       cp "$midx_chain" "$midx_chain.bak" &&
+       layer="$(git multi-pack-index write --bitmap --incremental \
+               --no-write-chain-file)" &&
+
+       test_cmp "$midx_chain.bak" "$midx_chain" &&
+       test_path_is_file "$midxdir/multi-pack-index-$layer.midx"
+'
+
+test_expect_success 'write non-incremental MIDX layer with --no-write-chain-file' '
+       test_must_fail git multi-pack-index write --bitmap --no-write-chain-file 2>err &&
+       test_grep "cannot use --no-write-chain-file without --incremental" err
+'
+
 for reuse in false single multi
 do
        test_expect_success "full clone (pack.allowPackReuse=$reuse)" '

diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh
index 40f3844282f04efce0a7a2630bf71057ccac95f4..1a65d48b62b848e992f5193edacedf2af22c973a 100755 (executable)
--- a/t/t5335-compact-multi-pack-index.sh
+++ b/t/t5335-compact-multi-pack-index.sh
@@ -290,4 +290,40 @@ test_expect_success 'MIDX compaction with bitmaps (non-trivial)' '
        )
 '
 
+test_expect_success 'MIDX compaction with --no-write-chain-file' '
+       git init midx-compact-with--no-write-chain-file &&
+       (
+               cd midx-compact-with--no-write-chain-file &&
+
+               git config maintenance.auto false &&
+
+               write_packs A B C D &&
+
+               test_line_count = 4 $midx_chain &&
+               cp "$midx_chain" "$midx_chain".bak &&
+
+               layer="$(git multi-pack-index compact --incremental \
+                       --no-write-chain-file \
+                       "$(nth_line 2 "$midx_chain")" \
+                       "$(nth_line 3 "$midx_chain")")" &&
+
+               test_cmp "$midx_chain.bak" "$midx_chain" &&
+
+               # After writing the new layer, insert it into the chain
+               # manually. This is done in order to make $layer visible
+               # to the read-midx test helper below, and matches what
+               # the MIDX command would do without --no-write-chain-file.
+               {
+                       nth_line 1 "$midx_chain.bak" &&
+                       echo $layer &&
+                       nth_line 4 "$midx_chain.bak"
+               } >$midx_chain &&
+
+               test-tool read-midx $objdir $layer >midx.data &&
+               grep "^pack-B-.*\.idx" midx.data &&
+               grep "^pack-C-.*\.idx" midx.data
+
+       )
+'
+
 test_done