From: Taylor Blau Date: Tue, 19 May 2026 15:57:51 +0000 (-0400) Subject: midx: introduce `--no-write-chain-file` for incremental MIDX writes X-Git-Url: http://git.ipfire.org/gitweb/index.cgi?a=commitdiff_plain;h=8d342ed4b5dfbaa16f8bbc4537907d1e1224358e;p=thirdparty%2Fgit.git midx: introduce `--no-write-chain-file` for incremental MIDX writes When writing an incremental MIDX layer, the MIDX machinery writes the new layer into the multi-pack-index.d directory and then updates the multi-pack-index-chain file to include the freshly written layer. Future callers however may not wish to immediately update the MIDX chain itself, preferring instead to write out new layer(s) themselves before atomically updating the chain. Concretely, the new incremental MIDX-based repacking strategy will want to do exactly this (that is, assemble the new MIDX chain itself before writing a new chain file and atomically linking it into place). Introduce a `--no-write-chain-file` flag that: * writes the new MIDX layer into the multi-pack-index.d directory * prints its checksum * does not update the multi-pack-index-chain file. The MIDX chain file (and thus, the lock protecting it) remain untouched, allowing callers to assemble the chain themselves. This flag requires `--incremental`, since the notion of a separate layer only makes sense for incremental MIDXs. Signed-off-by: Taylor Blau Signed-off-by: Junio C Hamano --- diff --git a/Documentation/git-multi-pack-index.adoc b/Documentation/git-multi-pack-index.adoc index 3a5aa22778..c26196815e 100644 --- a/Documentation/git-multi-pack-index.adoc +++ b/Documentation/git-multi-pack-index.adoc @@ -11,9 +11,9 @@ SYNOPSIS [verse] 'git multi-pack-index' [] write [--preferred-pack=] [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs] - [--refs-snapshot=] + [--refs-snapshot=] [--[no-]write-chain-file] 'git multi-pack-index' [] compact [--[no-]incremental] - [--[no-]bitmap] + [--[no-]bitmap] [--[no-]write-chain-file] 'git multi-pack-index' [] verify 'git multi-pack-index' [] expire 'git multi-pack-index' [] repack [--batch-size=] @@ -83,6 +83,13 @@ marker). and packs not present in an existing MIDX layer. Migrates non-incremental MIDXs to incremental ones when necessary. + + --[no-]write-chain-file:: + When used with `--incremental`, write a new MIDX layer + but do not update the multi-pack-index-chain file. + The checksum of the new layer is printed to standard + output, allowing the caller to assemble and write the + chain itself. Requires `--incremental`. -- compact:: @@ -97,6 +104,12 @@ compact:: --[no-]bitmap:: Control whether or not a multi-pack bitmap is written. + + --[no-]write-chain-file:: + When used with `--incremental`, write a new compacted + MIDX layer but do not update the multi-pack-index-chain + file. The checksum of the new layer is printed to + standard output. Requires `--incremental`. -- + Note that the compact command requires writing a version-2 midx that diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 0f72d96c02..f861b4b839 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -16,11 +16,11 @@ #define BUILTIN_MIDX_WRITE_USAGE \ N_("git multi-pack-index [] write [--preferred-pack=]\n" \ " [--[no-]bitmap] [--[no-]incremental] [--[no-]stdin-packs]\n" \ - " [--refs-snapshot=]") + " [--refs-snapshot=] [--[no-]write-chain-file]") #define BUILTIN_MIDX_COMPACT_USAGE \ N_("git multi-pack-index [] compact [--[no-]incremental]\n" \ - " [--[no-]bitmap] ") + " [--[no-]bitmap] [--[no-]write-chain-file] ") #define BUILTIN_MIDX_VERIFY_USAGE \ N_("git multi-pack-index [] verify") @@ -153,6 +153,9 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX), OPT_BIT(0, "incremental", &opts.flags, N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL), + OPT_NEGBIT(0, "write-chain-file", &opts.flags, + N_("write the multi-pack-index chain file"), + MIDX_WRITE_NO_CHAIN), OPT_BOOL(0, "stdin-packs", &opts.stdin_packs, N_("write multi-pack index containing only given indexes")), OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot, @@ -178,6 +181,15 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, if (argc) usage_with_options(builtin_multi_pack_index_write_usage, options); + + if (opts.flags & MIDX_WRITE_NO_CHAIN && + !(opts.flags & MIDX_WRITE_INCREMENTAL)) { + error(_("cannot use %s without %s"), + "--no-write-chain-file", "--incremental"); + usage_with_options(builtin_multi_pack_index_write_usage, + options); + } + source = handle_object_dir_option(repo); FREE_AND_NULL(options); @@ -221,6 +233,9 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX), OPT_BIT(0, "incremental", &opts.flags, N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL), + OPT_NEGBIT(0, "write-chain-file", &opts.flags, + N_("write the multi-pack-index chain file"), + MIDX_WRITE_NO_CHAIN), OPT_END(), }; @@ -239,6 +254,15 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, if (argc != 2) usage_with_options(builtin_multi_pack_index_compact_usage, options); + + if (opts.flags & MIDX_WRITE_NO_CHAIN && + !(opts.flags & MIDX_WRITE_INCREMENTAL)) { + error(_("cannot use %s without %s"), + "--no-write-chain-file", "--incremental"); + usage_with_options(builtin_multi_pack_index_compact_usage, + options); + } + source = handle_object_dir_option(the_repository); FREE_AND_NULL(options); diff --git a/midx-write.c b/midx-write.c index 5d9409a974..38c898e5ff 100644 --- a/midx-write.c +++ b/midx-write.c @@ -1257,7 +1257,7 @@ static int write_midx_internal(struct write_midx_opts *opts) unsigned char midx_hash[GIT_MAX_RAWSZ]; uint32_t start_pack; struct hashfile *f = NULL; - struct lock_file lk; + struct lock_file lk = LOCK_INIT; struct tempfile *incr; struct write_midx_context ctx = { .preferred_pack_idx = NO_PREFERRED_PACK, @@ -1601,11 +1601,14 @@ static int write_midx_internal(struct write_midx_opts *opts) } if (ctx.incremental) { - struct strbuf lock_name = STRBUF_INIT; + if (!(opts->flags & MIDX_WRITE_NO_CHAIN)) { + struct strbuf lock_name = STRBUF_INIT; - get_midx_chain_filename(opts->source, &lock_name); - hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR); - strbuf_release(&lock_name); + get_midx_chain_filename(opts->source, &lock_name); + hold_lock_file_for_update(&lk, lock_name.buf, + LOCK_DIE_ON_ERROR); + strbuf_release(&lock_name); + } incr = mks_tempfile_m(midx_name.buf, 0444); if (!incr) { @@ -1707,16 +1710,23 @@ static int write_midx_internal(struct write_midx_opts *opts) if (ctx.num_multi_pack_indexes_before == UINT32_MAX) die(_("too many multi-pack-indexes")); + if (!is_lock_file_locked(&lk)) + printf("%s\n", hash_to_hex_algop(midx_hash, r->hash_algo)); + else if (opts->flags & MIDX_WRITE_NO_CHAIN) + BUG("lockfile held with MIDX_WRITE_NO_CHAIN set?"); + if (ctx.incremental) { - FILE *chainf = fdopen_lock_file(&lk, "w"); struct strbuf final_midx_name = STRBUF_INIT; struct multi_pack_index *m = ctx.base_midx; struct multi_pack_index **layers = NULL; size_t layers_nr = 0, layers_alloc = 0; - if (!chainf) { - error_errno(_("unable to open multi-pack-index chain file")); - goto cleanup; + if (is_lock_file_locked(&lk)){ + FILE *chainf = fdopen_lock_file(&lk, "w"); + if (!chainf) { + error_errno(_("unable to open multi-pack-index chain file")); + goto cleanup; + } } if (link_midx_to_chain(ctx.base_midx) < 0) @@ -1773,8 +1783,10 @@ static int write_midx_internal(struct write_midx_opts *opts) free(layers); - for (size_t i = 0; i < keep_hashes.nr; i++) - fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes.v[i]); + if (is_lock_file_locked(&lk)) + for (size_t i = 0; i < keep_hashes.nr; i++) + fprintf(get_lock_file_fp(&lk), "%s\n", + keep_hashes.v[i]); } else { strvec_push(&keep_hashes, hash_to_hex_algop(midx_hash, r->hash_algo)); @@ -1783,10 +1795,12 @@ static int write_midx_internal(struct write_midx_opts *opts) if (ctx.m || ctx.base_midx) odb_close(ctx.repo->objects); - if (commit_lock_file(&lk) < 0) - die_errno(_("could not write multi-pack-index")); + if (is_lock_file_locked(&lk)) { + if (commit_lock_file(&lk) < 0) + die_errno(_("could not write multi-pack-index")); - clear_midx_files(opts->source, &keep_hashes, ctx.incremental); + clear_midx_files(opts->source, &keep_hashes, ctx.incremental); + } result = 0; cleanup: diff --git a/midx.h b/midx.h index 08f3728e52..5b193882dc 100644 --- a/midx.h +++ b/midx.h @@ -83,6 +83,7 @@ struct multi_pack_index { #define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4) #define MIDX_WRITE_INCREMENTAL (1 << 5) #define MIDX_WRITE_COMPACT (1 << 6) +#define MIDX_WRITE_NO_CHAIN (1 << 7) #define MIDX_EXT_REV "rev" #define MIDX_EXT_BITMAP "bitmap" diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh index c9f5b4e87a..66d6894761 100755 --- a/t/t5334-incremental-multi-pack-index.sh +++ b/t/t5334-incremental-multi-pack-index.sh @@ -96,6 +96,23 @@ test_expect_success 'show object from second pack' ' git cat-file -p 2.2 ' +test_expect_success 'write MIDX layer with --no-write-chain-file' ' + test_commit no-write-chain-file && + git repack -d && + + cp "$midx_chain" "$midx_chain.bak" && + layer="$(git multi-pack-index write --bitmap --incremental \ + --no-write-chain-file)" && + + test_cmp "$midx_chain.bak" "$midx_chain" && + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" +' + +test_expect_success 'write non-incremental MIDX layer with --no-write-chain-file' ' + test_must_fail git multi-pack-index write --bitmap --no-write-chain-file 2>err && + test_grep "cannot use --no-write-chain-file without --incremental" err +' + for reuse in false single multi do test_expect_success "full clone (pack.allowPackReuse=$reuse)" ' diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh index 40f3844282..1a65d48b62 100755 --- a/t/t5335-compact-multi-pack-index.sh +++ b/t/t5335-compact-multi-pack-index.sh @@ -290,4 +290,40 @@ test_expect_success 'MIDX compaction with bitmaps (non-trivial)' ' ) ' +test_expect_success 'MIDX compaction with --no-write-chain-file' ' + git init midx-compact-with--no-write-chain-file && + ( + cd midx-compact-with--no-write-chain-file && + + git config maintenance.auto false && + + write_packs A B C D && + + test_line_count = 4 $midx_chain && + cp "$midx_chain" "$midx_chain".bak && + + layer="$(git multi-pack-index compact --incremental \ + --no-write-chain-file \ + "$(nth_line 2 "$midx_chain")" \ + "$(nth_line 3 "$midx_chain")")" && + + test_cmp "$midx_chain.bak" "$midx_chain" && + + # After writing the new layer, insert it into the chain + # manually. This is done in order to make $layer visible + # to the read-midx test helper below, and matches what + # the MIDX command would do without --no-write-chain-file. + { + nth_line 1 "$midx_chain.bak" && + echo $layer && + nth_line 4 "$midx_chain.bak" + } >$midx_chain && + + test-tool read-midx $objdir $layer >midx.data && + grep "^pack-B-.*\.idx" midx.data && + grep "^pack-C-.*\.idx" midx.data + + ) +' + test_done