Merge branch 'tb/repack-write-midx'

"git repack" has been taught to generate multi-pack reachability
bitmaps.

* tb/repack-write-midx:
  test-read-midx: fix leak of bitmap_index struct
  builtin/repack.c: pass `--refs-snapshot` when writing bitmaps
  builtin/repack.c: make largest pack preferred
  builtin/repack.c: support writing a MIDX while repacking
  builtin/repack.c: extract showing progress to a variable
  builtin/repack.c: rename variables that deal with non-kept packs
  builtin/repack.c: keep track of existing packs unconditionally
  midx: preliminary support for `--refs-snapshot`
  builtin/multi-pack-index.c: support `--stdin-packs` mode
  midx: expose `write_midx_file_only()` publicly

diff --cc Documentation/git-multi-pack-index.txt
Simple merge

diff --cc builtin/multi-pack-index.c
index 6426bbdaceab63a6664f492c34eeaf910ff97abd,4b827a07c0c1809e9880c4c09402601cafb27696..075d15d706246a7e4f6a4ec3faecaebce220399f
--- 1/builtin/multi-pack-index.c
--- 2/builtin/multi-pack-index.c
+++ b/builtin/multi-pack-index.c
@@@ -60,23 -64,16 +63,33 @@@ static struct option *add_common_option
        return parse_options_concat(common_opts, prev);
  }
  
 +static int git_multi_pack_index_write_config(const char *var, const char *value,
 +                                           void *cb)
 +{
 +      if (!strcmp(var, "pack.writebitmaphashcache")) {
 +              if (git_config_bool(var, value))
 +                      opts.flags |= MIDX_WRITE_BITMAP_HASH_CACHE;
 +              else
 +                      opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
 +      }
 +
 +      /*
 +       * We should never make a fall-back call to 'git_default_config', since
 +       * this was already called in 'cmd_multi_pack_index()'.
 +       */
 +      return 0;
 +}
 +
+ static void read_packs_from_stdin(struct string_list *to)
+ {
+       struct strbuf buf = STRBUF_INIT;
+       while (strbuf_getline(&buf, stdin) != EOF)
+               string_list_append(to, buf.buf);
+       string_list_sort(to);
+ 
+       strbuf_release(&buf);
+ }
+ 
  static int cmd_multi_pack_index_write(int argc, const char **argv)
  {
        struct option *options;
@@@ -86,8 -83,10 +99,12 @@@
                           N_("pack for reuse when computing a multi-pack bitmap")),
                OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
                        MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
 +              OPT_BIT(0, "progress", &opts.flags,
 +                      N_("force progress reporting"), MIDX_PROGRESS),
+               OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
+                        N_("write multi-pack index containing only given indexes")),
+               OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,
+                            N_("refs snapshot for selecting bitmap commits")),
                OPT_END(),
        };
  

diff --cc builtin/repack.c
Simple merge

diff --cc midx.c
index 4a01583fe81546920d4dcf990c8bafc85b82758a,18a4f25aa35364f73018610a61f0f0617e597c3c..7e06e8597561fe1341a53983a36b4a8a0eafe378
--- 1/midx.c
--- 2/midx.c
+++ b/midx.c
@@@ -997,12 -1072,9 +1058,12 @@@ static int write_midx_bitmap(char *midx
        char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash));
        int ret;
  
 +      if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
 +              options |= BITMAP_OPT_HASH_CACHE;
 +
        prepare_midx_packing_data(&pdata, ctx);
  
-       commits = find_commits_for_midx_bitmap(&commits_nr, ctx);
+       commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, ctx);
  
        /*
         * Build the MIDX-order index based on pdata.objects (which is already

diff --cc midx.h
Simple merge

diff --cc pack-bitmap.c
Simple merge

diff --cc pack-bitmap.h
Simple merge

diff --cc t/t5319-multi-pack-index.sh
Simple merge

diff --cc t/t5326-multi-pack-bitmaps.sh
index ec4aa89f63e3fc473b6e4c548d63aa2088e10576,069dab3e1721cfc8d32c3117e6420eeac06a37f9..e187f90f29e2b592b4074add76d76212b9d0d729
--- 1/t/t5326-multi-pack-bitmaps.sh
--- 2/t/t5326-multi-pack-bitmaps.sh
+++ b/t/t5326-multi-pack-bitmaps.sh
@@@ -283,34 -283,86 +283,116 @@@ test_expect_success 'pack.preferBitmapT
        )
  '
  
+ test_expect_success 'writing a bitmap with --refs-snapshot' '
+       git init repo &&
+       test_when_finished "rm -fr repo" &&
+       (
+               cd repo &&
+ 
+               test_commit one &&
+               test_commit two &&
+ 
+               git rev-parse one >snapshot &&
+ 
+               git repack -ad &&
+ 
+               # First, write a MIDX which see both refs/tags/one and
+               # refs/tags/two (causing both of those commits to receive
+               # bitmaps).
+               git multi-pack-index write --bitmap &&
+ 
+               test_path_is_file $midx &&
+               test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
+ 
+               test-tool bitmap list-commits | sort >bitmaps &&
+               grep "$(git rev-parse one)" bitmaps &&
+               grep "$(git rev-parse two)" bitmaps &&
+ 
+               rm -fr $midx-$(midx_checksum $objdir).bitmap &&
+               rm -fr $midx-$(midx_checksum $objdir).rev &&
+               rm -fr $midx &&
+ 
+               # Then again, but with a refs snapshot which only sees
+               # refs/tags/one.
+               git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
+ 
+               test_path_is_file $midx &&
+               test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
+ 
+               test-tool bitmap list-commits | sort >bitmaps &&
+               grep "$(git rev-parse one)" bitmaps &&
+               ! grep "$(git rev-parse two)" bitmaps
+       )
+ '
+ 
+ test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
+       git init repo &&
+       test_when_finished "rm -fr repo" &&
+       (
+               cd repo &&
+ 
+               test_commit_bulk --message="%s" 103 &&
+ 
+               git log --format="%H" >commits.raw &&
+               sort <commits.raw >commits &&
+ 
+               git log --format="create refs/tags/%s %H" HEAD >refs &&
+               git update-ref --stdin <refs &&
+ 
+               git multi-pack-index write --bitmap &&
+               test_path_is_file $midx &&
+               test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
+ 
+               test-tool bitmap list-commits | sort >bitmaps &&
+               comm -13 bitmaps commits >before &&
+               test_line_count = 1 before &&
+ 
+               (
+                       grep -vf before commits.raw &&
+                       # mark missing commits as preferred
+                       sed "s/^/+/" before
+               ) >snapshot &&
+ 
+               rm -fr $midx-$(midx_checksum $objdir).bitmap &&
+               rm -fr $midx-$(midx_checksum $objdir).rev &&
+               rm -fr $midx &&
+ 
+               git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
+               test-tool bitmap list-commits | sort >bitmaps &&
+               comm -13 bitmaps commits >after &&
+ 
+               ! test_cmp before after
+       )
+ '
+ 
 +test_expect_success 'hash-cache values are propagated from pack bitmaps' '
 +      rm -fr repo &&
 +      git init repo &&
 +      test_when_finished "rm -fr repo" &&
 +      (
 +              cd repo &&
 +
 +              test_commit base &&
 +              test_commit base2 &&
 +              git repack -adb &&
 +
 +              test-tool bitmap dump-hashes >pack.raw &&
 +              test_file_not_empty pack.raw &&
 +              sort pack.raw >pack.hashes &&
 +
 +              test_commit new &&
 +              git repack &&
 +              git multi-pack-index write --bitmap &&
 +
 +              test-tool bitmap dump-hashes >midx.raw &&
 +              sort midx.raw >midx.hashes &&
 +
 +              # ensure that every namehash in the pack bitmap can be found in
 +              # the midx bitmap (i.e., that there are no oid-namehash pairs
 +              # unique to the pack bitmap).
 +              comm -23 pack.hashes midx.hashes >dropped.hashes &&
 +              test_must_be_empty dropped.hashes
 +      )
 +'
 +
  test_done