Merge branch 'tb/cruft-packs'

A mechanism to pack unreachable objects into a "cruft pack",
instead of ejecting them into loose form to be reclaimed later, has
been introduced.

* tb/cruft-packs:
  sha1-file.c: don't freshen cruft packs
  builtin/gc.c: conditionally avoid pruning objects via loose
  builtin/repack.c: add cruft packs to MIDX during geometric repack
  builtin/repack.c: use named flags for existing_packs
  builtin/repack.c: allow configuring cruft pack generation
  builtin/repack.c: support generating a cruft pack
  builtin/pack-objects.c: --cruft with expiration
  reachable: report precise timestamps from objects in cruft packs
  reachable: add options to add_unseen_recent_objects_to_traversal
  builtin/pack-objects.c: --cruft without expiration
  builtin/pack-objects.c: return from create_object_entry()
  t/helper: add 'pack-mtimes' test-tool
  pack-mtimes: support writing pack .mtimes files
  chunk-format.h: extract oid_version()
  pack-write: pass 'struct packing_data' to 'stage_tmp_packfiles'
  pack-mtimes: support reading .mtimes files
  Documentation/technical: add cruft-packs.txt

diff --cc Documentation/Makefile
index d3f043f50d2817421e3d9d42ae454d541c51e68e,3e884f55c1773d437d8a33dcafb3bb099c5059bc..f2e7fc1daa5f95696510cd38d689a119fd5f2a72
--- 1/Documentation/Makefile
--- 2/Documentation/Makefile
+++ b/Documentation/Makefile
@@@ -93,8 -93,8 +93,9 @@@ SP_ARTICLES += $(API_DOCS
  TECH_DOCS += MyFirstContribution
  TECH_DOCS += MyFirstObjectWalk
  TECH_DOCS += SubmittingPatches
 +TECH_DOCS += ToolsForGit
  TECH_DOCS += technical/bundle-format
+ TECH_DOCS += technical/cruft-packs
  TECH_DOCS += technical/hash-function-transition
  TECH_DOCS += technical/http-protocol
  TECH_DOCS += technical/index-format

diff --cc Makefile
Simple merge

diff --cc builtin/gc.c
Simple merge

diff --cc builtin/pack-objects.c
Simple merge

diff --cc builtin/repack.c
index 0e4aae80c09913e732da4275f41f7d16b8f43587,15071fadbe8bf56de5cfeb7736659e81ffd13cd0..c957b2959f7865ce58eea0737f3b8d20c340bcec
--- 1/builtin/repack.c
--- 2/builtin/repack.c
+++ b/builtin/repack.c
@@@ -131,14 -160,17 +160,19 @@@ static void collect_pack_filenames(stru
                fname = xmemdupz(e->d_name, len);
  
                if ((extra_keep->nr > 0 && i < extra_keep->nr) ||
-                   (file_exists(mkpath("%s/%s.keep", packdir, fname))))
+                   (file_exists(mkpath("%s/%s.keep", packdir, fname)))) {
                        string_list_append_nodup(fname_kept_list, fname);
-               else
-                       string_list_append_nodup(fname_nonkept_list, fname);
+               } else {
+                       struct string_list_item *item;
+                       item = string_list_append_nodup(fname_nonkept_list,
+                                                       fname);
+                       if (file_exists(mkpath("%s/%s.mtimes", packdir, fname)))
+                               item->util = (void*)(uintptr_t)CRUFT_PACK;
+               }
        }
        closedir(dir);
 +
 +      string_list_sort(fname_kept_list);
  }
  
  static void remove_redundant_pack(const char *dir_name, const char *base_name)
@@@ -345,27 -361,10 +365,29 @@@ static void init_pack_geometry(struct p
        geometry = *geometry_p;
  
        for (p = get_all_packs(the_repository); p; p = p->next) {
 -              if (!pack_kept_objects && p->pack_keep)
 -                      continue;
 +              if (!pack_kept_objects) {
 +                      /*
 +                       * Any pack that has its pack_keep bit set will appear
 +                       * in existing_kept_packs below, but this saves us from
 +                       * doing a more expensive check.
 +                       */
 +                      if (p->pack_keep)
 +                              continue;
 +
 +                      /*
 +                       * The pack may be kept via the --keep-pack option;
 +                       * check 'existing_kept_packs' to determine whether to
 +                       * ignore it.
 +                       */
 +                      strbuf_reset(&buf);
 +                      strbuf_addstr(&buf, pack_basename(p));
 +                      strbuf_strip_suffix(&buf, ".pack");
 +
 +                      if (string_list_has_string(existing_kept_packs, buf.buf))
 +                              continue;
 +              }
+               if (p->is_cruft)
+                       continue;
  
                ALLOC_GROW(geometry->pack,
                           geometry->pack_nr + 1,
@@@ -856,8 -941,33 +965,35 @@@ int cmd_repack(int argc, const char **a
        if (!names.nr && !po_args.quiet)
                printf_ln(_("Nothing new to pack."));
  
+       if (pack_everything & PACK_CRUFT) {
+               const char *pack_prefix;
+               if (!skip_prefix(packtmp, packdir, &pack_prefix))
+                       die(_("pack prefix %s does not begin with objdir %s"),
+                           packtmp, packdir);
+               if (*pack_prefix == '/')
+                       pack_prefix++;
+ 
+               if (!cruft_po_args.window)
+                       cruft_po_args.window = po_args.window;
+               if (!cruft_po_args.window_memory)
+                       cruft_po_args.window_memory = po_args.window_memory;
+               if (!cruft_po_args.depth)
+                       cruft_po_args.depth = po_args.depth;
+               if (!cruft_po_args.threads)
+                       cruft_po_args.threads = po_args.threads;
+ 
+               cruft_po_args.local = po_args.local;
+               cruft_po_args.quiet = po_args.quiet;
+ 
+               ret = write_cruft_pack(&cruft_po_args, pack_prefix, &names,
+                                      &existing_nonkept_packs,
+                                      &existing_kept_packs);
+               if (ret)
+                       return ret;
+       }
+ 
 +      string_list_sort(&names);
 +
        for_each_string_list_item(item, &names) {
                item->util = (void *)(uintptr_t)populate_pack_exts(item->string);
        }

diff --cc bulk-checkin.c
Simple merge

diff --cc commit-graph.c
Simple merge

diff --cc midx.c
Simple merge

diff --cc object-file.c
Simple merge

diff --cc object-store.h
Simple merge

diff --cc packfile.c
Simple merge