extindex: support --no-multi-pack-index

git multi-pack-index files were creating swap storms and OOM-ing
on my system; so providing an option to disable it seems prudent
given the minor startup time regression.

diff --git a/Documentation/public-inbox-extindex.pod b/Documentation/public-inbox-extindex.pod
index b53e45ed4a3eb21869530ff96c6b7d9a14cdb1a7..2db7d7e90527b63cbde1480cf774551c4c8cd88c 100644 (file)
--- a/Documentation/public-inbox-extindex.pod
+++ b/Documentation/public-inbox-extindex.pod
@@ -80,6 +80,19 @@ doubles the size of the already-large Xapian database.
 Used with C<--reindex>, it will only look for new and stale
 entries and not touch already-indexed messages.
 
+=item --no-multi-pack-index
+
+Disable writing a L<git-multi-pack-index(1)> file to save memory.
+Normally, enabling multi-pack-index speeds up startup time of
+subsequent L<git-cat-file(1)> processes by 3-4%, but generating
+this file requires several GB of memory with large repos.
+
+Unlike the C<core.multiPackIndex> directive in git, it's still
+possible to read existing multi-pack-index files if they are
+created elsewhere.
+
+Available in public-inbox 2.0.0+
+
 =back
 
 =head1 FILES

diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index 14f157a5a66e2cc67198f48730dc610e9044a875..f1a2180ad1061061e9c841c987cc9adf8334b9e7 100644 (file)
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -192,6 +192,13 @@ external indices are configured.
 Do not update the C<all> external index by default.  This negates
 all uses of C<-E> / C<--update-extindex=> on the command-line.
 
+=item --no-multi-pack-index
+
+Disables writing the multi-pack-index when using L</--update-extindex>.
+See L<public-inbox-extindex(1)/--no-multi-pack-index> for details.
+
+Available in public-inbox 2.0.0+
+
 =item --since=DATESTRING
 
 =item --after=DATESTRING

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 763a124c1f4e005fbc72af2057ca0750b96a77c9..774fa47bbbbf25a7e9cfcb5ea8d6900fe225372f 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -1287,7 +1287,7 @@ sub idx_init { # similar to V2Writable
        ($has_new || $prune_nr || $new ne '') and
                $self->{mg}->write_alternates($mode, $alt, $new);
        my $restore = $self->with_umask;
-       if ($git_midx) {
+       if ($git_midx && ($opt->{'multi-pack-index'} // 1)) {
                my @cmd = ('multi-pack-index');
                push @cmd, '--no-progress' if ($opt->{quiet}//0) > 1;
                my $lk = $self->lock_for_scope;

diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
index bee824b1878611691344af7f725063a47fd6b421..2e5a5d2cdc2ff3b51901523e509cc5eab3ef1750 100755 (executable)
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -32,7 +32,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i
                indexlevel|index-level|L=s max_size|max-size=s
                batch_size|batch-size=s
                dedupe:s@ gc commit-interval=i watch scan! dry-run|n
-               all C=s@ help|h))
+               multi-pack-index! all C=s@ help|h))
        or die $help;
 if ($opt->{help}) { print $help; exit 0 };
 die "--jobs must be >= 0\n" if defined $opt->{jobs} && $opt->{jobs} < 0;

diff --git a/script/public-inbox-index b/script/public-inbox-index
index 74232ebfb26636dcd405bde747dfa02ba8f9e561..a13e44bf78c9ecafbd642fbc28ca8c49ffdc0a13 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -44,6 +44,7 @@ GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
                batch_size|batch-size=s
                since|after=s until|before=s
                sequential-shard|seq-shard
+               multi-pack-index!
                no-update-extindex update-extindex|E=s@
                fast-noop|F skip-docdata all C=s@ help|h))
        or die $help;

diff --git a/t/extsearch.t b/t/extsearch.t
index 090f6db5cc45eba90930f5c2a53b31b81cc0f7b7..797aa8f54fd088fd6a9e2148970351ab469f2317 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -559,6 +559,15 @@ EOM
        for (@xdb) {
                ok(!$_->get_metadata('indexlevel'), 'no indexlevel in >0 shard')
        }
+       my $mpi = "$d/ALL.git/objects/pack/multi-pack-index";
+       SKIP: {
+               skip 'git too old for for multi-pack-index', 2 if !-f $mpi;
+               unlink glob("$d/ALL.git/objects/pack/*");
+               ok run_script([qw(-extindex --all -L medium -j3
+                               --no-multi-pack-index), $d]),
+                               'test --no-multi-pack-index';
+               ok !-f $mpi, '--no-multi-pack-index respected';
+       }
 }
 
 test_lei(sub {