searchidx: run `git patch-id' in parallel

Informal benchmarks show a rough 5% indexing improvement on an
SMP system when there are idle cores due to Xapian shards being
I/O bound (since `git patch-id' is mainly CPU bound).

This is only parallelized on a per-patch basis.  Further
increasing parallelism would increase complexity and probably
not be worth it since `git patch-id' is reasonably fast while
our text indexing tends to be slow.

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index 3a30b5e5c6825a2a32d550e8fe091d9766f22413..54a2ba2205d09b6ebfce1d70815fe805ea5bc42b 100644 (file)
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -675,7 +675,7 @@ sub get_roots ($$) {
                { 0 => $refs, 1 => $roots_fh }, \&index_repo, $self, $git)
 }
 
-# for PublicInbox::SearchIdx::patch_id and with_umask
+# for PublicInbox::SearchIdx `git patch-id' call and with_umask
 sub git { $_[0]->{git} }
 
 sub load_existing ($) { # for -u/--update

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 9566b14d133be8cb35082cf538e4669cd13f075d..32598b7c02a1bf956690ef7799718b35d7199b5c 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -22,7 +22,7 @@ use POSIX qw(strftime);
 use Fcntl qw(SEEK_SET);
 use Time::Local qw(timegm);
 use PublicInbox::OverIdx;
-use PublicInbox::Spawn qw(run_wait run_qx);
+use PublicInbox::Spawn qw(run_wait popen_rd);
 use PublicInbox::Git qw(git_unquote);
 use PublicInbox::MsgTime qw(msg_timestamp msg_datestamp);
 use PublicInbox::Address;
@@ -350,20 +350,13 @@ sub index_diff ($$$) {
        index_text($self, join("\n", @$xnq), 1, 'XNQ');
 }
 
-sub patch_id {
-       my ($self, $sref) = @_;
-       my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
-       my $opt = { 0 => [ ':utf8', $sref ], 2 => \(my $err) };
-       my $id = run_qx($git->cmd(qw(patch-id --stable)), undef, $opt);
-       warn $err if $err;
-       $id =~ /\A([a-f0-9]{40,})/ ? $1 : undef;
-}
-
 sub index_body_text {
        my ($self, $doc, $sref) = @_;
-       if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) {
-               my $id = patch_id($self, $sref);
-               $doc->add_term('XDFID'.$id) if defined($id);
+       my $rd;
+       if ($$sref =~ /^(?:diff|---|\+\+\+) /ms) { # start patch-id in parallel
+               my $git = ($self->{ibx} // $self->{eidx} // $self)->git;
+               $rd = popen_rd($git->cmd(qw(patch-id --stable)), undef,
+                               { 0 => [ ':utf8', $sref ] });
        }
 
        # split off quoted and unquoted blocks:
@@ -387,6 +380,11 @@ sub index_body_text {
                }
                undef $txt; # free memory
        }
+       if (defined $rd) { # reap `git patch-id'
+               (readline($rd) // '') =~ /\A([a-f0-9]{40,})/ and
+                       $doc->add_term('XDFID'.$1);
+               $rd->close or warn "W: git patch-id failed: \$?=$? (non-fatal)"
+       }
 }
 
 sub index_xapian { # msg_iter callback