lei_mirror+fetch: don't slurp `git show-ref' output

While uncommon, some git repos have hundreds of thousands of
refs and slurping that output into memory can bloat the heap.
Introduce a sha_all sub in PublicInbox::SHA to loop until EOF
and rely on autodie for checking sysread errors.

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index e31432b9a5857b3e99f29053781836d9fe101b1b..aeee37c00def6ab8b8d1ccbe08fd3be1b487baaa 100644 (file)
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -45,7 +45,7 @@ use POSIX qw(WNOHANG SEEK_SET);
 use File::Path ();
 use File::Spec ();
 use List::Util qw(max);
-use PublicInbox::SHA qw(sha256_hex);
+use PublicInbox::SHA qw(sha256_hex sha_all);
 use PublicInbox::Search qw(xap_terms);
 use PublicInbox::SearchIdx qw(add_val);
 use PublicInbox::Config qw(glob2re rel2abs_collapsed);
@@ -386,10 +386,7 @@ sub fp_fini { # run_git cb
        my (undef, $self, $git, $prep_repo) = @_;
        my $refs = $git->{-repo}->{refs} // die 'BUG: no {-repo}->{refs}';
        sysseek($refs, 0, SEEK_SET);
-       my $buf;
-       my $dig = PublicInbox::SHA->new(256);
-       while (sysread($refs, $buf, 65536)) { $dig->add($buf) }
-       $git->{-repo}->{fp} = $dig->hexdigest;
+       $git->{-repo}->{fp} = sha_all(256, $refs)->hexdigest;
 }
 
 sub ct_start ($$$) {

diff --git a/lib/PublicInbox/Fetch.pm b/lib/PublicInbox/Fetch.pm
index 6e9b1e94ef53cb2ef47c9087ee919dee1b02f84b..e41dd448a14e0c54afe9f07caec144ee814cbf15 100644 (file)
--- a/lib/PublicInbox/Fetch.pm
+++ b/lib/PublicInbox/Fetch.pm
@@ -10,6 +10,7 @@ use PublicInbox::Admin;
 use PublicInbox::LEI;
 use PublicInbox::LeiCurl;
 use PublicInbox::LeiMirror;
+use PublicInbox::SHA qw(sha_all);
 use File::Temp ();
 
 sub new { bless {}, __PACKAGE__ }
@@ -92,9 +93,8 @@ sub do_manifest ($$$) {
 
 sub get_fingerprint2 {
        my ($git_dir) = @_;
-       require PublicInbox::SHA;
        my $rd = popen_rd([qw(git show-ref)], undef, { -C => $git_dir });
-       PublicInbox::SHA::sha256(do { local $/; <$rd> });
+       sha_all(256, $rd)->digest; # ignore show-ref errors
 }
 
 sub writable_dir ($) {

diff --git a/lib/PublicInbox/Git.pm b/lib/PublicInbox/Git.pm
index 476dcf30f016b1a84e4dedfec4a40b085e85d819..9c26d8bfb400bf0b8f23f359d36e4168b70676eb 100644 (file)
--- a/lib/PublicInbox/Git.pm
+++ b/lib/PublicInbox/Git.pm
@@ -23,7 +23,7 @@ use PublicInbox::ProcessIONBF;
 use PublicInbox::Tmpfile;
 use IO::Poll qw(POLLIN);
 use Carp qw(croak carp);
-use PublicInbox::SHA ();
+use PublicInbox::SHA qw(sha_all);
 our %HEXLEN2SHA = (40 => 1, 64 => 256);
 our %OFMT2HEXLEN = (sha1 => 40, sha256 => 64);
 our @EXPORT_OK = qw(git_unquote git_quote %HEXLEN2SHA %OFMT2HEXLEN read_all);
@@ -620,10 +620,8 @@ sub manifest_entry {
                        $ent->{reference} = $buf;
                }
        }
-       my $dig = PublicInbox::SHA->new(1);
-       while (CORE::read($sr, $buf, 65536)) { $dig->add($buf) }
+       $ent->{fingerprint} = sha_all(1, $sr)->hexdigest;
        CORE::close $sr or return; # empty, uninitialized git repo
-       $ent->{fingerprint} = $dig->hexdigest;
        $ent->{modified} = modified(undef, $mod);
        chomp($buf = <$own> // '');
        utf8::decode($buf);

diff --git a/lib/PublicInbox/LeiMirror.pm b/lib/PublicInbox/LeiMirror.pm
index 47fb767b398c58ecfabf62d4b468ca0af9b80997..43e59e6c495f7b0f88342a617bbcfc64ebbe1949 100644 (file)
--- a/lib/PublicInbox/LeiMirror.pm
+++ b/lib/PublicInbox/LeiMirror.pm
@@ -19,10 +19,10 @@ use PublicInbox::Inbox;
 use PublicInbox::Git qw(read_all);
 use PublicInbox::LeiCurl;
 use PublicInbox::OnDestroy;
-use PublicInbox::SHA qw(sha256_hex sha1_hex);
+use PublicInbox::SHA qw(sha256_hex sha_all);
 use POSIX qw(strftime);
-use autodie qw(chdir chmod close open pipe readlink seek symlink sysopen
-               truncate unlink);
+use autodie qw(chdir chmod close open pipe readlink
+               seek symlink sysopen sysseek truncate unlink);
 
 our $LIVE; # pid => callback
 our $FGRP_TODO; # objstore -> [[ to resume ], [ to clone ]]
@@ -533,10 +533,10 @@ sub fp_done {
        }
        return if !keep_going($self);
        my $fh = delete $self->{-show_ref} // die 'BUG: no show-ref output';
-       seek($fh, SEEK_SET, 0);
+       sysseek($fh, SEEK_SET, 0);
        $self->{-ent} // die 'BUG: no -ent';
        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
-       my $B = sha1_hex(read_all($fh));
+       my $B = sha_all(1, $fh)->hexdigest;
        return $cb->($self, @arg) if $A ne $B;
        $self->{lei}->qerr("# $self->{-key} up-to-date");
 }
@@ -730,10 +730,10 @@ sub up_fp_done {
        my ($self) = @_;
        return if !keep_going($self);
        my $fh = delete $self->{-show_ref_up} // die 'BUG: no show-ref output';
-       seek($fh, SEEK_SET, 0);
+       sysseek($fh, SEEK_SET, 0);
        $self->{-ent} // die 'BUG: no -ent';
        my $A = $self->{-ent}->{fingerprint} // die 'BUG: no fingerprint';
-       my $B = sha1_hex(read_all($fh));
+       my $B = sha_all(1, $fh)->hexdigest;
        return if $A eq $B;
        $self->{-ent}->{fingerprint} = $B;
        push @{$self->{chg}->{fp_mismatch}}, $self->{-key};

diff --git a/lib/PublicInbox/SHA.pm b/lib/PublicInbox/SHA.pm
index 81f626188fe28c01f99e245d3902089c33035762..3fa8530e61d39539ce695583611d76d4bc0c4424 100644 (file)
--- a/lib/PublicInbox/SHA.pm
+++ b/lib/PublicInbox/SHA.pm
@@ -12,7 +12,8 @@
 package PublicInbox::SHA;
 use v5.12;
 require Exporter;
-our @EXPORT_OK = qw(sha1_hex sha256_hex sha256);
+our @EXPORT_OK = qw(sha1_hex sha256_hex sha256 sha_all);
+use autodie qw(sysread);
 our @ISA;
 
 BEGIN {
@@ -55,4 +56,12 @@ EOM
 }
 
 } # /BEGIN
+
+sub sha_all ($$) {
+       my ($n, $fh) = @_;
+       my ($dig, $buf) = (PublicInbox::SHA->new($n));
+       while (sysread($fh, $buf, 65536)) { $dig->add($buf) }
+       $dig
+}
+
 1;