cindex: implement --max-size=SIZE

This matches existing behavior of -index and -extindex, and
will hopefully allow me to avoid OOM problems by skipping
problematic commits.

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index fcd28671ba1d7d7013d9a98891600ad6d257a5da..b185731d20ac241c6b7791cd44ef71eb9c3f9ab0 100644 (file)
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -161,6 +161,7 @@ sub shard_index { # via wq_io_do
        my $op_p = delete($self->{1}) // die 'BUG: no {1} op_p';
        my $batch_bytes = $self->{-opt}->{batch_size} //
                                $PublicInbox::SearchIdx::BATCH_BYTES;
+       my $max_size = $self->{-opt}->{max_size};
        # local-ized in parent before fork
        $TXN_BYTES = $batch_bytes;
        local $self->{git} = $git; # for patchid
@@ -177,6 +178,11 @@ sub shard_index { # via wq_io_do
        $self->begin_txn_lazy;
        while (defined($buf = <$rd>)) {
                chomp($buf);
+               if ($max_size && length($buf) >= $max_size) {
+                       my ($H, undef) = split(/\n/, $buf, 2);
+                       warn "W: skipping $H (", length($buf)," >= $max_size)\n";
+                       next;
+               }
                $TXN_BYTES -= length($buf);
                @$cmt{@FMT} = split(/\n/, $buf, scalar(@FMT));
                $/ = "\n";

diff --git a/script/public-inbox-cindex b/script/public-inbox-cindex
index 420ef4dee75d56621b7ea300c7d7b846f33bf11d..e2500b930ce2d81c7225839af38ffe3622af4f66 100755 (executable)
--- a/script/public-inbox-cindex
+++ b/script/public-inbox-cindex
@@ -16,6 +16,7 @@ usage: public-inbox-cindex [options] --project-list=FILE PROJECT_ROOT
   --update | -u       update previously-indexed code repos with `-d'
   --jobs=NUM          set or disable parallelization (NUM=0)
   --batch-size=BYTES  flush changes to OS after a given number of bytes
+  --max-size=BYTES    do not index commit diffs larger than the given size
   --prune             prune old repos and commits
   --reindex           reindex previously indexed repos
   --verbose | -v      increase verbosity (may be repeated)
@@ -25,7 +26,8 @@ See public-inbox-cindex(1) man page for full documentation.
 EOF
 my $opt = { fsync => 1, scan => 1 }; # --no-scan is hidden
 GetOptions($opt, qw(quiet|q verbose|v+ reindex jobs|j=i fsync|sync! dangerous
-               indexlevel|index-level|L=s batch_size|batch-size=s
+               indexlevel|index-level|L=s
+               batch_size|batch-size=s max_size|max-size=s
                project-list=s exclude=s@
                d=s update|u scan! prune dry-run|n C=s@ help|h))
        or die $help;