*index: support --commit=$SECONDS to adjust the commit interval

The default 5s interval is too low for initial indexing with
gian inboxes or extindices which aren't ready for public
consumption, yet.  It's also too low for --wal users since
WAL mode of SQLite (by default) allows readers to proceed
without waiting on a writer.

I'm not sure what the defaults should be, so allow users to
change it, for now.

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 8252af281f22449c53bb340e0542dccae4416e97..fef1892b1bc240b2ad593ee28ec4218b6e30dc53 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -819,7 +819,8 @@ sub update_checkpoint ($;$) {
        $self->{need_checkpoint} // return; # must be defined via local
        return ++$self->{need_checkpoint} if $nr >= $self->{batch_bytes};
        my $now = now;
-       my $next = $self->{next_checkpoint} //= $now + $CHECKPOINT_INTVL;
+       my $next = $self->{next_checkpoint} //= $now +
+               ($self->{-opt}->{'commit-interval'} // $CHECKPOINT_INTVL);
        $self->{need_checkpoint} += ($now > $next ? 1 : 0);
 }
 

diff --git a/script/public-inbox-cindex b/script/public-inbox-cindex
index e6b1e67b5d0e36213a664dbb1a36ebaba1ee52df..2c504a1731091e6cc147034ecaca2aa388389293 100755 (executable)
--- a/script/public-inbox-cindex
+++ b/script/public-inbox-cindex
@@ -27,6 +27,7 @@ EOF
 my $opt = { fsync => 1, scan => 1 }; # --no-scan is hidden
 GetOptions($opt, qw(quiet|q verbose|v+ reindex jobs|j=i fsync|sync! dangerous
                cow! indexlevel|index-level|L=s join:s@
+               commit-interval|commit=i
                batch_size|batch-size=s max_size|max-size=s
                include|I=s@ only=s@ all show:s@
                project-list=s exclude=s@ project-root|r=s

diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index 598636c940e5a71ddcc8eda79d9805bd9c9d7716..6f0aa209b5eb502ef172a026a3a5e4e0088a531d 100755 (executable)
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -37,7 +37,7 @@ GetOptions($opt, qw(jobs|j=i index! help|h C=s@),
                # index options
                qw(verbose|v+ rethread compact|c+ fsync|sync!
                indexlevel|index-level|L=s max_size|max-size=s
-               batch_size|batch-size=s cow! wal
+               commit-interval|commit=i batch_size|batch-size=s cow! wal
                sequential-shard|seq-shard
                )) or die $help;
 if ($opt->{help}) { print $help; exit 0 };

diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
index 2ba0a607f7f4c37631d0d3377524c99ccf63e5ad..d12b07c4612e4ce7c68dfb5efb712801d0659dac 100755 (executable)
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -29,9 +29,10 @@ EOF
 my $opt = { quiet => -1, compact => 0, fsync => 1, scan => 1 };
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i
                fsync|sync! fast dangerous cow! wal defrag=i
+               commit-interval|commit=i
                indexlevel|index-level|L=s max_size|max-size=s
                batch_size|batch-size=s
-               dedupe:s@ gc commit-interval=i watch scan! dry-run|n
+               dedupe:s@ gc watch scan! dry-run|n
                multi-pack-index! all C=s@ help|h))
        or die $help;
 if ($opt->{help}) { print $help; exit 0 };

diff --git a/script/public-inbox-index b/script/public-inbox-index
index 0eb88ba145ce90a473d325db9e72495ce6d3bfb7..67728091b6c611180117d6c65d190a669d2d7a31 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -38,7 +38,7 @@ my $opt = {
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
                fsync|sync! xapian_only|xapian-only dangerous cow! wal
                indexlevel|index-level|L=s max_size|max-size=s
-               defrag=i batch_size|batch-size=s
+               commit-interval|commit=i defrag=i batch_size|batch-size=s
                since|after=s until|before=s
                sequential-shard|seq-shard
                multi-pack-index!

diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index e93d41e050f73df484cbfd9303ce835838c807f7..3faa63afb598c39c5fae55d19ebed2beef89c68e 100755 (executable)
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -29,7 +29,7 @@ my $opt = { quiet => -1, compact => 0, fsync => 1,
        -eidx_ok => 1, -cidx_ok => 1 };
 GetOptions($opt, qw(
        cow! fsync|sync! compact|c reshard|R=i
-       max_size|max-size=s batch_size|batch-size=s
+       commit-interval|commit=i max_size|max-size=s batch_size|batch-size=s
        sequential-shard|seq-shard
        jobs|j=i quiet|q verbose|v
        blocksize|b=s no-full|n fuller|F