From: Eric Wong <e@80x24.org>
Date: Fri, 29 Aug 2025 19:04:01 +0000 (+0000)
Subject: *index: support --commit=$SECONDS to adjust the commit interval
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=660fb7381d69f47695e978e2257223545594dd70;p=thirdparty%2Fpublic-inbox.git

*index: support --commit=$SECONDS to adjust the commit interval

The default 5s interval is too low for initial indexing with
gian inboxes or extindices which aren't ready for public
consumption, yet.  It's also too low for --wal users since
WAL mode of SQLite (by default) allows readers to proceed
without waiting on a writer.

I'm not sure what the defaults should be, so allow users to
change it, for now.
---

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 8252af281..fef1892b1 100644
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -819,7 +819,8 @@ sub update_checkpoint ($;$) {
 	$self->{need_checkpoint} // return; # must be defined via local
 	return ++$self->{need_checkpoint} if $nr >= $self->{batch_bytes};
 	my $now = now;
-	my $next = $self->{next_checkpoint} //= $now + $CHECKPOINT_INTVL;
+	my $next = $self->{next_checkpoint} //= $now +
+		($self->{-opt}->{'commit-interval'} // $CHECKPOINT_INTVL);
 	$self->{need_checkpoint} += ($now > $next ? 1 : 0);
 }
 
diff --git a/script/public-inbox-cindex b/script/public-inbox-cindex
index e6b1e67b5..2c504a173 100755
--- a/script/public-inbox-cindex
+++ b/script/public-inbox-cindex
@@ -27,6 +27,7 @@ EOF
 my $opt = { fsync => 1, scan => 1 }; # --no-scan is hidden
 GetOptions($opt, qw(quiet|q verbose|v+ reindex jobs|j=i fsync|sync! dangerous
 		cow! indexlevel|index-level|L=s join:s@
+		commit-interval|commit=i
 		batch_size|batch-size=s max_size|max-size=s
 		include|I=s@ only=s@ all show:s@
 		project-list=s exclude=s@ project-root|r=s
diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index 598636c94..6f0aa209b 100755
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -37,7 +37,7 @@ GetOptions($opt, qw(jobs|j=i index! help|h C=s@),
 		# index options
 		qw(verbose|v+ rethread compact|c+ fsync|sync!
 		indexlevel|index-level|L=s max_size|max-size=s
-		batch_size|batch-size=s cow! wal
+		commit-interval|commit=i batch_size|batch-size=s cow! wal
 		sequential-shard|seq-shard
 		)) or die $help;
 if ($opt->{help}) { print $help; exit 0 };
diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
index 2ba0a607f..d12b07c46 100755
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -29,9 +29,10 @@ EOF
 my $opt = { quiet => -1, compact => 0, fsync => 1, scan => 1 };
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i
 		fsync|sync! fast dangerous cow! wal defrag=i
+		commit-interval|commit=i
 		indexlevel|index-level|L=s max_size|max-size=s
 		batch_size|batch-size=s
-		dedupe:s@ gc commit-interval=i watch scan! dry-run|n
+		dedupe:s@ gc watch scan! dry-run|n
 		multi-pack-index! all C=s@ help|h))
 	or die $help;
 if ($opt->{help}) { print $help; exit 0 };
diff --git a/script/public-inbox-index b/script/public-inbox-index
index 0eb88ba14..67728091b 100755
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -38,7 +38,7 @@ my $opt = {
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
 		fsync|sync! xapian_only|xapian-only dangerous cow! wal
 		indexlevel|index-level|L=s max_size|max-size=s
-		defrag=i batch_size|batch-size=s
+		commit-interval|commit=i defrag=i batch_size|batch-size=s
 		since|after=s until|before=s
 		sequential-shard|seq-shard
 		multi-pack-index!
diff --git a/script/public-inbox-xcpdb b/script/public-inbox-xcpdb
index e93d41e05..3faa63afb 100755
--- a/script/public-inbox-xcpdb
+++ b/script/public-inbox-xcpdb
@@ -29,7 +29,7 @@ my $opt = { quiet => -1, compact => 0, fsync => 1,
 	-eidx_ok => 1, -cidx_ok => 1 };
 GetOptions($opt, qw(
 	cow! fsync|sync! compact|c reshard|R=i
-	max_size|max-size=s batch_size|batch-size=s
+	commit-interval|commit=i max_size|max-size=s batch_size|batch-size=s
 	sequential-shard|seq-shard
 	jobs|j=i quiet|q verbose|v
 	blocksize|b=s no-full|n fuller|F