From: Eric Wong <e@80x24.org>
Date: Tue, 21 Mar 2023 23:07:33 +0000 (+0000)
Subject: cindex: check for checkpoint before giant messages
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c7b598b798749d17a7c6cf5bbc3fea9961ac0239;p=thirdparty%2Fpublic-inbox.git

cindex: check for checkpoint before giant messages

Giant messages may put us far over the batch limit if we're
close to it.
---

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index b185731d2..829fe28e5 100644
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -151,6 +151,14 @@ sub store_repo { # wq_do - returns docid
 	}
 }
 
+sub cidx_ckpoint ($$) {
+	my ($self, $msg) = @_;
+	progress($self, $msg);
+	return if $PublicInbox::Search::X{CLOEXEC_UNSET};
+	$self->{xdb}->commit_transaction;
+	$self->{xdb}->begin_transaction;
+}
+
 # sharded reader for `git log --pretty=format: --stdin'
 sub shard_index { # via wq_io_do
 	my ($self, $git, $n, $roots) = @_;
@@ -184,16 +192,18 @@ sub shard_index { # via wq_io_do
 			next;
 		}
 		$TXN_BYTES -= length($buf);
+		if ($TXN_BYTES <= 0) {
+			cidx_ckpoint($self, "[$n] $nr");
+			$TXN_BYTES = $batch_bytes - length($buf);
+		}
 		@$cmt{@FMT} = split(/\n/, $buf, scalar(@FMT));
 		$/ = "\n";
 		add_commit($self, $cmt);
 		last if $DO_QUIT;
 		++$nr;
-		if ($TXN_BYTES <= 0 && !$PublicInbox::Search::X{CLOEXEC_UNSET}) {
-			progress($self, "[$n] $nr");
-			$self->{xdb}->commit_transaction;
+		if ($TXN_BYTES <= 0) {
+			cidx_ckpoint($self, "[$n] $nr");
 			$TXN_BYTES = $batch_bytes;
-			$self->{xdb}->begin_transaction;
 		}
 		$/ = $FS;
 	}