From: Eric Wong <e@80x24.org>
Date: Fri, 29 Nov 2024 06:45:11 +0000 (+0000)
Subject: lei/store: use WAL for over.sqlite3
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f0cee1dcfcfc36e36573ef89ccdd84b0387e1a7b;p=thirdparty%2Fpublic-inbox.git

lei/store: use WAL for over.sqlite3

WAL (write-ahead log) improves parallelism for readers when
they also have write access to the SQLite DB.  While we
can't use WAL for public-inboxes where the -netd processes
are intended to only have read-only permssions, lei/store
always assumes read-write access.

The lei/store */ei15/over.sqlite3 DB was the only SQLite DB used
by lei without WAL.  lei already set WAL for mail_sync.sqlite3
and the saved-searches/*/over.sqlite3 DBs.

Now that all SQLite DBs used by lei are WAL, commit 807abf67
(lei/store: auto-commit for long-running imports, 2024-11-15)
is no longer strictly necessary for parallelism during
long-running imports.  However, 807abf67 may continue to be
useful to minimize the need to refetch after a power outage
during `lei import').

For saved-searches, we'll make use of the new mechanism for
setting {journal_mode} per-instance.
---

diff --git a/lib/PublicInbox/LeiSavedSearch.pm b/lib/PublicInbox/LeiSavedSearch.pm
index ab0a28587..83e19357d 100644
--- a/lib/PublicInbox/LeiSavedSearch.pm
+++ b/lib/PublicInbox/LeiSavedSearch.pm
@@ -234,11 +234,9 @@ sub prepare_dedupe {
 		my $lk = $self->lock_for_scope; # git-config doesn't wait
 		my $oidx = PublicInbox::OverIdx->new($self->{-ovf});
 		$oidx->{-no_fsync} = 1;
+		$oidx->{journal_mode} = 'WAL';
 		$oidx->dbh;
-		if ($creat) {
-			$oidx->{dbh}->do('PRAGMA journal_mode = WAL');
-			$oidx->eidx_prep; # for xref3
-		}
+		$oidx->eidx_prep if $creat; # for xref3
 		$oidx
 	};
 }
diff --git a/lib/PublicInbox/Over.pm b/lib/PublicInbox/Over.pm
index f68964c27..ff5332e77 100644
--- a/lib/PublicInbox/Over.pm
+++ b/lib/PublicInbox/Over.pm
@@ -62,8 +62,14 @@ sub dbh_new {
 		# If an admin is willing to give read-only daemons R/W
 		# permissions; they can enable WAL manually and we will
 		# respect that by not clobbering it.
-		my $jm = $dbh->selectrow_array('PRAGMA journal_mode');
-		$dbh->do('PRAGMA journal_mode = TRUNCATE') if $jm ne 'wal';
+		my $jm = $self->{journal_mode}; # set by lei
+		if (defined $jm) {
+			$dbh->do('PRAGMA journal_mode = '.$jm);
+		} else {
+			$jm = $dbh->selectrow_array('PRAGMA journal_mode');
+			$jm eq 'wal' or
+				$dbh->do('PRAGMA journal_mode = TRUNCATE');
+		}
 
 		$dbh->do('PRAGMA synchronous = OFF') if $rw > 1;
 	}
diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index 879ae0453..10cf8c395 100644
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -464,7 +464,7 @@ sub dbh_close {
 }
 
 sub create {
-	my ($self) = @_;
+	my ($self, $opt) = @_;
 	my $fn = $self->{filename} // do {
 		croak('BUG: no {filename}') unless $self->{dbh};
 		return;
@@ -474,6 +474,7 @@ sub create {
 		my ($dir) = ($fn =~ m!(.*?/)[^/]+\z!);
 		File::Path::mkpath($dir);
 	}
+	$self->{journal_mode} = 'WAL' if $opt->{-private};
 	# create the DB:
 	PublicInbox::Over::dbh($self);
 	$self->dbh_close;
diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 9f686bfac..257519660 100644
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -219,7 +219,7 @@ sub v2_num_for_harder {
 sub _idx_init { # with_umask callback
 	my ($self, $opt) = @_;
 	$self->lock_acquire unless $opt && $opt->{-skip_lock};
-	$self->{oidx}->create;
+	$self->{oidx}->create($opt);
 
 	# xcpdb can change shard count while -watch is idle
 	my $nshards = count_shards($self);