init|*index|convert: --wal enables Write-Ahead Log in SQLite

SQLite WAL <https://sqlite.org/wal.html> ought to improve
concurrency for readers when writers are active.  While users
always had the option of enabling it with the sqlite3(1) tool,
having a `--wal' switch may encourage its use.

It's already on by default for lei/store and mail_sync.sqlite3
where readers and writers are the same user, but enabling it by
default for public-facing daemons would cause permissions
problems so it remains optional.

The key downside of WAL is readers (-(netd|httpd|imapd|nntpd|pop3d)
require write access to the directories containing over.sqlite3
and msgmap.sqlite3).  In the past, public-facing read-only
daemons were not expected to ever have write permissions to
inbox and extindex directories, but WAL requires writability
since SQLite||DBD::SQLite currently doesn't provide a way to
guarantee the persistence of -wal and -shm files, especially
when accessed by sqlite3(1) or 3rd-party scripts.

The main advantage of supporting WAL is so the writers won't
block readers, avoiding busy timeouts on the read side and
reducing the need for writers to commit transactions
periodically to keep readers happy (currently a ridiculously
short 5s).  We can investigate higher commit intervals (or
eliminating them entirely) for users of WAL.

WAL should also reduce fragmentation in btrfs and similar CoW
filesystems by reducing fsync(2) calls, this remains to be
proven as measuring such changes takes a while and I don't
want to put more wear on my SSD.

diff --git a/Documentation/public-inbox-convert.pod b/Documentation/public-inbox-convert.pod
index a2f8caf59cf8ffa35d01f5481c2ff58d1be237b6..86e8265b353b953021a0526d9d96a25bc9a60d77 100644 (file)
--- a/Documentation/public-inbox-convert.pod
+++ b/Documentation/public-inbox-convert.pod
@@ -49,6 +49,8 @@ process which distributes work to the Xapian shards.
 
 =item --max-size=BYTES
 
+=item --wal
+
 These options affect indexing.  They have no effect if
 L</--no-index> is specified
 

diff --git a/Documentation/public-inbox-extindex.pod b/Documentation/public-inbox-extindex.pod
index 2db7d7e90527b63cbde1480cf774551c4c8cd88c..d9d4d901180f4ecae6bb389fd41b0df6dbf25e73 100644 (file)
--- a/Documentation/public-inbox-extindex.pod
+++ b/Documentation/public-inbox-extindex.pod
@@ -34,6 +34,8 @@ along with L<DBD::SQLite> and L<DBI> Perl modules.
 
 =item --batch-size SIZE
 
+=item --wal
+
 These switches behave as they do for L<public-inbox-index(1)>
 
 =item --all

diff --git a/Documentation/public-inbox-index.pod b/Documentation/public-inbox-index.pod
index f1a2180ad1061061e9c841c987cc9adf8334b9e7..c839f74ecafff423c554a80dfcaea23f6fd9ff94 100644 (file)
--- a/Documentation/public-inbox-index.pod
+++ b/Documentation/public-inbox-index.pod
@@ -209,6 +209,14 @@ Available in public-inbox 2.0.0+
 
 Passed directly to L<git-log(1)> to limit changes for C<--reindex>
 
+=item --wal
+
+Enable WAL (Write-Ahead-Log) on SQLite files.  This may reduce
+fragmentation and improve performance; but requires read-only
+processes (e.g. public-inbox-netd) to have write access
+to all directories containing *.sqlite3 files.
+See L<https://sqlite.org/wal.html> for more info.
+
 =back
 
 =head1 FILES

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index 33c25134df357833e5d3ed05641fa0c41b249fa2..bc1e68d77e5cbe80dfa67616f2b424b90ab013fc 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -68,6 +68,7 @@ sub new {
        $self->{shards} = $self->count_shards ||
                nproc_shards { nproc => $opt->{jobs} };
        my $oidx = PublicInbox::OverIdx->new("$self->{xpfx}/over.sqlite3");
+       $oidx->{journal_mode} = 'wal' if $opt->{wal};
        $self->{-no_fsync} = $oidx->{-no_fsync} = 1 if !$opt->{fsync};
        $self->{-dangerous} = 1 if $opt->{dangerous};
        $self->{oidx} = $oidx;

diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index f69aec75460377832bd1cc95c3614e4e9a3ec7b7..a4e63a5c4a921b70c8d8fce277ff561adb57224c 100644 (file)
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -30,13 +30,16 @@ sub assert_usable_dir {
 
 sub _init_v1 {
        my ($self) = @_;
-       my $skip_artnum = ($self->{-creat_opt} // {})->{'skip-artnum'};
-       if (defined($self->{indexlevel}) || defined($skip_artnum)) {
+       my $opt = $self->{-creat_opt} // {};
+       my $skip_artnum = $opt->{'skip-artnum'};
+       if (defined($self->{indexlevel}) || defined($skip_artnum) ||
+                       $opt->{wal}) {
                require PublicInbox::SearchIdx;
                require PublicInbox::Msgmap;
                my $sidx = PublicInbox::SearchIdx->new($self, 1); # just create
+               $sidx->{oidx}->{journal_mode} = 'wal' if $opt->{wal};
                $sidx->begin_txn_lazy;
-               my $mm = PublicInbox::Msgmap->new_file($self, 1);
+               my $mm = PublicInbox::Msgmap->new_file($self, 1, $opt);
                if (defined $skip_artnum) {
                        $mm->{dbh}->begin_work;
                        $mm->skip_artnum($skip_artnum);

diff --git a/lib/PublicInbox/LeiStore.pm b/lib/PublicInbox/LeiStore.pm
index 46460c569e7b11407a97ce1b29cdfb6eb55637c4..9d97ef58afbbb57669006266dac28d84ddcd14a5 100644 (file)
--- a/lib/PublicInbox/LeiStore.pm
+++ b/lib/PublicInbox/LeiStore.pm
@@ -139,7 +139,7 @@ sub eidx_init {
        my $tl = wantarray && $self->{-err_wr} ?
                        on_destroy(\&_tail_err, $self) :
                        undef;
-       $eidx->idx_init({-private => 1}); # acquires lock
+       $eidx->idx_init({wal => 1, -private => 1}); # acquires lock
        wantarray ? ($eidx, $tl) : $eidx;
 }
 

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 71415febca674a204af6ec19cecfbc0d6418f834..0dacdb848a1f9754281d05fa90a688cc428f6251 100644 (file)
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -16,7 +16,7 @@ use PublicInbox::Over;
 use Scalar::Util qw(blessed);
 
 sub new_file {
-       my ($class, $ibx, $rw) = @_;
+       my ($class, $ibx, $rw, $opt) = @_;
        my $f;
        if (blessed($ibx)) {
                $f = $ibx->mm_file;
@@ -27,6 +27,7 @@ sub new_file {
        return if !$rw && !-r $f;
 
        my $self = bless { filename => $f }, $class;
+       $self->{journal_mode} = 'wal' if $opt->{wal};
        my $dbh = $self->{dbh} = PublicInbox::Over::dbh_new($self, $rw);
        if ($rw) {
                $dbh->begin_work;

diff --git a/lib/PublicInbox/OverIdx.pm b/lib/PublicInbox/OverIdx.pm
index b7bebf58518f54e02b02a9bde8e06b1e19731762..93f2f11b23e07f607f402c699ff0eb283de6e5fd 100644 (file)
--- a/lib/PublicInbox/OverIdx.pm
+++ b/lib/PublicInbox/OverIdx.pm
@@ -473,7 +473,7 @@ sub create {
                my ($dir) = ($fn =~ m!(.*?/)[^/]+\z!);
                File::Path::mkpath($dir);
        }
-       $self->{journal_mode} = 'WAL' if $opt->{-private};
+       $self->{journal_mode} = 'wal' if $opt->{wal};
        # create the DB:
        PublicInbox::Over::dbh($self);
        $self->dbh_close;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 125f43f797242004e98f82bf16f9cd26988204b3..8ddbc07a0f33d3c15c76e6ee65da0f96628dd269 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -540,7 +540,7 @@ sub v1_mm_init ($) {
        die "BUG: v1_mm_init is only for v1\n" if $self->{ibx}->version != 1;
        $self->{mm} //= do {
                require PublicInbox::Msgmap;
-               PublicInbox::Msgmap->new_file($self->{ibx}, 1);
+               PublicInbox::Msgmap->new_file($self->{ibx}, 1, $self->{-opt});
        };
 }
 
@@ -1108,6 +1108,7 @@ sub _index_sync {
        local $SIG{QUIT} = $quit;
        local $SIG{INT} = $quit;
        local $SIG{TERM} = $quit;
+       $self->{oidx}->{journal_mode} = 'wal' if $opt->{wal};
        my $xdb = $self->begin_txn_lazy;
        $self->{oidx}->rethread_prepare($opt);
        my $mm = v1_mm_init $self;

diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index ab988de407a34a4392476b27c71124a43f8e63e1..ac802878891e14751197727ab827686fa18a1659 100644 (file)
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -949,6 +949,7 @@ sub create_inbox ($;@) {
        $opt{-primary_address} //= $addr->[0] // "$ident\@example.com";
        my $parallel = delete($opt{importer_parallel}) // 0;
        my $creat_opt = { nproc => delete($opt{nproc}) // 1 };
+       $creat_opt->{wal} = 1 if $opt{wal};
        my $ibx = PublicInbox::InboxWritable->new({ %opt }, $creat_opt);
        if (!-f "$dir/creat.stamp") {
                my $im = $ibx->importer($parallel);

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 3ae6c4d08843821c5dca6c415b37cd18a203b603..7186b2cce034b98f6b0b3c3d7ea016a4aecc6d83 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -85,7 +85,7 @@ sub init_inbox {
        my $skip_epoch = ($self->{ibx}->{-creat_opt} // {})->{'skip-epoch'};
        $max = $skip_epoch if (defined($skip_epoch) && !defined($max));
        $self->{mg}->add_epoch($max // 0);
-       $self->idx_init;
+       $self->idx_init($self->{ibx}->{-creat_opt});
        my $skip_artnum = ($self->{ibx}->{-creat_opt} // {})->{'skip-artnum'};
        $self->{mm}->skip_artnum($skip_artnum) if defined $skip_artnum;
        $self->done;
@@ -243,7 +243,7 @@ sub _idx_init { # with_umask callback
 
        # Now that all subprocesses are up, we can open the FDs
        # for SQLite:
-       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1);
+       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1, $opt);
        $mm->{dbh}->begin_work;
 }
 

diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index 713c288159787d73225e2402167185ae1dd7849e..cf87b4ebb3c7e5c0a56907650f72e16a43dbb636 100755 (executable)
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -37,7 +37,7 @@ GetOptions($opt, qw(jobs|j=i index! help|h C=s@),
                # index options
                qw(verbose|v+ rethread compact|c+ fsync|sync!
                indexlevel|index-level|L=s max_size|max-size=s
-               batch_size|batch-size=s
+               batch_size|batch-size=s wal
                sequential-shard|seq-shard
                )) or die $help;
 if ($opt->{help}) { print $help; exit 0 };
@@ -72,12 +72,19 @@ if ($opt->{'index'}) {
        PublicInbox::Admin::require_or_die(keys %$mods);
        PublicInbox::Admin::progress_prepare($opt);
        $env = PublicInbox::Admin::index_prepare($opt, $cfg);
+       if (!$opt->{wal}) {
+               $opt->{wal} = 1 if $old->over->dbh->selectrow_array(
+                                       'PRAGMA journal_mode') eq 'wal';
+               $old->cleanup;
+       }
 }
 local %ENV = (%$env, %ENV) if $env;
 my $new = { %$old };
 $new->{inboxdir} = PublicInbox::Config::rel2abs_collapsed($new_dir);
 $new->{version} = 2;
-$new = PublicInbox::InboxWritable->new($new, { nproc => $opt->{jobs} });
+my $creat_opt = { nproc => $opt->{jobs} };
+$creat_opt->{wal} = 1 if $opt->{wal};
+$new = PublicInbox::InboxWritable->new($new, $creat_opt);
 $new->{-no_fsync} = 1 if !$opt->{fsync};
 my $v2w;
 

diff --git a/script/public-inbox-extindex b/script/public-inbox-extindex
index 2e5a5d2cdc2ff3b51901523e509cc5eab3ef1750..6b1b06c724cf4c714302f04aa27d97e9214e2989 100755 (executable)
--- a/script/public-inbox-extindex
+++ b/script/public-inbox-extindex
@@ -28,7 +28,7 @@ See public-inbox-extindex(1) man page for full documentation.
 EOF
 my $opt = { quiet => -1, compact => 0, fsync => 1, scan => 1 };
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i
-               fsync|sync! fast dangerous
+               fsync|sync! fast dangerous wal
                indexlevel|index-level|L=s max_size|max-size=s
                batch_size|batch-size=s
                dedupe:s@ gc commit-interval=i watch scan! dry-run|n

diff --git a/script/public-inbox-index b/script/public-inbox-index
index 82baa7b483be850feff5e34c55710b793c984fef..72f5613817080afcb9e3babcf711ba042387a7e1 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -36,7 +36,7 @@ my $opt = {
        'update-extindex' => [], # ":s@" optional arg sets '' if no arg given
 };
 GetOptions($opt, qw(verbose|v+ reindex rethread compact|c+ jobs|j=i prune
-               fsync|sync! xapian_only|xapian-only dangerous
+               fsync|sync! xapian_only|xapian-only dangerous wal
                indexlevel|index-level|L=s max_size|max-size=s
                batch_size|batch-size=s
                since|after=s until|before=s

diff --git a/script/public-inbox-init b/script/public-inbox-init
index b959fd917d6ea7bc8c82adfd2e2a3a10cb3c8344..7497d058b450a0beb7ff6e37a9c341620df354c1 100755 (executable)
--- a/script/public-inbox-init
+++ b/script/public-inbox-init
@@ -41,6 +41,7 @@ my (@c_extra, @chdir);
 my $creat_opt = {};
 my %opts = (
        'V|version=i' => \$version,
+       wal => \($creat_opt->{wal}),
        'L|index-level|indexlevel=s' => \$indexlevel,
        'S|skip|skip-epoch=i' => \($creat_opt->{'skip-epoch'}),
        'skip-artnum=i' => \($creat_opt->{'skip-artnum'}),

diff --git a/t/convert-compact.t b/t/convert-compact.t
index b123f17b2c8d2769c4cef219f01e357be9dd7433..533dd67d529df545564a3f7da0d94d81df019c50 100644 (file)
--- a/t/convert-compact.t
+++ b/t/convert-compact.t
@@ -11,6 +11,7 @@ require_mods(qw(DBD::SQLite Xapian));
 have_xapian_compact;
 my ($tmpdir, $for_destroy) = tmpdir();
 my $ibx = create_inbox 'v1', indexlevel => 'medium', tmpdir => "$tmpdir/v1",
+               wal => 1,
                pre_cb => sub {
                        my ($inboxdir) = @_;
                        PublicInbox::Import::init_bare($inboxdir);
@@ -46,6 +47,12 @@ foreach (@xdir) {
                'sharedRepository respected on file after convert');
 }
 
+is $ibx->mm->{dbh}->selectrow_array('PRAGMA journal_mode'), 'wal',
+       '-compact preserves msgmap.sqlite3 wal';
+is $ibx->over->dbh->selectrow_array('PRAGMA journal_mode'), 'wal',
+       '-compact preserves over.sqlite3 wal';
+$ibx->cleanup;
+
 local $ENV{PI_CONFIG} = '/dev/null';
 my ($out, $err) = ('', '');
 my $rdr = { 1 => \$out, 2 => \$err };
@@ -83,6 +90,11 @@ ok(run_script($cmd, $env, $rdr), 'v2 compact works');
 $ibx->{inboxdir} = "$tmpdir/x/v2";
 $ibx->{version} = 2;
 is($ibx->mm->num_highwater, $hwm, 'highwater mark unchanged in v2 inbox');
+is $ibx->mm->{dbh}->selectrow_array('PRAGMA journal_mode'), 'wal',
+       '-convert preserves msgmap.sqlite3 wal';
+is $ibx->over->dbh->selectrow_array('PRAGMA journal_mode'), 'wal',
+       '-convert preserves over.sqlite3 wal';
+$ibx->cleanup;
 
 @xdir = glob("$tmpdir/x/v2/xap*/*");
 foreach (@xdir) {

diff --git a/t/extsearch.t b/t/extsearch.t
index 698421235598c83ebd1b15e5751e2c6dc00ee324..bb1fcfc50113f767e1f6080d64f49c630f95cd07 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -44,11 +44,13 @@ run_script(['-mda', '--no-precheck'], $env, { 0 => $in }) or BAIL_OUT '-mda';
 
 run_script([qw(-index -Lbasic), "$home/v1test"]) or BAIL_OUT "index $?";
 
-ok(run_script([qw(-extindex --dangerous --all), "$home/extindex"]),
+ok(run_script([qw(-extindex --dangerous --all --wal), "$home/extindex"]),
        'extindex init');
 {
        my $es = PublicInbox::ExtSearch->new("$home/extindex");
        ok($es->has_threadid, '->has_threadid');
+       my $jm = $es->over->dbh->selectrow_array('PRAGMA journal_mode');
+       is $jm, 'wal', "--wal enables `journal_mode = wal' in over.sqlite3";
 }
 
 if ('with boost') {

diff --git a/t/indexlevels-mirror.t b/t/indexlevels-mirror.t
index c852f72cb0150496c82d48be2a1bfaa841620452..cf813d0887bf90a061891e82eb796021e3eefae3 100644 (file)
--- a/t/indexlevels-mirror.t
+++ b/t/indexlevels-mirror.t
@@ -34,7 +34,8 @@ my $import_index_incremental = sub {
        local $ENV{PI_CONFIG} = "$tmpdir/config";
 
        # index master (required for v1)
-       my @cmd = (qw(-index -j0 --dangerous), $ibx->{inboxdir}, "-L$level");
+       my @cmd = (qw(-index --wal -j0 --dangerous),
+               $ibx->{inboxdir}, "-L$level");
        push @cmd, '-c' if have_xapian_compact;
        ok(run_script(\@cmd, undef, { 2 => \$err }), 'index master');
        my $ro_master = PublicInbox::Inbox->new({
@@ -44,6 +45,10 @@ my $import_index_incremental = sub {
        my $msgs = $ro_master->over->recent;
        is(scalar(@$msgs), 1, 'only one message in master, so far');
        is($msgs->[0]->{mid}, 'm@1', 'first message in master indexed');
+       my $jm = $ro_master->over->dbh->selectrow_array('PRAGMA journal_mode');
+       is $jm, 'wal', 'over.sqlite3 respects --wal';
+       $jm = $ro_master->mm->{dbh}->selectrow_array('PRAGMA journal_mode');
+       is $jm, 'wal', 'msgmap.sqlite3 respects --wal';
 
        # clone
        @cmd = (qw(git clone --mirror -q));

diff --git a/t/init.t b/t/init.t
index 275192cfefcde079ddabed62ff7e8c07652942a0..05cbc94213bcb6b6d05676c14557e411514ca50b 100644 (file)
--- a/t/init.t
+++ b/t/init.t
@@ -14,10 +14,19 @@ sub quiet_fail {
        ok(!run_script($cmd, undef, { 2 => \$err, 1 => \$err }), $msg);
 }
 
+my $check_wal = sub {
+       my ($bn) = ($_[0] =~ m!/([^/]+)/*\z!);
+       my $ibx = PublicInbox::Inbox->new({inboxdir => $_[0]});
+       my $jm = $ibx->over->dbh->selectrow_array('PRAGMA journal_mode');
+       is $jm, 'wal', "--wal works with $bn (over.sqlite3)";
+       $jm = $ibx->mm->{dbh}->selectrow_array('PRAGMA journal_mode');
+       is $jm, 'wal', "--wal works with $bn (msgmap.sqlite3)";
+};
+
 {
        local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
        my $cfgfile = "$ENV{PI_DIR}/config";
-       my $cmd = [ '-init', 'blist', "$tmpdir/blist",
+       my $cmd = [ qw(-init blist), "$tmpdir/blist",
                   qw(http://example.com/blist blist@example.com) ];
        my $umask = umask(070) // xbail "umask: $!";
        ok(run_script($cmd), 'public-inbox-init OK');
@@ -127,12 +136,13 @@ SKIP: {
        local $ENV{PI_DIR} = "$tmpdir/.public-inbox/";
        local $ENV{PI_EMERGENCY} = "$tmpdir/.public-inbox/emergency";
        my $cfgfile = "$ENV{PI_DIR}/config";
-       my $cmd = [ '-init', '-V2', 'v2list', "$tmpdir/v2list",
+       my $cmd = [ qw(-init -V2 --wal v2list), "$tmpdir/v2list",
                   qw(http://example.com/v2list v2list@example.com) ];
        ok(run_script($cmd), 'public-inbox-init -V2 OK');
        ok(-d "$tmpdir/v2list", 'v2list directory exists');
        ok(-f "$tmpdir/v2list/msgmap.sqlite3", 'msgmap exists');
        ok(-d "$tmpdir/v2list/all.git", 'catch-all.git directory exists');
+       $check_wal->("$tmpdir/v2list");
        $cmd = [ '-init', 'v2list', "$tmpdir/v2list",
                   qw(http://example.com/v2list v2list@example.com) ];
        ok(run_script($cmd), 'public-inbox-init is idempotent');
@@ -204,8 +214,8 @@ SKIP: {
 
        $addr = 'skip4@example.com';
        $env = { ORIGINAL_RECIPIENT => $addr };
-       $cmd = [ qw(-init -V1 --skip-artnum 12 -Lmedium skip4), "$tmpdir/skip4",
-                  qw(http://example.com/skip4), $addr ];
+       $cmd = [ qw(-init -V1 --wal --skip-artnum 12 -Lmedium skip4),
+                       "$tmpdir/skip4", qw(http://example.com/skip4), $addr ];
        ok(run_script($cmd), '--skip-artnum -V1');
        $err = '';
        ok(run_script([qw(-mda --no-precheck)], $env, $rdr), 'deliver V1');
@@ -214,6 +224,14 @@ SKIP: {
                        "$tmpdir/skip4/public-inbox/msgmap.sqlite3");
        $n = $mm->num_for($mid);
        is($n, 13, 'V1 NNTP article numbers skipped via --skip-artnum');
+       $check_wal->("$tmpdir/skip4");
+
+       # n.b. for now, indexlevel defaults to `full' if unspecified w/
+       # --wal or --skip-artnum
+       $cmd = [ qw(-init -V1 --wal wal-only-v1), "$tmpdir/wal-1",
+                       qw(http://example.com/wal-1 wal@example.com) ];
+       ok run_script($cmd), '--wal with -V1';
+       $check_wal->("$tmpdir/wal-1");
 }
 
 {