msgmap: take Getopt::Long options hashref for read-write DBs

Another step towards making it easier to support more SQLite and
Xapian-specific options on the CLI for tuning.

diff --git a/lib/PublicInbox/AltId.pm b/lib/PublicInbox/AltId.pm
index 76dc23e6ca76fa4d27e9eb0f581aea65fd50beed..84d4d4e9da4a4496edab08bdf5919162de95dcca 100644 (file)
--- a/lib/PublicInbox/AltId.pm
+++ b/lib/PublicInbox/AltId.pm
@@ -42,7 +42,9 @@ sub mm_alt ($) {
        my ($self) = @_;
        $self->{mm_alt} ||= eval {
                require PublicInbox::Msgmap;
-               PublicInbox::Msgmap->new_file(@$self{qw(filename writable)});
+               # TODO: expose a way to disable fsync + enable WAL
+               my $opt = $self->{writable} ? { fsync => 1 } : undef;
+               PublicInbox::Msgmap->new_file($self->{filename}, $opt);
        };
 }
 

diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm
index e64848e3d561e93494cb5a1c5777241d33242080..538e32b2431fc05cf3e982494b3cc2c2cb09e6b1 100644 (file)
--- a/lib/PublicInbox/InboxWritable.pm
+++ b/lib/PublicInbox/InboxWritable.pm
@@ -39,7 +39,7 @@ sub _init_v1 {
                my $sidx = PublicInbox::SearchIdx->new($self, $opt);
                $sidx->{oidx}->{journal_mode} = 'wal' if $opt->{wal};
                $sidx->begin_txn_lazy;
-               my $mm = PublicInbox::Msgmap->new_file($self, 1, $opt);
+               my $mm = PublicInbox::Msgmap->new_file($self, $opt);
                if (defined $skip_artnum) {
                        $mm->{dbh}->begin_work;
                        $mm->skip_artnum($skip_artnum);

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 0dacdb848a1f9754281d05fa90a688cc428f6251..d7eb246731bce8e8226f8127efd6e69ac04ccde9 100644 (file)
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -16,8 +16,9 @@ use PublicInbox::Over;
 use Scalar::Util qw(blessed);
 
 sub new_file {
-       my ($class, $ibx, $rw, $opt) = @_;
+       my ($class, $ibx, $opt) = @_;
        my $f;
+       my $rw = !!$opt;
        if (blessed($ibx)) {
                $f = $ibx->mm_file;
                $rw = 2 if $rw && $ibx->{-no_fsync};
@@ -56,7 +57,7 @@ sub tmp_clone {
        require PublicInbox::Syscall;
        PublicInbox::Syscall::nodatacow_fh($fh);
        $self->{dbh}->sqlite_backup_to_file($fn);
-       $tmp = ref($self)->new_file($fn, 2);
+       $tmp = ref($self)->new_file($fn, {});
        $tmp->{dbh}->do('PRAGMA journal_mode = MEMORY');
        $tmp->{pid} = $$;
        $tmp;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index e2fe19c79c0d6f53c5c84dfb93b64b8dfcac0a57..046e123902bad46a5be8b6b0d5f8a463d52143dc 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -92,7 +92,8 @@ sub new {
        } else {
                die "unsupported inbox version=$version\n";
        }
-       $self->{creat} = !!$creat_opt;
+       $self->{creat} = !!$creat_opt; # TODO: eliminate
+       $self->{-opt} = $creat_opt;
        $self;
 }
 
@@ -546,7 +547,7 @@ sub v1_mm_init ($) {
        die "BUG: v1_mm_init is only for v1\n" if $self->{ibx}->version != 1;
        $self->{mm} //= do {
                require PublicInbox::Msgmap;
-               PublicInbox::Msgmap->new_file($self->{ibx}, 1, $self->{-opt});
+               PublicInbox::Msgmap->new_file($self->{ibx}, $self->{-opt});
        };
 }
 

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 17cf2efa8f6e4b052468bdd03b315208baf69f14..2f869817b781512c62ffebb98e5d5aaae6b32383 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -246,7 +246,7 @@ sub _idx_init { # with_umask callback
 
        # Now that all subprocesses are up, we can open the FDs
        # for SQLite:
-       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1, $opt);
+       my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, $opt);
        $mm->{dbh}->begin_work;
 }
 

diff --git a/scripts/xhdr-num2mid b/scripts/xhdr-num2mid
index 3ca33f5d8fc072045a10b0ded774be1f3a7da7b3..f937a3cfd0c2b90f590f9f4f041ca9dd74b2b3fc 100755 (executable)
--- a/scripts/xhdr-num2mid
+++ b/scripts/xhdr-num2mid
@@ -1,5 +1,5 @@
 #!/usr/bin/perl -w
-# Copyright (C) 2016-2021 all contributors <meta@public-inbox.org>
+# Copyright (C) all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 # Useful for mapping article IDs from existing NNTP servers to MIDs
 use strict;
@@ -14,7 +14,7 @@ GetOptions(%opts) or die "bad command-line args\n$usage";
 if ($msgmap) {
        require PublicInbox::Msgmap;
        require PublicInbox::MID; # mid_clean
-       $mm = PublicInbox::Msgmap->new_file($msgmap, 1);
+       $mm = PublicInbox::Msgmap->new_file($msgmap, {});
 }
 
 my $group = shift or die $usage;

diff --git a/t/altid.t b/t/altid.t
index b09236d50edadb32b402a67ea9d3839ac9551d50..1ab3791bedac30720a846c8bf991f235f37cfcea 100644 (file)
--- a/t/altid.t
+++ b/t/altid.t
@@ -15,7 +15,7 @@ my $altid = [ "serial:gmane:file=$alt_file" ];
 my $ibx;
 
 {
-       my $mm = PublicInbox::Msgmap->new_file($alt_file, 2);
+       my $mm = PublicInbox::Msgmap->new_file($alt_file, { wal => 1 });
        is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set once OK');
        ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent');
        ok(0 == $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID');
@@ -48,7 +48,7 @@ EOF
 };
 
 {
-       my $mm = PublicInbox::Msgmap->new_file($alt_file, 2);
+       my $mm = PublicInbox::Msgmap->new_file($alt_file, { wal => 1 });
        my ($min, $max) = $mm->minmax;
        my $num = $mm->mid_insert('b@example.com');
        ok($num > $max, 'auto-increment goes beyond mid_set');

diff --git a/t/altid_v2.t b/t/altid_v2.t
index 6bc9045366261e01c99a75570d4bc24c02cdcc99..d4018ae89aa2060587f17b935675f80960e3d299 100644 (file)
--- a/t/altid_v2.t
+++ b/t/altid_v2.t
@@ -13,7 +13,8 @@ my $altid = [ "serial:gmane:file=$another" ];
 my $ibx = create_inbox 'v2', version => 2, indexlevel => 'medium',
                        altid => $altid, sub {
        my ($im, $ibx) = @_;
-       my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2);
+       my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another",
+                                               { wal => 1 });
        is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set') or xbail 'once';
        is($mm->mid_set(1234, 'a@example.com')+0, 0, 'mid_set not idempotent');
        is($mm->mid_set(1, 'a@example.com')+0, 0, 'mid_set fails with dup MID');
@@ -26,7 +27,8 @@ Message-ID: <a@example.com>
 hello world gmane:666
 EOF
 };
-my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2);
+my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another",
+                                       { wal => 1 });
 is($mm->mid_set(1234, 'a@example.com') + 0, 0, 'mid_set not idempotent');
 is($mm->mid_set(1, 'a@example.com') + 0, 0, 'mid_set fails with dup MID');
 my $mset = $ibx->search->mset('gmane:1234');

diff --git a/t/extsearch.t b/t/extsearch.t
index 553ff4056aedb936550091a0e78377ecf1f28442..e97a735f65d674a7e9b332623c4d242fcbcd19fb 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -634,7 +634,8 @@ if ('per-inbox altid w/ extindex') {
                                altid => $altid, sub {
                my ($im, $ibx) = @_;
                my $mm = PublicInbox::Msgmap->new_file(
-                                       "$ibx->{inboxdir}/$another", 2);
+                                       "$ibx->{inboxdir}/$another",
+                                       { wal => 1 });
                $mm->mid_set(1234, 'a@example.com') == 1 or xbail 'mid_set';
                $im->add(PublicInbox::Eml->new(<<'EOF')) or BAIL_OUT;
 From: a@example.com

diff --git a/t/msgmap.t b/t/msgmap.t
index 6ed01e76d2f147b2686c5322f03b76322094cd91..e40a6b407165efac0171f3416c369aef677be85b 100644 (file)
--- a/t/msgmap.t
+++ b/t/msgmap.t
@@ -10,7 +10,9 @@ require_mods('DBD::SQLite');
 use_ok 'PublicInbox::Msgmap';
 my ($tmpdir, $for_destroy) = tmpdir();
 my $f = "$tmpdir/msgmap.sqlite3";
-my $d = PublicInbox::Msgmap->new_file($f, 1);
+my $d = PublicInbox::Msgmap->new_file($f, { wal => 1 });
+is $d->{dbh}->selectrow_array('PRAGMA journal_mode'), 'wal',
+       'wal in options hashref respected';
 
 my %mid2num;
 my %num2mid;
@@ -53,7 +55,7 @@ is($d->mid_delete('a@b') + 0, 0, 'delete again returns zero');
 is(undef, $d->num_for('a@b'), 'num_for fails on deleted msg');
 $d = undef;
 
-ok($d = PublicInbox::Msgmap->new_file($f, 1), 'idempotent DB creation');
+ok($d = PublicInbox::Msgmap->new_file($f, {}), 'idempotent DB creation');
 my ($min, $max) = $d->minmax;
 ok($min > 0, "article min OK");
 ok($max > 0 && $max < 10, "article max OK");