From d4ba2361216f2f014d9028a30f500ef01f32d450 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Tue, 26 Aug 2025 19:50:46 +0000 Subject: [PATCH] msgmap: take Getopt::Long options hashref for read-write DBs Another step towards making it easier to support more SQLite and Xapian-specific options on the CLI for tuning. --- lib/PublicInbox/AltId.pm | 4 +++- lib/PublicInbox/InboxWritable.pm | 2 +- lib/PublicInbox/Msgmap.pm | 5 +++-- lib/PublicInbox/SearchIdx.pm | 5 +++-- lib/PublicInbox/V2Writable.pm | 2 +- scripts/xhdr-num2mid | 4 ++-- t/altid.t | 4 ++-- t/altid_v2.t | 6 ++++-- t/extsearch.t | 3 ++- t/msgmap.t | 6 ++++-- 10 files changed, 25 insertions(+), 16 deletions(-) diff --git a/lib/PublicInbox/AltId.pm b/lib/PublicInbox/AltId.pm index 76dc23e6c..84d4d4e9d 100644 --- a/lib/PublicInbox/AltId.pm +++ b/lib/PublicInbox/AltId.pm @@ -42,7 +42,9 @@ sub mm_alt ($) { my ($self) = @_; $self->{mm_alt} ||= eval { require PublicInbox::Msgmap; - PublicInbox::Msgmap->new_file(@$self{qw(filename writable)}); + # TODO: expose a way to disable fsync + enable WAL + my $opt = $self->{writable} ? { fsync => 1 } : undef; + PublicInbox::Msgmap->new_file($self->{filename}, $opt); }; } diff --git a/lib/PublicInbox/InboxWritable.pm b/lib/PublicInbox/InboxWritable.pm index e64848e3d..538e32b24 100644 --- a/lib/PublicInbox/InboxWritable.pm +++ b/lib/PublicInbox/InboxWritable.pm @@ -39,7 +39,7 @@ sub _init_v1 { my $sidx = PublicInbox::SearchIdx->new($self, $opt); $sidx->{oidx}->{journal_mode} = 'wal' if $opt->{wal}; $sidx->begin_txn_lazy; - my $mm = PublicInbox::Msgmap->new_file($self, 1, $opt); + my $mm = PublicInbox::Msgmap->new_file($self, $opt); if (defined $skip_artnum) { $mm->{dbh}->begin_work; $mm->skip_artnum($skip_artnum); diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm index 0dacdb848..d7eb24673 100644 --- a/lib/PublicInbox/Msgmap.pm +++ b/lib/PublicInbox/Msgmap.pm @@ -16,8 +16,9 @@ use PublicInbox::Over; use Scalar::Util qw(blessed); sub new_file { - my ($class, $ibx, $rw, $opt) = @_; + my ($class, $ibx, $opt) = @_; my $f; + my $rw = !!$opt; if (blessed($ibx)) { $f = $ibx->mm_file; $rw = 2 if $rw && $ibx->{-no_fsync}; @@ -56,7 +57,7 @@ sub tmp_clone { require PublicInbox::Syscall; PublicInbox::Syscall::nodatacow_fh($fh); $self->{dbh}->sqlite_backup_to_file($fn); - $tmp = ref($self)->new_file($fn, 2); + $tmp = ref($self)->new_file($fn, {}); $tmp->{dbh}->do('PRAGMA journal_mode = MEMORY'); $tmp->{pid} = $$; $tmp; diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm index e2fe19c79..046e12390 100644 --- a/lib/PublicInbox/SearchIdx.pm +++ b/lib/PublicInbox/SearchIdx.pm @@ -92,7 +92,8 @@ sub new { } else { die "unsupported inbox version=$version\n"; } - $self->{creat} = !!$creat_opt; + $self->{creat} = !!$creat_opt; # TODO: eliminate + $self->{-opt} = $creat_opt; $self; } @@ -546,7 +547,7 @@ sub v1_mm_init ($) { die "BUG: v1_mm_init is only for v1\n" if $self->{ibx}->version != 1; $self->{mm} //= do { require PublicInbox::Msgmap; - PublicInbox::Msgmap->new_file($self->{ibx}, 1, $self->{-opt}); + PublicInbox::Msgmap->new_file($self->{ibx}, $self->{-opt}); }; } diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm index 17cf2efa8..2f869817b 100644 --- a/lib/PublicInbox/V2Writable.pm +++ b/lib/PublicInbox/V2Writable.pm @@ -246,7 +246,7 @@ sub _idx_init { # with_umask callback # Now that all subprocesses are up, we can open the FDs # for SQLite: - my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, 1, $opt); + my $mm = $self->{mm} = PublicInbox::Msgmap->new_file($ibx, $opt); $mm->{dbh}->begin_work; } diff --git a/scripts/xhdr-num2mid b/scripts/xhdr-num2mid index 3ca33f5d8..f937a3cfd 100755 --- a/scripts/xhdr-num2mid +++ b/scripts/xhdr-num2mid @@ -1,5 +1,5 @@ #!/usr/bin/perl -w -# Copyright (C) 2016-2021 all contributors +# Copyright (C) all contributors # License: AGPL-3.0+ # Useful for mapping article IDs from existing NNTP servers to MIDs use strict; @@ -14,7 +14,7 @@ GetOptions(%opts) or die "bad command-line args\n$usage"; if ($msgmap) { require PublicInbox::Msgmap; require PublicInbox::MID; # mid_clean - $mm = PublicInbox::Msgmap->new_file($msgmap, 1); + $mm = PublicInbox::Msgmap->new_file($msgmap, {}); } my $group = shift or die $usage; diff --git a/t/altid.t b/t/altid.t index b09236d50..1ab3791be 100644 --- a/t/altid.t +++ b/t/altid.t @@ -15,7 +15,7 @@ my $altid = [ "serial:gmane:file=$alt_file" ]; my $ibx; { - my $mm = PublicInbox::Msgmap->new_file($alt_file, 2); + my $mm = PublicInbox::Msgmap->new_file($alt_file, { wal => 1 }); is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set once OK'); ok(0 == $mm->mid_set(1234, 'a@example.com'), 'mid_set not idempotent'); ok(0 == $mm->mid_set(1, 'a@example.com'), 'mid_set fails with dup MID'); @@ -48,7 +48,7 @@ EOF }; { - my $mm = PublicInbox::Msgmap->new_file($alt_file, 2); + my $mm = PublicInbox::Msgmap->new_file($alt_file, { wal => 1 }); my ($min, $max) = $mm->minmax; my $num = $mm->mid_insert('b@example.com'); ok($num > $max, 'auto-increment goes beyond mid_set'); diff --git a/t/altid_v2.t b/t/altid_v2.t index 6bc904536..d4018ae89 100644 --- a/t/altid_v2.t +++ b/t/altid_v2.t @@ -13,7 +13,8 @@ my $altid = [ "serial:gmane:file=$another" ]; my $ibx = create_inbox 'v2', version => 2, indexlevel => 'medium', altid => $altid, sub { my ($im, $ibx) = @_; - my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2); + my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", + { wal => 1 }); is($mm->mid_set(1234, 'a@example.com'), 1, 'mid_set') or xbail 'once'; is($mm->mid_set(1234, 'a@example.com')+0, 0, 'mid_set not idempotent'); is($mm->mid_set(1, 'a@example.com')+0, 0, 'mid_set fails with dup MID'); @@ -26,7 +27,8 @@ Message-ID: hello world gmane:666 EOF }; -my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", 2); +my $mm = PublicInbox::Msgmap->new_file("$ibx->{inboxdir}/$another", + { wal => 1 }); is($mm->mid_set(1234, 'a@example.com') + 0, 0, 'mid_set not idempotent'); is($mm->mid_set(1, 'a@example.com') + 0, 0, 'mid_set fails with dup MID'); my $mset = $ibx->search->mset('gmane:1234'); diff --git a/t/extsearch.t b/t/extsearch.t index 553ff4056..e97a735f6 100644 --- a/t/extsearch.t +++ b/t/extsearch.t @@ -634,7 +634,8 @@ if ('per-inbox altid w/ extindex') { altid => $altid, sub { my ($im, $ibx) = @_; my $mm = PublicInbox::Msgmap->new_file( - "$ibx->{inboxdir}/$another", 2); + "$ibx->{inboxdir}/$another", + { wal => 1 }); $mm->mid_set(1234, 'a@example.com') == 1 or xbail 'mid_set'; $im->add(PublicInbox::Eml->new(<<'EOF')) or BAIL_OUT; From: a@example.com diff --git a/t/msgmap.t b/t/msgmap.t index 6ed01e76d..e40a6b407 100644 --- a/t/msgmap.t +++ b/t/msgmap.t @@ -10,7 +10,9 @@ require_mods('DBD::SQLite'); use_ok 'PublicInbox::Msgmap'; my ($tmpdir, $for_destroy) = tmpdir(); my $f = "$tmpdir/msgmap.sqlite3"; -my $d = PublicInbox::Msgmap->new_file($f, 1); +my $d = PublicInbox::Msgmap->new_file($f, { wal => 1 }); +is $d->{dbh}->selectrow_array('PRAGMA journal_mode'), 'wal', + 'wal in options hashref respected'; my %mid2num; my %num2mid; @@ -53,7 +55,7 @@ is($d->mid_delete('a@b') + 0, 0, 'delete again returns zero'); is(undef, $d->num_for('a@b'), 'num_for fails on deleted msg'); $d = undef; -ok($d = PublicInbox::Msgmap->new_file($f, 1), 'idempotent DB creation'); +ok($d = PublicInbox::Msgmap->new_file($f, {}), 'idempotent DB creation'); my ($min, $max) = $d->minmax; ok($min > 0, "article min OK"); ok($max > 0 && $max < 10, "article max OK"); -- 2.47.3