use Getopt::Long hashref for --no-fsync and --dangerous

We can eliminate the {-no_fsync} and {-dangerous} fields by
using the Getopt::Long hashref directly.  This will make it
easier to support additional CLI options (e.g. --cow).

Our internal APIs now defaults to disabling fsync, however the
CLI tools still override that internal default to enable fsync.
Having our internals default to disabling fsync can slightly
improve test performance, since they're the main users of our
unstable internal API.

diff --git a/lib/PublicInbox/CodeSearchIdx.pm b/lib/PublicInbox/CodeSearchIdx.pm
index ddb5980301ae2b6dd8809d5bd0677b629a55d227..dbb0fe3a18a0abe078277b76dbe292d821ef7953 100644 (file)
--- a/lib/PublicInbox/CodeSearchIdx.pm
+++ b/lib/PublicInbox/CodeSearchIdx.pm
@@ -152,8 +152,6 @@ sub new {
        }, __PACKAGE__;
        $self->{nshard} = count_shards($self) ||
                nproc_shards({nproc => $opt->{jobs}});
-       $self->{-no_fsync} = 1 if !$opt->{fsync};
-       $self->{-dangerous} = 1 if $opt->{dangerous};
        $self;
 }
 

diff --git a/lib/PublicInbox/ExtSearchIdx.pm b/lib/PublicInbox/ExtSearchIdx.pm
index cb8a992aebd280d4960d21b18edb4c307d0748ff..911a900f5d96be383244d29d3b269e374e0bcb5d 100644 (file)
--- a/lib/PublicInbox/ExtSearchIdx.pm
+++ b/lib/PublicInbox/ExtSearchIdx.pm
@@ -94,8 +94,6 @@ sub new {
                die "invalid indexlevel=$l\n";
        $self->{indexlevel} = $l;
        $self->{oidx} = PublicInbox::OverIdx->new($over_file, $opt);
-       $self->{-no_fsync} = 1 if !$opt->{fsync};
-       $self->{-dangerous} = 1 if $opt->{dangerous};
        $self
 }
 

diff --git a/lib/PublicInbox/MiscIdx.pm b/lib/PublicInbox/MiscIdx.pm
index e2ed9d1118f2d2e89606e1c48a5a7b14d5df9310..257bd7a8399490c005400a7d6dc75706b7d68d6c 100644 (file)
--- a/lib/PublicInbox/MiscIdx.pm
+++ b/lib/PublicInbox/MiscIdx.pm
@@ -30,8 +30,9 @@ sub new {
        File::Path::mkpath($mi_dir);
        PublicInbox::Syscall::nodatacow_dir($mi_dir);
        my $flags = $PublicInbox::SearchIdx::DB_CREATE_OR_OPEN;
-       $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if $eidx->{-no_fsync};
-       $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $eidx->{-dangerous};
+       my $opt = $eidx->{-opt};
+       $flags |= $PublicInbox::SearchIdx::DB_NO_SYNC if !$opt->{fsync};
+       $flags |= $PublicInbox::SearchIdx::DB_DANGEROUS if $opt->{dangerous};
        $json //= PublicInbox::Config::json();
        bless {
                mi_dir => $mi_dir,

diff --git a/lib/PublicInbox/Msgmap.pm b/lib/PublicInbox/Msgmap.pm
index 9ef87d7893e148305e1d6273850aef69dc41f0ba..fffaa7b57e7b3ca602b48f4924df4b91e6d332ad 100644 (file)
--- a/lib/PublicInbox/Msgmap.pm
+++ b/lib/PublicInbox/Msgmap.pm
@@ -17,18 +17,12 @@ use Scalar::Util qw(blessed);
 
 sub new_file {
        my ($class, $ibx, $opt) = @_;
-       my $f;
        my $rw = !!$opt;
-       if (blessed($ibx)) {
-               $f = $ibx->mm_file;
-               $rw = 2 if $rw && $ibx->{-no_fsync};
-       } else {
-               $f = $ibx;
-       }
+       my $f = blessed($ibx) ? $ibx->mm_file : $ibx;
        return if !$rw && !-r $f;
 
        my $self = bless { filename => $f }, $class;
-       $self->{-opt} = $opt if $opt; # for WAL
+       $self->{-opt} = $opt if $opt; # for WAL and fsync
        my $dbh = $self->{dbh} = PublicInbox::Over::dbh_new($self, $rw);
        if ($rw) {
                $dbh->begin_work;

diff --git a/lib/PublicInbox/SearchIdx.pm b/lib/PublicInbox/SearchIdx.pm
index 6e05f95673a07c34eb1c3bd325fb84a8d6870489..c0056ee5fcecc798b3bded77c1d0c9b9a974ec73 100644 (file)
--- a/lib/PublicInbox/SearchIdx.pm
+++ b/lib/PublicInbox/SearchIdx.pm
@@ -153,11 +153,11 @@ sub idx_acquire {
                        PublicInbox::Syscall::nodatacow_dir($dir);
                        # owner == self for CodeSearchIdx
                        $self->{-set_has_threadid_once} = 1 if $owner != $self;
-                       $flag |= $DB_DANGEROUS if $owner->{-dangerous};
+                       $flag |= $DB_DANGEROUS if $self->{-opt}->{dangerous};
                }
        }
        return unless defined $flag;
-       $flag |= $DB_NO_SYNC if $owner->{-no_fsync};
+       $flag |= $DB_NO_SYNC if !$self->{-opt}->{fsync};
        my $xdb = eval { ($X->{WritableDatabase})->new($dir, $flag) };
        croak "Failed opening $dir: $@" if $@;
        $self->{xdb} = $xdb;
@@ -1196,6 +1196,7 @@ sub eidx_shard_new {
        my ($class, $eidx, $shard) = @_;
        my $self = bless {
                eidx => $eidx,
+               -opt => $eidx->{-opt}, # hmm...
                xpfx => $eidx->{xpfx},
                indexlevel => $eidx->{indexlevel},
                -skip_docdata => 1,

diff --git a/lib/PublicInbox/SearchIdxShard.pm b/lib/PublicInbox/SearchIdxShard.pm
index 108aaaeb71663b4d9a69d12403f739a0b3584414..8f340071f39381c688d3f6adae272db9d45b5510 100644 (file)
--- a/lib/PublicInbox/SearchIdxShard.pm
+++ b/lib/PublicInbox/SearchIdxShard.pm
@@ -12,7 +12,7 @@ use PublicInbox::Syscall qw($F_SETPIPE_SZ);
 sub new {
        my ($class, $v2w, $shard) = @_; # v2w may be ExtSearchIdx
        my $ibx = $v2w->{ibx};
-       my $self = $ibx ? $class->SUPER::new($ibx, 1, $shard)
+       my $self = $ibx ? $class->SUPER::new($ibx, $v2w->{-opt}, $shard)
                        : $class->eidx_shard_new($v2w, $shard);
        # create the DB before forking:
        $self->idx_acquire;

diff --git a/lib/PublicInbox/TestCommon.pm b/lib/PublicInbox/TestCommon.pm
index 33bd9fabac0a14d9fef99d35d7174a17e92b6104..7b173b61124a62f399386975f1deca1aec0e6036 100644 (file)
--- a/lib/PublicInbox/TestCommon.pm
+++ b/lib/PublicInbox/TestCommon.pm
@@ -857,7 +857,6 @@ sub setup_public_inboxes () {
        my $seen = 0;
        $cfg->each_inbox(sub {
                my ($ibx) = @_;
-               $ibx->{-no_fsync} = 1;
                my $im = PublicInbox::InboxWritable->new($ibx)->importer(0);
                my $V = $ibx->version;
                my @eml = (glob('t/*.eml'), 't/data/0001.patch');
@@ -944,7 +943,6 @@ sub create_inbox ($;@) {
        my $scope = $lk->lock_for_scope;
        my $pre_cb = delete $opt{pre_cb};
        $pre_cb->($dir) if $pre_cb && $new;
-       $opt{-no_fsync} = 1;
        my $no_gc = delete $opt{-no_gc};
        my $addr = $opt{address} // [];
        $opt{-primary_address} //= $addr->[0] // "$ident\@example.com";

diff --git a/lib/PublicInbox/V2Writable.pm b/lib/PublicInbox/V2Writable.pm
index 09e2777cef4b370915bcbd9c9670b30d76976eea..bd5d9d320a72d53fa1c69d7f3aa823f764a54a92 100644 (file)
--- a/lib/PublicInbox/V2Writable.pm
+++ b/lib/PublicInbox/V2Writable.pm
@@ -70,8 +70,8 @@ sub new {
                # limit each git repo (epoch) to 1GB or so
                rotate_bytes => int((1024 * 1024 * 1024) / $PACKING_FACTOR),
                last_commit => [], # git epoch -> commit
+               -opt => $creat_opt // {},
        };
-       $self->{oidx}->{-no_fsync} = 1 if $v2ibx->{-no_fsync};
        $self->{shards} = count_shards($self) || nproc_shards($creat_opt);
        bless $self, $class;
 }

diff --git a/script/public-inbox-convert b/script/public-inbox-convert
index cf87b4ebb3c7e5c0a56907650f72e16a43dbb636..4ca8ecf2856ae4c11aa2fcc13b998a526f2a2925 100755 (executable)
--- a/script/public-inbox-convert
+++ b/script/public-inbox-convert
@@ -84,8 +84,8 @@ $new->{inboxdir} = PublicInbox::Config::rel2abs_collapsed($new_dir);
 $new->{version} = 2;
 my $creat_opt = { nproc => $opt->{jobs} };
 $creat_opt->{wal} = 1 if $opt->{wal};
+$creat_opt->{fsync} = $opt->{fsync};
 $new = PublicInbox::InboxWritable->new($new, $creat_opt);
-$new->{-no_fsync} = 1 if !$opt->{fsync};
 my $v2w;
 
 sub link_or_copy ($$) {

diff --git a/script/public-inbox-index b/script/public-inbox-index
index b4965ec9537a768d526b6a3f6268753901fa894a..acdec3a9c85c572b6d85d245b8c93cd12d04ab3c 100755 (executable)
--- a/script/public-inbox-index
+++ b/script/public-inbox-index
@@ -122,8 +122,6 @@ for my $ibx (@ibxs) {
        if ($opt->{compact} >= 2) {
                PublicInbox::Xapcmd::run($ibx, 'compact', $opt->{compact_opt});
        }
-       $ibx->{-no_fsync} = 1 if !$opt->{fsync};
-       $ibx->{-dangerous} = 1 if $opt->{dangerous};
        $ibx->{-skip_docdata} //= $opt->{'skip-docdata'};
 
        my $ibx_opt = $opt;

diff --git a/script/public-inbox-purge b/script/public-inbox-purge
index 0100cf48afa9ed2502900703839fe9a7e62cb11e..7649950eaf8b411849f7a6851e3dee523f114d70 100755 (executable)
--- a/script/public-inbox-purge
+++ b/script/public-inbox-purge
@@ -25,7 +25,7 @@ options:
 See public-inbox-purge(1) man page for full documentation.
 EOF
 
-my $opt = { verbose => 1, all => 0, -min_inbox_version => 2 };
+my $opt = { verbose => 1, all => 0, -min_inbox_version => 2, fsync => 1 };
 GetOptions($opt, @PublicInbox::AdminEdit::OPT, 'C=s@') or die $help;
 if ($opt->{help}) { print $help; exit 0 };
 
@@ -39,7 +39,7 @@ my $n_purged = 0;
 
 foreach my $ibx (@ibxs) {
        my $eml = PublicInbox::Eml->new($data);
-       my $v2w = PublicInbox::V2Writable->new($ibx);
+       my $v2w = PublicInbox::V2Writable->new($ibx, $opt);
 
        my $commits = $v2w->purge($eml) || [];
 

diff --git a/scripts/import_slrnspool b/scripts/import_slrnspool
index 81df6c2ef32545a5d29090b99c91e32b0386ad55..7b0dbb3653e2864b72af210a7943d656f46aa9b6 100755 (executable)
--- a/scripts/import_slrnspool
+++ b/scripts/import_slrnspool
@@ -38,7 +38,7 @@ my $git = $ibx->git;
 my $im;
 if ($ibx->version == 2) {
        require PublicInbox::V2Writable;
-       $im = PublicInbox::V2Writable->new($ibx);
+       $im = PublicInbox::V2Writable->new($ibx, { fsync => 1 });
        $im->{parallel} = 0; # pointless to be parallel for a single message
 } else {
        $im = PublicInbox::Import->new($git, $ibx->{name},

diff --git a/t/extsearch.t b/t/extsearch.t
index e97a735f65d674a7e9b332623c4d242fcbcd19fb..5c9b6dbe0ab2c5e496a002b04b33d95572a04982 100644 (file)
--- a/t/extsearch.t
+++ b/t/extsearch.t
@@ -266,7 +266,6 @@ if ('inject w/o indexing') {
 
 if ('reindex catches missed messages') {
        my $v2ibx = $cfg->lookup_name('v2test');
-       $v2ibx->{-no_fsync} = 1;
        my $im = PublicInbox::InboxWritable->new($v2ibx)->importer(0);
        my $cmt_a = $v2ibx->mm->last_commit_xap($schema_version, 0);
        my $eml = eml_load('t/data/0001.patch');
@@ -337,7 +336,6 @@ if ('reindex catches missed messages') {
 if ('reindex catches content bifurcation') {
        use PublicInbox::MID qw(mids);
        my $v2ibx = $cfg->lookup_name('v2test');
-       $v2ibx->{-no_fsync} = 1;
        my $im = PublicInbox::InboxWritable->new($v2ibx)->importer(0);
        my $eml = eml_load('t/data/message_embed.eml');
        my $cmt_a = $v2ibx->mm->last_commit_xap($schema_version, 0);

diff --git a/t/miscsearch.t b/t/miscsearch.t
index ec837153d7a8828b49ebb9764c63a3a095d1aab9..a95cfe9199ba232af660ec6ae45774dcec8aebcf 100644 (file)
--- a/t/miscsearch.t
+++ b/t/miscsearch.t
@@ -10,7 +10,7 @@ use_ok 'PublicInbox::MiscSearch';
 use_ok 'PublicInbox::MiscIdx';
 
 my ($tmp, $for_destroy) = tmpdir();
-my $eidx = { xpfx => "$tmp/eidx", -no_fsync => 1 }; # mock ExtSearchIdx
+my $eidx = { xpfx => "$tmp/eidx" }; # mock ExtSearchIdx
 my $v1 = create_inbox 'hope', address => [ 'nope@example.com' ],
                        indexlevel => 'basic', -no_gc => 1, sub {
        my ($im, $ibx) = @_;

diff --git a/t/purge.t b/t/purge.t
index a33cd3290fc6ca7b000a1bac0e8e5ec9a4b08e58..f4281c1397bd5d87d2dbdef7cdc7644a1d15b9d8 100644 (file)
--- a/t/purge.t
+++ b/t/purge.t
@@ -16,7 +16,6 @@ my $ibx = PublicInbox::Inbox->new({
        inboxdir => $inboxdir,
        name => 'test-v2purge',
        version => 2,
-       -no_fsync => 1,
        -primary_address => 'test@example.com',
        indexlevel => 'basic',
 });

diff --git a/t/replace.t b/t/replace.t
index a61c3ca0656d0b494278cf953ff1c939e398b08e..b733c556239b4a15616f34a27ecf60151de6eccb 100644 (file)
--- a/t/replace.t
+++ b/t/replace.t
@@ -20,7 +20,6 @@ sub test_replace ($$$) {
                inboxdir => "$tmpdir/testbox",
                name => $this,
                version => $v,
-               -no_fsync => 1,
                -primary_address => 'test@example.com',
                indexlevel => $level,
        });

diff --git a/t/v1reindex.t b/t/v1reindex.t
index 5c00ca91935c3f544e765a288024e4f532d09435..3643319402c14678c7ea21b11a183e78990862ad 100644 (file)
--- a/t/v1reindex.t
+++ b/t/v1reindex.t
@@ -18,7 +18,6 @@ my $ibx_config = {
        name => 'test-v1reindex',
        -primary_address => 'test@example.com',
        indexlevel => 'full',
-       -no_fsync => 1,
 };
 my $mime = PublicInbox::Eml->new(<<'EOF');
 From: a@example.com

diff --git a/t/v2-add-remove-add.t b/t/v2-add-remove-add.t
index 070b1ac1e57a0094803ce505d5c181ef2195f0db..f8ca2c210330598e288c30f1a0c448938abe660c 100644 (file)
--- a/t/v2-add-remove-add.t
+++ b/t/v2-add-remove-add.t
@@ -13,7 +13,6 @@ my $ibx = {
        inboxdir => "$inboxdir/v2",
        name => 'test-v2writable',
        version => 2,
-       -no_fsync => 1,
        -primary_address => 'test@example.com',
 };
 $ibx = PublicInbox::Inbox->new($ibx);

diff --git a/t/v2mirror.t b/t/v2mirror.t
index 9d8ba6277ae56a541c382a7184a7e4e1fb1ab2c6..8cda0a0580643d730f73bc1c2503f48772c4b4d3 100644 (file)
--- a/t/v2mirror.t
+++ b/t/v2mirror.t
@@ -37,7 +37,6 @@ my $cfg = PublicInbox::Config->new($pi_config);
 my $ibx = $cfg->lookup('test@example.com');
 ok($ibx, 'inbox found');
 $ibx->{version} = 2;
-$ibx->{-no_fsync} = 1;
 my $v2w = PublicInbox::V2Writable->new($ibx, { nproc => 1 });
 ok $v2w, 'v2w loaded';
 $v2w->{parallel} = 0;

diff --git a/t/v2reindex.t b/t/v2reindex.t
index 0e018481e4128bc1cc84c862067393305223951e..f830ae9b3460cf63c7d86a3e83377870f75241af 100644 (file)
--- a/t/v2reindex.t
+++ b/t/v2reindex.t
@@ -15,7 +15,6 @@ my $ibx_config = {
        version => 2,
        -primary_address => 'test@example.com',
        indexlevel => 'full',
-       -no_fsync => 1,
 };
 my $agpl = do {
        open my $fh, '<', 'COPYING' or die "can't open COPYING: $!";

diff --git a/t/v2writable.t b/t/v2writable.t
index 242088f9601895d4b8f581b9c2a3a84ebab5ad24..a1593d95e21eaaf2ac01621cc78cbadd9ba1618d 100644 (file)
--- a/t/v2writable.t
+++ b/t/v2writable.t
@@ -19,7 +19,6 @@ my $ibx = {
        inboxdir => $inboxdir,
        name => 'test-v2writable',
        version => 2,
-       -no_fsync => 1,
        -primary_address => 'test@example.com',
 };
 $ibx = PublicInbox::Inbox->new($ibx);

diff --git a/t/watch_filter_rubylang.t b/t/watch_filter_rubylang.t
index 81f6b00e1dc1a27cd00b8cd0d19888ffef433a05..dee208849c2c1a268d3705285320e81e2d3631eb 100644 (file)
--- a/t/watch_filter_rubylang.t
+++ b/t/watch_filter_rubylang.t
@@ -67,7 +67,6 @@ EOF
        watchspam = maildir:$spamdir
 EOM
        my $ibx = $cfg->lookup_name($v);
-       $ibx->{-no_fsync} = 1;
        ok($ibx, 'found inbox by name');
 
        my $w = PublicInbox::Watch->new($cfg);
@@ -100,7 +99,6 @@ EOM
 
        $cfg = PublicInbox::Config->new($cfg->{-f});
        $ibx = $cfg->lookup_name($v);
-       $ibx->{-no_fsync} = 1;
        is($ibx->search->reopen->mset('b:spam')->size, 0, 'spam removed');
 
        is_deeply [], [ grep !/^#/, @warn ], 'no warnings';

diff --git a/t/watch_maildir_v2.t b/t/watch_maildir_v2.t
index fa86f7bf80362b88281573476e21dbd85ee10bef..bdc73bf38893841a4ef25e8f6804362dfb2d026b 100644 (file)
--- a/t/watch_maildir_v2.t
+++ b/t/watch_maildir_v2.t
@@ -47,7 +47,6 @@ EOF
 my $cfg = cfg_new $tmpdir, $orig;
 my $ibx = $cfg->lookup_name('test');
 ok($ibx, 'found inbox by name');
-$ibx->{-no_fsync} = 1;
 
 PublicInbox::Watch->new($cfg)->scan('full');
 my $total = scalar @{$ibx->over->recent};

diff --git a/xt/create-many-inboxes.t b/xt/create-many-inboxes.t
index 3d8932b7e602c84cd2123908a0557ba594455231..5cd9f448b6e8bedb98b50e9a8a99d619d55ad65a 100644 (file)
--- a/xt/create-many-inboxes.t
+++ b/xt/create-many-inboxes.t
@@ -40,7 +40,6 @@ my $v2_init_add = sub {
                address => [ "test-$i\@example.com" ],
                url => [ "//example.com/test-$i" ],
                version => 2,
-               -no_fsync => 1,
        });
        $ibx->{indexlevel} = $indexlevel if $level_cfg ne '';
        my $entry = <<EOF;