lei: MH: support inotify to detect updates

This should help us deal with MH sequence number packing and
invalidating mail_sync.sqlite3.

diff --git a/lib/PublicInbox/LEI.pm b/lib/PublicInbox/LEI.pm
index e0cfd55ad8e5cb38b976ac5bfd711bb286b40b33..81f940fe13adb58c0b5f02c63dd4103edbe64237 100644 (file)
--- a/lib/PublicInbox/LEI.pm
+++ b/lib/PublicInbox/LEI.pm
@@ -28,7 +28,7 @@ use PublicInbox::IPC;
 use Time::HiRes qw(stat); # ctime comparisons for config cache
 use File::Path ();
 use File::Spec;
-use Carp ();
+use Carp qw(carp);
 use Sys::Syslog qw(openlog syslog closelog);
 our $quit = \&CORE::exit;
 our ($current_lei, $errors_log, $listener, $oldset, $dir_idle);
@@ -38,7 +38,7 @@ my $GLP_PASS = Getopt::Long::Parser->new;
 $GLP_PASS->configure(qw(gnu_getopt no_ignore_case auto_abbrev pass_through));
 
 our (%PATH2CFG, # persistent for socket daemon
-$MDIR2CFGPATH, # /path/to/maildir => { /path/to/config => [ ino watches ] }
+$MDIR2CFGPATH, # location => { /path/to/config => [ ino watches ] }
 $OPT, # shared between optparse and opt_dash callback (for Getopt::Long)
 $daemon_pid
 );
@@ -606,7 +606,7 @@ sub _lei_atfork_child {
        $dir_idle->force_close if $dir_idle;
        undef $dir_idle;
        %PATH2CFG = ();
-       $MDIR2CFGPATH = {};
+       $MDIR2CFGPATH = undef;
        eval 'no warnings; undef $PublicInbox::LeiNoteEvent::to_flush';
        undef $errors_log;
        $quit = \&CORE::exit;
@@ -1252,32 +1252,43 @@ sub cfg2lei ($) {
        $lei;
 }
 
+sub note_event ($@) { # runs lei_note_event for a given config file
+       my ($cfg_f, @args) = @_;
+       my $cfg = $PATH2CFG{$cfg_f} // return;
+       eval { cfg2lei($cfg)->dispatch('note-event', @args) };
+       carp "E: note-event $cfg_f: $@\n" if $@;
+}
+
 sub dir_idle_handler ($) { # PublicInbox::DirIdle callback
        my ($ev) = @_; # Linux::Inotify2::Event or duck type
        my $fn = $ev->fullname;
        if ($fn =~ m!\A(.+)/(new|cur)/([^/]+)\z!) { # Maildir file
-               my ($mdir, $nc, $bn) = ($1, $2, $3);
-               $nc = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
-               for my $f (keys %{$MDIR2CFGPATH->{$mdir} // {}}) {
-                       my $cfg = $PATH2CFG{$f} // next;
-                       eval {
-                               my $lei = cfg2lei($cfg);
-                               $lei->dispatch('note-event',
-                                               "maildir:$mdir", $nc, $bn, $fn);
-                       };
-                       warn "E: note-event $f: $@\n" if $@;
+               my ($loc, $new_cur, $bn) = ("maildir:$1", $2, $3);
+               $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
+               for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                       note_event($cfg_f, $loc, $new_cur, $bn, $fn);
                }
-       }
+       } elsif ($fn =~ m!\A(.+)/([0-9]+)\z!) { # MH mail message file
+               my ($loc, $n, $new_cur) = ("mh:$1", $2, '+');
+               $new_cur = '' if $ev->IN_DELETE || $ev->IN_MOVED_FROM;
+               for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                       note_event($cfg_f, $loc, $new_cur, $n, $fn);
+               }
+       } elsif ($fn =~ m!\A(.+)/\.mh_sequences\z!) { # reread flags
+               my $loc = "mh:$1";
+               for my $cfg_f (keys %{$MDIR2CFGPATH->{$loc} // {}}) {
+                       note_event($cfg_f, $loc, '.mh_sequences')
+               }
+       } # else we don't care
        if ($ev->can('cancel') && ($ev->IN_IGNORE || $ev->IN_UNMOUNT)) {
                $ev->cancel;
        }
        if ($fn =~ m!\A(.+)/(?:new|cur)\z! && !-e $fn) {
-               delete $MDIR2CFGPATH->{$1};
+               delete $MDIR2CFGPATH->{"maildir:$1"};
        }
-       if (!-e $fn) { # config file or Maildir gone
-               for my $cfgpaths (values %$MDIR2CFGPATH) {
-                       delete $cfgpaths->{$fn};
-               }
+       if (!-e $fn) { # config file, Maildir, or MH dir gone
+               delete $_->{$fn} for values %$MDIR2CFGPATH; # config file
+               delete @$MDIR2CFGPATH{"maildir:$fn", "mh:$fn"};
                delete $PATH2CFG{$fn};
        }
 }
@@ -1442,19 +1453,22 @@ sub watch_state_ok ($) {
        $state =~ /\Apause|(?:import|index|tag)-(?:ro|rw)\z/;
 }
 
-sub cancel_maildir_watch ($$) {
-       my ($d, $cfg_f) = @_;
-       my $w = delete $MDIR2CFGPATH->{$d}->{$cfg_f};
-       scalar(keys %{$MDIR2CFGPATH->{$d}}) or
-               delete $MDIR2CFGPATH->{$d};
-       for my $x (@{$w // []}) { $x->cancel }
+sub cancel_dir_watch ($$$) {
+       my ($type, $d, $cfg_f) = @_;
+       my $loc = "$type:".canonpath_harder($d);
+       my $w = delete $MDIR2CFGPATH->{$loc}->{$cfg_f};
+       delete $MDIR2CFGPATH->{$loc} if !(keys %{$MDIR2CFGPATH->{$loc}});
+       $_->cancel for @$w;
 }
 
-sub add_maildir_watch ($$) {
-       my ($d, $cfg_f) = @_;
-       if (!exists($MDIR2CFGPATH->{$d}->{$cfg_f})) {
-               my @w = $dir_idle->add_watches(["$d/cur", "$d/new"], 1);
-               push @{$MDIR2CFGPATH->{$d}->{$cfg_f}}, @w if @w;
+sub add_dir_watch ($$$) {
+       my ($type, $d, $cfg_f) = @_;
+       $d = canonpath_harder($d);
+       my $loc = "$type:$d";
+       my @dirs = $type eq 'mh' ? ($d) : ("$d/cur", "$d/new");
+       if (!exists($MDIR2CFGPATH->{$loc}->{$cfg_f})) {
+               my @w = $dir_idle->add_watches(\@dirs, 1);
+               push @{$MDIR2CFGPATH->{$loc}->{$cfg_f}}, @w if @w;
        }
 }
 
@@ -1467,24 +1481,20 @@ sub refresh_watches {
        my %seen;
        my $cfg_f = $cfg->{'-f'};
        for my $w (grep(/\Awatch\..+\.state\z/, keys %$cfg)) {
-               my $url = substr($w, length('watch.'), -length('.state'));
+               my $loc = substr($w, length('watch.'), -length('.state'));
                require PublicInbox::LeiWatch;
-               $watches->{$url} //= PublicInbox::LeiWatch->new($url);
-               $seen{$url} = undef;
-               my $state = $cfg->get_1("watch.$url.state");
+               $watches->{$loc} //= PublicInbox::LeiWatch->new($loc);
+               $seen{$loc} = undef;
+               my $state = $cfg->get_1("watch.$loc.state");
                if (!watch_state_ok($state)) {
-                       warn("watch.$url.state=$state not supported\n");
-                       next;
-               }
-               if ($url =~ /\Amaildir:(.+)/i) {
-                       my $d = canonpath_harder($1);
-                       if ($state eq 'pause') {
-                               cancel_maildir_watch($d, $cfg_f);
-                       } else {
-                               add_maildir_watch($d, $cfg_f);
-                       }
+                       warn("watch.$loc.state=$state not supported\n");
+               } elsif ($loc =~ /\A(maildir|mh):(.+)\z/i) {
+                       my ($type, $d) = ($1, $2);
+                       $state eq 'pause' ?
+                               cancel_dir_watch($type, $d, $cfg_f) :
+                               add_dir_watch($type, $d, $cfg_f);
                } else { # TODO: imap/nntp/jmap
-                       $lei->child_error(0, "E: watch $url not supported, yet")
+                       $lei->child_error(0, "E: watch $loc not supported, yet")
                }
        }
 
@@ -1492,29 +1502,28 @@ sub refresh_watches {
        my $lms = $lei->lms;
        if ($lms) {
                $lms->lms_write_prepare;
-               for my $d ($lms->folders('maildir:')) {
-                       substr($d, 0, length('maildir:')) = '';
-
+               for my $loc ($lms->folders(qr/\A(?:maildir|mh):/)) {
+                       my $old = $loc;
+                       my ($type, $d) = split /:/, $loc, 2;
                        # fixup old bugs while we're iterating:
-                       my $cd = canonpath_harder($d);
-                       my $f = "maildir:$cd";
-                       $lms->rename_folder("maildir:$d", $f) if $d ne $cd;
-                       next if $watches->{$f}; # may be set to pause
+                       $d = canonpath_harder($d);
+                       $loc = "$type:$d";
+                       $lms->rename_folder($old, $loc) if $old ne $loc;
+                       next if $watches->{$loc}; # may be set to pause
                        require PublicInbox::LeiWatch;
-                       $watches->{$f} = PublicInbox::LeiWatch->new($f);
-                       $seen{$f} = undef;
-                       add_maildir_watch($cd, $cfg_f);
+                       $watches->{$loc} = PublicInbox::LeiWatch->new($loc);
+                       $seen{$loc} = undef;
+                       add_dir_watch($type, $d, $cfg_f);
                }
        }
        if ($old) { # cull old non-existent entries
-               for my $url (keys %$old) {
-                       next if exists $seen{$url};
-                       delete $old->{$url};
-                       if ($url =~ /\Amaildir:(.+)/i) {
-                               my $d = canonpath_harder($1);
-                               cancel_maildir_watch($d, $cfg_f);
+               for my $loc (keys %$old) {
+                       next if exists $seen{$loc};
+                       delete $old->{$loc};
+                       if ($loc =~ /\A(maildir|mh):(.+)\z/i) {
+                               cancel_dir_watch($1, $2, $cfg_f);
                        } else { # TODO: imap/nntp/jmap
-                               $lei->child_error(0, "E: watch $url TODO");
+                               $lei->child_error(0, "E: watch $loc TODO");
                        }
                }
        }

diff --git a/lib/PublicInbox/LeiMailSync.pm b/lib/PublicInbox/LeiMailSync.pm
index 593715dc0af96998411eb8c708fdab26f0e41ac6..c498421c9d34b49240f1c0574d67d2bcbb7c98aa 100644 (file)
--- a/lib/PublicInbox/LeiMailSync.pm
+++ b/lib/PublicInbox/LeiMailSync.pm
@@ -425,9 +425,13 @@ sub folders {
        my $re;
        if (defined($pfx[0])) {
                $sql .= ' WHERE loc REGEXP ?'; # DBD::SQLite uses perlre
-               $re = !!$pfx[1] ? '.*' : '';
-               $re .= quotemeta($pfx[0]);
-               $re .= '.*';
+               if (ref($pfx[0])) { # assume qr// "Regexp"
+                       $re = $pfx[0];
+               } else {
+                       $re = !!$pfx[1] ? '.*' : '';
+                       $re .= quotemeta($pfx[0]);
+                       $re .= '.*';
+               }
        }
        my $sth = ($self->{dbh} //= dbh_new($self))->prepare($sql);
        $sth->bind_param(1, $re) if defined($re);

diff --git a/lib/PublicInbox/LeiNoteEvent.pm b/lib/PublicInbox/LeiNoteEvent.pm
index 8581bd9af7f6c9318413bf98cf5846b0adc414cf..8d900d0c0cd96403b18204668a4498ce9945ffd7 100644 (file)
--- a/lib/PublicInbox/LeiNoteEvent.pm
+++ b/lib/PublicInbox/LeiNoteEvent.pm
@@ -60,6 +60,18 @@ sub maildir_event { # via wq_nonblock_do
        } # else: eml_from_path already warns
 }
 
+sub _mh_cb { # mh_read_one cb
+       my ($dir, $bn, $kw, $eml, $self, $state) = @_;
+}
+
+sub mh_event { # via wq_nonblock_do
+       my ($self, $folder, $bn, $state) = @_;
+       my $dir = substr($folder, 3);
+       require PublicInbox::MHreader; # if we forked early
+       my $mhr = PublicInbox::MHreader->new($dir, $self->{lei}->{3});
+       $mhr->mh_read_one($bn, \&_mh_cb, $self, $state);
+}
+
 sub lei_note_event {
        my ($lei, $folder, $new_cur, $bn, $fn, @rest) = @_;
        die "BUG: unexpected: @rest" if @rest;
@@ -72,11 +84,14 @@ sub lei_note_event {
        $lms->arg2folder($lei, [ $folder ]);
        my $state = $cfg->get_1("watch.$folder.state") // 'tag-rw';
        return if $state eq 'pause';
-       return $lms->clear_src($folder, \$bn) if $new_cur eq '';
+       if ($new_cur eq '') {
+               my $id = $folder =~ /\Amaildir:/ ? \$bn : $bn + 0;
+               return $lms->clear_src($folder, $id);
+       }
        $lms->lms_pause;
        $lei->ale; # prepare
        $sto->write_prepare($lei);
-       require PublicInbox::MdirReader;
+       require PublicInbox::MHreader if $folder =~ /\Amh:/; # optimistic
        my $self = $cfg->{-lei_note_event} //= do {
                my $wq = bless { lms => $lms }, __PACKAGE__;
                # MUAs such as mutt can trigger massive rename() storms so
@@ -91,12 +106,15 @@ sub lei_note_event {
                $lei->{lne} = $wq;
        };
        if ($folder =~ /\Amaildir:/i) {
+               require PublicInbox::MdirReader;
                my $fl = PublicInbox::MdirReader::maildir_basename_flags($bn)
                        // return;
                return if index($fl, 'T') >= 0;
                my $kw = PublicInbox::MdirReader::flags2kw($fl);
                my $vmd = { kw => $kw, sync_info => [ $folder, \$bn ] };
                $self->wq_nonblock_do('maildir_event', $fn, $vmd, $state);
+       } elsif ($folder =~ /\Amh:/) {
+               $self->wq_nonblock_do('mh_event', $folder, $bn, $state);
        } # else: TODO: imap
 }
 

diff --git a/lib/PublicInbox/LeiWatch.pm b/lib/PublicInbox/LeiWatch.pm
index 35267b586c4de552780f104b773d57c1ae86fb13..b30e51524996edadfdcf5c9e2cc68b4d5060d66f 100644 (file)
--- a/lib/PublicInbox/LeiWatch.pm
+++ b/lib/PublicInbox/LeiWatch.pm
@@ -1,13 +1,12 @@
 # Copyright all contributors <meta@public-inbox.org>
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 
-# represents a Maildir or IMAP "watch" item
+# represents a Maildir, MH or IMAP "watch" item
 package PublicInbox::LeiWatch;
-use strict;
-use v5.10.1;
+use v5.12;
 use parent qw(PublicInbox::IPC);
 
-# "url" may be something like "maildir:/path/to/dir"
+# "url" may be something like "maildir:/path/to/dir" or "mh:/path/to/dir"
 sub new { bless { url => $_[1] }, $_[0] }
 
 1;

diff --git a/lib/PublicInbox/MHreader.pm b/lib/PublicInbox/MHreader.pm
index 673e3e06dbb7320b1a8004c90344a0012cd52567..033aa7404c8438d59816faab174499bee81b00d0 100644 (file)
--- a/lib/PublicInbox/MHreader.pm
+++ b/lib/PublicInbox/MHreader.pm
@@ -82,7 +82,7 @@ sub kw_for ($$) {
        \@kw;
 }
 
-sub _file2eml { # mh_each_file cb
+sub _file2eml { # mh_each_file / mh_read_one cb
        my ($dir, $n, $self, $ucb, @arg) = @_;
        my $eml = eml_from_path($n);
        $ucb->($dir, $n, kw_for($self, $n), $eml, @arg) if $eml;

diff --git a/t/lei-watch.t b/t/lei-watch.t
index 7b357ee097b0f79845b046620c19bbd82789ac18..8ad50d13aa024e99ed0c8459f50ad1a6857be488 100644 (file)
--- a/t/lei-watch.t
+++ b/t/lei-watch.t
@@ -3,6 +3,7 @@
 # License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt>
 use strict; use v5.10.1; use PublicInbox::TestCommon;
 use File::Path qw(make_path remove_tree);
+use PublicInbox::IO qw(write_file);
 plan skip_all => "TEST_FLAKY not enabled for $0" if !$ENV{TEST_FLAKY};
 require_mods('lei');
 my $have_fast_inotify = eval { require PublicInbox::Inotify } ||
@@ -13,7 +14,7 @@ $have_fast_inotify or
 
 my ($ro_home, $cfg_path) = setup_public_inboxes;
 test_lei(sub {
-       my $md = "$ENV{HOME}/md";
+       my ($md, $mh1, $mh2) = map { "$ENV{HOME}/$_" } qw(md mh1 mh2);
        my $cfg_f = "$ENV{HOME}/.config/lei/config";
        my $md2 = $md.'2';
        lei_ok 'ls-watch';
@@ -45,13 +46,14 @@ test_lei(sub {
        }
 
        # first, make sure tag-ro works
-       make_path("$md/new", "$md/cur", "$md/tmp");
+       make_path("$md/new", "$md/cur", "$md/tmp", $mh1, $mh2);
        lei_ok qw(add-watch --state=tag-ro), $md;
        lei_ok 'ls-watch';
        like($lei_out, qr/^\Qmaildir:$md\E$/sm, 'maildir shown');
        lei_ok qw(q mid:testmessage@example.com -o), $md, '-I', "$ro_home/t1";
        my @f = glob("$md/cur/*:2,");
        is(scalar(@f), 1, 'got populated maildir with one result');
+
        rename($f[0], "$f[0]S") or xbail "rename $!"; # set (S)een
        tick($have_fast_inotify ? 0.2 : 2.2); # always needed for 1 CPU systems
        lei_ok qw(note-event done); # flushes immediately (instead of 5s)
@@ -94,6 +96,12 @@ test_lei(sub {
                my $cmp = [ <$fh> ];
                is_xdeeply($cmp, $ino_contents, 'inotify Maildir watches gone');
        };
+
+       write_file '>', "$mh1/.mh_sequences";
+       lei_ok qw(add-watch --state=tag-ro), $mh1, "mh:$mh2";
+       lei_ok 'ls-watch', \'refresh watches';
+       like $lei_out, qr/^\Qmh:$mh1\E$/sm, 'MH 1 shown';
+       like $lei_out, qr/^\Qmh:$mh2\E$/sm, 'MH 2 shown';
 });
 
 done_testing;