parsearchive: Support maildirs

At present, the 'parsearchive' command only supports parsing of mboxes.
Expand this to support maildirs. This allows us to rewrite the
'parsemail-bulk' script to deliver much improved performance.

Signed-off-by: Stephen Finucane <stephen@that.guru>
Suggested-by: Daniel Axtens <dja@axtens.net>

diff --git a/patchwork/bin/parsemail-batch.sh b/patchwork/bin/parsemail-batch.sh
index d42712ed099507c10cee0539b7696a47ca78a478..3d3725c6f792a6ff182d7d29a26fdac68057a4d0 100755 (executable)
--- a/patchwork/bin/parsemail-batch.sh
+++ b/patchwork/bin/parsemail-batch.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # Patchwork - automated patch tracking system
-# Copyright (C) 2008 Jeremy Kerr <jk@ozlabs.org>
+# Copyright (C) 2017 Stephen Finucane <stephen@that.guru>
 #
 # This file is part of the Patchwork package.
 #
@@ -20,25 +20,16 @@
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
 BIN_DIR=$(dirname "$0")
+PATCHWORK_BASE=$(readlink -e "$BIN_DIR/../..")
 
-if [ $# -lt 1 ]; then
-    echo "usage: $0 <dir> [options]" >&2
-    exit 1
+if [ -z "$PW_PYTHON" ]; then
+    PW_PYTHON=python2
 fi
 
-mail_dir="$1"
-
-echo "dir: $mail_dir"
-
-if [ ! -d "$mail_dir" ]; then
-    echo "$mail_dir should be a directory"? >&2
-    exit 1
+if [ -z "$DJANGO_SETTINGS_MODULE" ]; then
+    DJANGO_SETTINGS_MODULE=patchwork.settings.production
 fi
 
-shift
-
-find "$mail_dir" -maxdepth 1 |
-while read -r line; do
-    echo "$line"
-    "$BIN_DIR/parsemail.sh" "$@" < "$line"
-done
+PYTHONPATH="${PATCHWORK_BASE}:${PATCHWORK_BASE}/lib/python:$PYTHONPATH" \
+    DJANGO_SETTINGS_MODULE="$DJANGO_SETTINGS_MODULE" \
+    "$PW_PYTHON" "$PATCHWORK_BASE/manage.py" parsearchive "$@"

diff --git a/patchwork/management/commands/parsearchive.py b/patchwork/management/commands/parsearchive.py
index 40b2cc02a00e11b078d5130a66eb8d0f685b7fd0..a3c8360186c885d35da8e94c14c4fcbfde306299 100644 (file)
--- a/patchwork/management/commands/parsearchive.py
+++ b/patchwork/management/commands/parsearchive.py
@@ -69,7 +69,12 @@ class Command(BaseCommand):
             self.stdout.write('Invalid path: %s' % path)
             sys.exit(1)
 
-        mbox = mailbox.mbox(path)
+        # assume if <infile> is a directory, then we're passing a maildir
+        if os.path.isfile(path):
+            mbox = mailbox.mbox(path)
+        else:
+            mbox = mailbox.Maildir(path)
+
         count = len(mbox)
 
         logger.info('Parsing %d mails', count)