From: Stephen Finucane Date: Fri, 13 Nov 2015 03:21:03 +0000 (+0000) Subject: trivial: Cleanup of 'bin/parsemail' X-Git-Tag: v1.1.0~102 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2fea8aba0e8ad0555cb306f6e1539bdd162098c3;p=thirdparty%2Fpatchwork.git trivial: Cleanup of 'bin/parsemail' Do some cleanup of the script, removing excess whitespace and imports, localising variables where possible etc. This file still isn't py3 compatible but it's getting there. Signed-off-by: Stephen Finucane --- diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py index c15564e9..e05f0365 100755 --- a/patchwork/bin/parsemail.py +++ b/patchwork/bin/parsemail.py @@ -19,34 +19,35 @@ # along with Patchwork; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import sys -import re -import datetime -import time -import operator import codecs +import datetime from email import message_from_file from email.header import Header, decode_header from email.utils import parsedate_tz, mktime_tz import logging +import operator +import re +import sys -from patchwork.parser import parse_patch -from patchwork.models import Patch, Project, Person, Comment, State, \ - get_default_initial_patch_state import django from django.conf import settings from django.contrib.auth.models import User from django.utils.log import AdminEmailHandler +from patchwork.models import ( + Patch, Project, Person, Comment, State, get_default_initial_patch_state) +from patchwork.parser import parse_patch + list_id_headers = ['List-ID', 'X-Mailing-List', 'X-list'] -whitespace_re = re.compile('\s+') + def normalise_space(str): + whitespace_re = re.compile(r'\s+') return whitespace_re.sub(' ', str).strip() -def clean_header(header): - """ Decode (possibly non-ascii) headers """ +def clean_header(header): + """Decode (possibly non-ascii) headers.""" def decode(fragment): (frag_str, frag_encoding) = fragment if frag_encoding: @@ -57,10 +58,11 @@ def clean_header(header): return normalise_space(u' '.join(fragments)) + def find_project(mail): project = None - listid_res = [re.compile('.*<([^>]+)>.*', re.S), - re.compile('^([\S]+)$', re.S)] + listid_res = [re.compile(r'.*<([^>]+)>.*', re.S), + re.compile(r'^([\S]+)$', re.S)] for header in list_id_headers: if header in mail: @@ -76,13 +78,14 @@ def find_project(mail): listid = match.group(1) try: - project = Project.objects.get(listid = listid) + project = Project.objects.get(listid=listid) break except Project.DoesNotExist: pass return project + def find_author(mail): from_header = clean_header(mail.get('From')) @@ -93,13 +96,13 @@ def find_author(mail): # from re.match().groups() from_res = [ # for "Firstname Lastname" style addresses - (re.compile('"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))), + (re.compile(r'"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))), - # for example@example.com (Firstname Lastname) style addresses - (re.compile('"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))), + # for example@example.com (Firstname Lastname) style addresses + (re.compile(r'"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))), - # everything else - (re.compile('(.*)'), (lambda g: (None, g[0]))), + # everything else + (re.compile(r'(.*)'), (lambda g: (None, g[0]))), ] for regex, fn in from_res: @@ -118,35 +121,39 @@ def find_author(mail): new_person = False try: - person = Person.objects.get(email__iexact = email) + person = Person.objects.get(email__iexact=email) except Person.DoesNotExist: - person = Person(name = name, email = email) + person = Person(name=name, email=email) new_person = True return (person, new_person) + def mail_date(mail): t = parsedate_tz(mail.get('Date', '')) if not t: return datetime.datetime.utcnow() return datetime.datetime.utcfromtimestamp(mktime_tz(t)) + def mail_headers(mail): return reduce(operator.__concat__, - ['%s: %s\n' % (k, Header(v, header_name = k, \ - continuation_ws = '\t').encode()) \ - for (k, v) in mail.items()]) + ['%s: %s\n' % (k, Header(v, header_name=k, + continuation_ws='\t').encode()) + for (k, v) in mail.items()]) + def find_pull_request(content): - git_re = re.compile('^The following changes since commit.*' + - '^are available in the git repository at:\n' - '^\s*([\S]+://[^\n]+)$', - re.DOTALL | re.MULTILINE) + git_re = re.compile(r'^The following changes since commit.*' + + r'^are available in the git repository at:\n' + r'^\s*([\S]+://[^\n]+)$', + re.DOTALL | re.MULTILINE) match = git_re.search(content) if match: return match.group(1) return None + def try_decode(payload, charset): try: payload = unicode(payload, charset) @@ -154,6 +161,7 @@ def try_decode(payload, charset): return None return payload + def find_content(project, mail): patchbuf = None commentbuf = '' @@ -173,7 +181,7 @@ def find_content(project, mail): # ignore it and fallback to our standard set. if charset is not None: try: - codec = codecs.lookup(charset) + codecs.lookup(charset) except LookupError: charset = None @@ -214,27 +222,27 @@ def find_content(project, mail): if pullurl or patchbuf: name = clean_subject(mail.get('Subject'), [project.linkname]) - patch = Patch(name = name, pull_url = pullurl, content = patchbuf, - date = mail_date(mail), headers = mail_headers(mail)) + patch = Patch(name=name, pull_url=pullurl, content=patchbuf, + date=mail_date(mail), headers=mail_headers(mail)) if commentbuf: # If this is a new patch, we defer setting comment.patch until # patch has been saved by the caller if patch: - comment = Comment(date = mail_date(mail), - content = clean_content(commentbuf), - headers = mail_headers(mail)) - + comment = Comment(date=mail_date(mail), + content=clean_content(commentbuf), + headers=mail_headers(mail)) else: cpatch = find_patch_for_comment(project, mail) if not cpatch: return (None, None) - comment = Comment(patch = cpatch, date = mail_date(mail), - content = clean_content(commentbuf), - headers = mail_headers(mail)) + comment = Comment(patch=cpatch, date=mail_date(mail), + content=clean_content(commentbuf), + headers=mail_headers(mail)) return (patch, comment) + def find_patch_for_comment(project, mail): # construct a list of possible reply message ids refs = [] @@ -253,47 +261,50 @@ def find_patch_for_comment(project, mail): # first, check for a direct reply try: - patch = Patch.objects.get(project = project, msgid = ref) + patch = Patch.objects.get(project=project, msgid=ref) return patch except Patch.DoesNotExist: pass # see if we have comments that refer to a patch try: - comment = Comment.objects.get(patch__project = project, msgid = ref) + comment = Comment.objects.get(patch__project=project, msgid=ref) return comment.patch except Comment.DoesNotExist: pass - return None -split_re = re.compile('[,\s]+') def split_prefixes(prefix): - """ Turn a prefix string into a list of prefix tokens """ - + """Turn a prefix string into a list of prefix tokens.""" + split_re = re.compile(r'[,\s]+') matches = split_re.split(prefix) - return [ s for s in matches if s != '' ] - -re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I) -prefix_re = re.compile('^\[([^\]]*)\]\s*(.*)$') -def clean_subject(subject, drop_prefixes = None): - """ Clean a Subject: header from an incoming patch. + return [s for s in matches if s != ''] - Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By - default, only [PATCH] is removed, and we keep any other bracketed data - in the subject. If drop_prefixes is provided, remove those too, - comparing case-insensitively.""" +def clean_subject(subject, drop_prefixes=None): + """Clean a Subject: header from an incoming patch. + Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By + default, only [PATCH] is removed, and we keep any other bracketed + data in the subject. If drop_prefixes is provided, remove those + too, comparing case-insensitively. + + Args: + subject: Subject to be cleaned + drop_prefixes: Additional, case-insensitive prefixes to remove + from the subject + """ + re_re = re.compile(r'^(re|fwd?)[:\s]\s*', re.I) + prefix_re = re.compile(r'^\[([^\]]*)\]\s*(.*)$') subject = clean_header(subject) if drop_prefixes is None: drop_prefixes = [] else: - drop_prefixes = [ s.lower() for s in drop_prefixes ] + drop_prefixes = [s.lower() for s in drop_prefixes] drop_prefixes.append('patch') @@ -308,8 +319,8 @@ def clean_subject(subject, drop_prefixes = None): while match: prefix_str = match.group(1) - prefixes += [ p for p in split_prefixes(prefix_str) \ - if p.lower() not in drop_prefixes] + prefixes += [p for p in split_prefixes(prefix_str) + if p.lower() not in drop_prefixes] subject = match.group(2) match = prefix_re.match(subject) @@ -322,14 +333,20 @@ def clean_subject(subject, drop_prefixes = None): return subject -sig_re = re.compile('^(-- |_+)\n.*', re.S | re.M) -def clean_content(str): - """ Try to remove signature (-- ) and list footer (_____) cruft """ - str = sig_re.sub('', str) - return str.strip() + +def clean_content(content): + """Remove cruft from the email message. + + Catch ignature (-- ) and list footer (_____) cruft. + """ + sig_re = re.compile(r'^(-- |_+)\n.*', re.S | re.M) + content = sig_re.sub('', content) + + return content.strip() + def get_state(state_name): - """ Return the state with the given name or the default State """ + """Return the state with the given name or the default.""" if state_name: try: return State.objects.get(name__iexact=state_name) @@ -337,8 +354,9 @@ def get_state(state_name): pass return get_default_initial_patch_state() + def get_delegate(delegate_email): - """ Return the delegate with the given email or None """ + """Return the delegate with the given email or None.""" if delegate_email: try: return User.objects.get(email__iexact=delegate_email) @@ -346,8 +364,9 @@ def get_delegate(delegate_email): pass return None -def parse_mail(mail): +def parse_mail(mail): + """Parse a mail and add to the database.""" # some basic sanity checks if 'From' not in mail: return 0 @@ -360,11 +379,12 @@ def parse_mail(mail): hint = mail.get('X-Patchwork-Hint', '').lower() if hint == 'ignore': - return 0; + return 0 project = find_project(mail) + if project is None: - print "no project found" + print("no project found") return 0 msgid = mail.get('Message-Id').strip() @@ -383,7 +403,7 @@ def parse_mail(mail): patch.project = project patch.state = get_state(mail.get('X-Patchwork-State', '').strip()) patch.delegate = get_delegate( - mail.get('X-Patchwork-Delegate', '').strip()) + mail.get('X-Patchwork-Delegate', '').strip()) patch.save() if comment: @@ -408,10 +428,15 @@ extra_error_message = ''' ''' -# Send emails to settings.ADMINS when encountering errors + def setup_error_handler(): + """Configure error handler. + + Ensure emails are send to settings.ADMINS when errors are + encountered. + """ if settings.DEBUG: - return None + return mail_handler = AdminEmailHandler() mail_handler.setLevel(logging.ERROR) @@ -422,6 +447,7 @@ def setup_error_handler(): return logger + def main(args): django.setup() logger = setup_error_handler() @@ -435,5 +461,6 @@ def main(args): }) raise + if __name__ == '__main__': sys.exit(main(sys.argv))