From: Stephen Finucane Date: Thu, 26 Nov 2015 19:43:37 +0000 (+0000) Subject: py3: "Modernize" code base X-Git-Tag: v1.1.0~92 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bcb65281b432a86ae092989e4226561830b294b1;p=thirdparty%2Fpatchwork.git py3: "Modernize" code base Run code through the 'modernize' application to fix Python 3 compatibility while also retaining Python 2 backwards compatibility. There are some key changes made to the autogenerated code: * Don't wrap 'items()' in 'list' for for loops - it's not necessary * Don't wrap 'keys()' in 'list' - just drop 'keys()' * Use Django's version of six rather than the upstream one Many of the issues found are based upon the changed definitions of the map, keys and items functions, along with the removal of the iteritems function and reduce keyword. Signed-off-by: Stephen Finucane --- diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py index 866743f9..2bc080cb 100755 --- a/patchwork/bin/parsemail.py +++ b/patchwork/bin/parsemail.py @@ -27,6 +27,7 @@ import datetime from email import message_from_file from email.header import Header, decode_header from email.utils import parsedate_tz, mktime_tz +from functools import reduce import logging import operator import re @@ -36,6 +37,8 @@ import django from django.conf import settings from django.contrib.auth.models import User from django.utils.log import AdminEmailHandler +from django.utils import six +from django.utils.six.moves import map from patchwork.models import ( Patch, Project, Person, Comment, State, get_default_initial_patch_state) @@ -67,7 +70,7 @@ def clean_header(header): return frag_str.decode(frag_encoding) return frag_str.decode() - fragments = map(decode, decode_header(header)) + fragments = list(map(decode, decode_header(header))) return normalise_space(u' '.join(fragments)) @@ -161,7 +164,7 @@ def mail_headers(mail): return reduce(operator.__concat__, ['%s: %s\n' % (k, Header(v, header_name=k, continuation_ws='\t').encode()) - for (k, v) in mail.items()]) + for (k, v) in list(mail.items())]) def find_pull_request(content): @@ -177,7 +180,7 @@ def find_pull_request(content): def try_decode(payload, charset): try: - payload = unicode(payload, charset) + payload = six.text_type(payload, charset) except UnicodeDecodeError: return None return payload @@ -195,7 +198,7 @@ def find_content(project, mail): payload = part.get_payload(decode=True) subtype = part.get_content_subtype() - if not isinstance(payload, unicode): + if not isinstance(payload, six.text_type): charset = part.get_content_charset() # Check that we have a charset that we understand. Otherwise, @@ -491,7 +494,7 @@ def main(args): def list_logging_levels(): """Give a summary of all available logging levels.""" - return sorted(VERBOSITY_LEVELS.keys(), + return sorted(list(VERBOSITY_LEVELS.keys()), key=lambda x: VERBOSITY_LEVELS[x]) parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), diff --git a/patchwork/filters.py b/patchwork/filters.py index a260ef1f..4f5a584b 100644 --- a/patchwork/filters.py +++ b/patchwork/filters.py @@ -19,11 +19,11 @@ from __future__ import absolute_import -from urllib import quote - from django.contrib.auth.models import User from django.utils.html import escape from django.utils.safestring import mark_safe +from django.utils import six +from django.utils.six.moves.urllib.parse import quote from patchwork.models import Person, State @@ -55,7 +55,7 @@ class Filter(object): pass def parse(self, dict): - if self.param not in dict.keys(): + if self.param not in dict: return self._set_key(dict[self.param]) @@ -278,7 +278,7 @@ class ArchiveFilter(Filter): def _set_key(self, str): self.archive_state = False self.applied = True - for (k, v) in self.param_map.iteritems(): + for (k, v) in self.param_map.items(): if str == v: self.archive_state = k if self.archive_state == None: @@ -411,7 +411,7 @@ filterclasses = [SubmitterFilter, \ class Filters: def __init__(self, request): - self._filters = map(lambda c: c(self), filterclasses) + self._filters = [c(self) for c in filterclasses] self.dict = request.GET self.project = None @@ -441,29 +441,27 @@ class Filters: def querystring(self, remove = None): params = dict(self.params()) - for (k, v) in self.dict.iteritems(): + for (k, v) in self.dict.items(): if k not in params: params[k] = v if remove is not None: - if remove.param in params.keys(): + if remove.param in list(params.keys()): del params[remove.param] - pairs = params.iteritems() - def sanitise(s): - if not isinstance(s, basestring): - s = unicode(s) + if not isinstance(s, six.string_types): + s = six.text_type(s) return quote(s.encode('utf-8')) return '?' + '&'.join(['%s=%s' % (sanitise(k), sanitise(v)) - for (k, v) in pairs]) + for (k, v) in list(params.items())]) def querystring_without_filter(self, filter): return self.querystring(filter) def applied_filters(self): - return filter(lambda x: x.applied, self._filters) + return [x for x in self._filters if x.applied] def available_filters(self): return self._filters diff --git a/patchwork/paginator.py b/patchwork/paginator.py index dbbcbaa4..310aac52 100644 --- a/patchwork/paginator.py +++ b/patchwork/paginator.py @@ -21,6 +21,7 @@ from __future__ import absolute_import from django.conf import settings from django.core import paginator +from django.utils.six.moves import range DEFAULT_PATCHES_PER_PAGE = 100 diff --git a/patchwork/parser.py b/patchwork/parser.py index a7fa89c3..84f704a5 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -26,6 +26,8 @@ from collections import Counter import hashlib import re +from django.utils.six.moves import map + _hunk_re = re.compile('^\@\@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? \@\@') _filename_re = re.compile('^(---|\+\+\+) (\S+)') @@ -113,7 +115,7 @@ def parse_patch(text): return 1 return int(x) - lc = map(fn, match.groups()) + lc = list(map(fn, match.groups())) state = 4 patchbuf += buf + line @@ -217,7 +219,7 @@ def hash_patch(str): if not x: return 1 return int(x) - line_nos = map(fn, hunk_match.groups()) + line_nos = list(map(fn, hunk_match.groups())) line = '@@ -%d +%d @@' % tuple(line_nos) elif line[0] in prefixes: diff --git a/patchwork/templatetags/listurl.py b/patchwork/templatetags/listurl.py index 118fd58a..7e0e3d77 100644 --- a/patchwork/templatetags/listurl.py +++ b/patchwork/templatetags/listurl.py @@ -39,7 +39,7 @@ class ListURLNode(template.defaulttags.URLNode): def __init__(self, kwargs): super(ListURLNode, self).__init__(None, [], {}, False) self.params = {} - for (k, v) in kwargs.iteritems(): + for (k, v) in kwargs.items(): if k in list_params: self.params[k] = v @@ -68,14 +68,14 @@ class ListURLNode(template.defaulttags.URLNode): except Exception: pass - for (k, v) in self.params.iteritems(): + for (k, v) in self.params.items(): params[smart_str(k, 'ascii')] = v.resolve(context) if not params: return str return str + '?' + '&'.join( - ['%s=%s' % (k, escape(v)) for (k, v) in params.iteritems()]) + ['%s=%s' % (k, escape(v)) for (k, v) in list(params.items())]) @register.tag diff --git a/patchwork/templatetags/syntax.py b/patchwork/templatetags/syntax.py index 3988f615..9d060449 100644 --- a/patchwork/templatetags/syntax.py +++ b/patchwork/templatetags/syntax.py @@ -24,6 +24,7 @@ import re from django import template from django.utils.html import escape from django.utils.safestring import mark_safe +from django.utils.six.moves import map register = template.Library() @@ -32,17 +33,17 @@ def _compile(t): (r, str) = t return (re.compile(r, re.M | re.I), str) -_patch_span_res = map(_compile, [ +_patch_span_res = list(map(_compile, [ ('^(Index:?|diff|\-\-\-|\+\+\+|\*\*\*) .*$', 'p_header'), ('^\+.*$', 'p_add'), ('^-.*$', 'p_del'), ('^!.*$', 'p_mod'), - ]) + ])) _patch_chunk_re = \ re.compile('^(@@ \-\d+(?:,\d+)? \+\d+(?:,\d+)? @@)(.*)$', re.M | re.I) -_comment_span_res = map(_compile, [ +_comment_span_res = list(map(_compile, [ ('^\s*Signed-off-by: .*$', 'signed-off-by'), ('^\s*Acked-by: .*$', 'acked-by'), ('^\s*Nacked-by: .*$', 'nacked-by'), @@ -50,7 +51,7 @@ _comment_span_res = map(_compile, [ ('^\s*Reviewed-by: .*$', 'reviewed-by'), ('^\s*From: .*$', 'from'), ('^\s*>.*$', 'quote'), - ]) + ])) _span = '%s' diff --git a/patchwork/tests/test_bundles.py b/patchwork/tests/test_bundles.py index 233e1296..4b982a5f 100644 --- a/patchwork/tests/test_bundles.py +++ b/patchwork/tests/test_bundles.py @@ -29,6 +29,7 @@ from django.utils.http import urlencode from patchwork.models import Patch, Bundle, BundlePatch, Person from patchwork.tests.utils import defaults, create_user, find_in_context +from django.utils.six.moves import range, zip def bundle_url(bundle): return '/bundle/%s/%s/' % (bundle.owner.username, bundle.name) @@ -573,11 +574,11 @@ class BundleInitialOrderTest(BundleTestBase): bundle.delete() def testBundleForwardOrder(self): - ids = map(lambda p: p.id, self.patches) + ids = [p.id for p in self.patches] self._testOrder(ids, self.patches) def testBundleReverseOrder(self): - ids = map(lambda p: p.id, self.patches) + ids = [p.id for p in self.patches] ids.reverse() self._testOrder(ids, self.patches) @@ -611,7 +612,7 @@ class BundleReorderTest(BundleTestBase): # check if order field is still sequential: order_numbers = [ bp.order for bp in bps ] - expected_order = range(1, len(neworder)+1) # [1 ... len(neworder)] + expected_order = list(range(1, len(neworder)+1)) # [1 ... len(neworder)] self.failUnlessEqual(order_numbers, expected_order) def testBundleReorderAll(self): diff --git a/patchwork/tests/test_checks.py b/patchwork/tests/test_checks.py index 8cf997b6..d3bae77a 100644 --- a/patchwork/tests/test_checks.py +++ b/patchwork/tests/test_checks.py @@ -81,7 +81,7 @@ class PatchChecksTest(TransactionTestCase): self.assertEqual(self.patch.check_set.count(), total) - for state in state_counts.keys(): + for state in state_counts: self.assertEqual(counts[state], state_counts[state]) # also check the ones we didn't explicitly state diff --git a/patchwork/tests/test_list.py b/patchwork/tests/test_list.py index 69c3d24d..0f23d131 100644 --- a/patchwork/tests/test_list.py +++ b/patchwork/tests/test_list.py @@ -28,6 +28,7 @@ import unittest from django.core.urlresolvers import reverse from django.test.client import Client from django.test import TestCase +from django.utils.six.moves import zip from patchwork.models import Person, Patch from patchwork.tests.utils import defaults, create_user, find_in_context @@ -83,7 +84,7 @@ class PatchOrderTest(TestCase): ids = self._extract_patch_ids(response) self.assertTrue(bool(ids)) patches = [ Patch.objects.get(id = i) for i in ids ] - pairs = zip(patches, patches[1:]) + pairs = list(zip(patches, patches[1:])) [ test_fn(p1, p2) for (p1, p2) in pairs ] def testDateOrder(self): diff --git a/patchwork/tests/test_person.py b/patchwork/tests/test_person.py index 7d82ce8c..ef35da64 100644 --- a/patchwork/tests/test_person.py +++ b/patchwork/tests/test_person.py @@ -24,6 +24,7 @@ import unittest from django.test.client import Client from django.test import TestCase +from django.utils.six.moves import map, range from patchwork.models import EmailConfirmation, Person, Bundle @@ -33,7 +34,7 @@ class SubmitterCompletionTest(TestCase): Person(name = "Test Name", email = "test1@example.com"), Person(email = "test2@example.com"), ] - map(lambda p: p.save(), self.people) + list(map(lambda p: p.save(), self.people)) def testNameComplete(self): response = self.client.get('/submitter/', {'q': 'name'}) diff --git a/patchwork/tests/test_xmlrpc.py b/patchwork/tests/test_xmlrpc.py index 06522ff8..0d032ed5 100644 --- a/patchwork/tests/test_xmlrpc.py +++ b/patchwork/tests/test_xmlrpc.py @@ -18,11 +18,11 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import unittest -import xmlrpclib from django.conf import settings from django.core.urlresolvers import reverse from django.test import LiveServerTestCase +from django.utils.six.moves import xmlrpc_client from patchwork.models import Person, Patch from patchwork.tests.utils import defaults @@ -36,7 +36,7 @@ class XMLRPCTest(LiveServerTestCase): def setUp(self): self.url = (self.live_server_url + reverse('patchwork.views.xmlrpc.xmlrpc')) - self.rpc = xmlrpclib.Server(self.url) + self.rpc = xmlrpc_client.Server(self.url) def testGetRedirect(self): response = self.client.patch(self.url) diff --git a/patchwork/tests/utils.py b/patchwork/tests/utils.py index 061a64e7..fb324936 100644 --- a/patchwork/tests/utils.py +++ b/patchwork/tests/utils.py @@ -98,7 +98,7 @@ def read_patch(filename, encoding = None): if encoding is not None: f = codecs.open(file_path, encoding = encoding) else: - f = file(file_path) + f = open(file_path) return f.read() diff --git a/patchwork/utils.py b/patchwork/utils.py index f151c127..0fbf317d 100644 --- a/patchwork/utils.py +++ b/patchwork/utils.py @@ -76,7 +76,7 @@ class Order(object): str = str[1:] reversed = True - if str not in self.order_map.keys(): + if str not in self.order_map: return self.order = str diff --git a/patchwork/views/mail.py b/patchwork/views/mail.py index 5e38bba3..af8249a5 100644 --- a/patchwork/views/mail.py +++ b/patchwork/views/mail.py @@ -108,7 +108,7 @@ def optinout(request, action, description): conf_settings.DEFAULT_FROM_EMAIL, [email]) context['email'] = mail context['email_sent'] = True - except Exception, ex: + except Exception as ex: context['error'] = 'An error occurred during confirmation . ' + \ 'Please try again later.' context['admins'] = conf_settings.ADMINS diff --git a/patchwork/views/patch.py b/patchwork/views/patch.py index 3c43cd7c..cb017fbd 100644 --- a/patchwork/views/patch.py +++ b/patchwork/views/patch.py @@ -64,7 +64,7 @@ def patch(request, patch_id): bundle.append_patch(patch) bundle.save() context.add_message('Patch added to bundle "%s"' % bundle.name) - except Exception, ex: + except Exception as ex: context.add_message("Couldn't add patch '%s' to bundle %s: %s" \ % (patch.name, bundle.name, ex.message)) diff --git a/patchwork/views/xmlrpc.py b/patchwork/views/xmlrpc.py index 9d7ef07f..64909e24 100644 --- a/patchwork/views/xmlrpc.py +++ b/patchwork/views/xmlrpc.py @@ -23,16 +23,21 @@ from __future__ import absolute_import import base64 -from DocXMLRPCServer import XMLRPCDocGenerator -from SimpleXMLRPCServer import SimpleXMLRPCDispatcher +# NOTE(stephenfin) six does not seem to support this +try: + from DocXMLRPCServer import XMLRPCDocGenerator +except ImportError: + from xmlrpc.server import XMLRPCDocGenerator import sys -import xmlrpclib from django.core import urlresolvers from django.contrib.auth import authenticate from django.http import ( HttpResponse, HttpResponseRedirect, HttpResponseServerError) from django.views.decorators.csrf import csrf_exempt +from django.utils import six +from django.utils.six.moves import map, xmlrpc_client +from django.utils.six.moves.xmlrpc_server import SimpleXMLRPCDispatcher from patchwork.models import Patch, Project, Person, State, Check from patchwork.views import patch_to_mbox @@ -52,7 +57,7 @@ class PatchworkXMLRPCDispatcher(SimpleXMLRPCDispatcher, def _dumps(obj, *args, **kwargs): kwargs['allow_none'] = self.allow_none kwargs['encoding'] = self.encoding - return xmlrpclib.dumps(obj, *args, **kwargs) + return xmlrpc_client.dumps(obj, *args, **kwargs) self.dumps = _dumps @@ -90,7 +95,7 @@ class PatchworkXMLRPCDispatcher(SimpleXMLRPCDispatcher, return authenticate(username=username, password=password) def _dispatch(self, request, method, params): - if method not in self.func_map.keys(): + if method not in list(self.func_map.keys()): raise Exception('method "%s" is not supported' % method) auth_required, fn = self.func_map[method] @@ -106,18 +111,18 @@ class PatchworkXMLRPCDispatcher(SimpleXMLRPCDispatcher, def _marshaled_dispatch(self, request): try: - params, method = xmlrpclib.loads(request.body) + params, method = six.moves.xmlrpc_client.loads(request.body) response = self._dispatch(request, method, params) # wrap response in a singleton tuple response = (response,) response = self.dumps(response, methodresponse=1) - except xmlrpclib.Fault, fault: + except six.moves.xmlrpc_client.Fault as fault: response = self.dumps(fault) except: # report exception back to server response = self.dumps( - xmlrpclib.Fault(1, '%s:%s' % (sys.exc_type, sys.exc_value)), + six.moves.xmlrpc_client.Fault(1, '%s:%s' % (sys.exc_info()[0], sys.exc_info()[1])), ) return response @@ -225,7 +230,7 @@ def person_to_dict(obj): 'id': obj.id, 'email': obj.email, 'name': name, - 'user': unicode(obj.user).encode('utf-8'), + 'user': six.text_type(obj.user).encode('utf-8'), } @@ -261,18 +266,18 @@ def patch_to_dict(obj): """ return { 'id': obj.id, - 'date': unicode(obj.date).encode('utf-8'), + 'date': six.text_type(obj.date).encode('utf-8'), 'filename': obj.filename(), 'msgid': obj.msgid, 'name': obj.name, - 'project': unicode(obj.project).encode('utf-8'), + 'project': six.text_type(obj.project).encode('utf-8'), 'project_id': obj.project_id, - 'state': unicode(obj.state).encode('utf-8'), + 'state': six.text_type(obj.state).encode('utf-8'), 'state_id': obj.state_id, 'archived': obj.archived, - 'submitter': unicode(obj.submitter).encode('utf-8'), + 'submitter': six.text_type(obj.submitter).encode('utf-8'), 'submitter_id': obj.submitter_id, - 'delegate': unicode(obj.delegate).encode('utf-8'), + 'delegate': six.text_type(obj.delegate).encode('utf-8'), 'delegate_id': max(obj.delegate_id, 0), 'commit_ref': max(obj.commit_ref, ''), } @@ -333,10 +338,10 @@ def check_to_dict(obj): object which is OK to send to the client.""" return { 'id': obj.id, - 'date': unicode(obj.date).encode('utf-8'), - 'patch': unicode(obj.patch).encode('utf-8'), + 'date': six.text_type(obj.date).encode('utf-8'), + 'patch': six.text_type(obj.patch).encode('utf-8'), 'patch_id': obj.patch_id, - 'user': unicode(obj.user).encode('utf-8'), + 'user': six.text_type(obj.user).encode('utf-8'), 'user_id': obj.user_id, 'state': obj.get_state_display(), 'target_url': obj.target_url, @@ -401,7 +406,7 @@ def project_list(search_str=None, max_count=0): if max_count > 0: return map(project_to_dict, projects)[:max_count] else: - return map(project_to_dict, projects) + return list(map(project_to_dict, projects)) except Project.DoesNotExist: return [] @@ -453,7 +458,7 @@ def person_list(search_str=None, max_count=0): if max_count > 0: return map(person_to_dict, people)[:max_count] else: - return map(person_to_dict, people) + return list(map(person_to_dict, people)) except Person.DoesNotExist: return [] @@ -594,9 +599,9 @@ def patch_list(filt=None): patches = Patch.objects.filter(**dfilter) if max_count > 0: - return map(patch_to_dict, patches[:max_count]) + return [patch_to_dict(patch) for patch in patches[:max_count]] else: - return map(patch_to_dict, patches) + return [patch_to_dict(patch) for patch in patches] except Patch.DoesNotExist: return [] @@ -746,7 +751,7 @@ def patch_set(user, patch_id, params): if not patch.is_editable(user): raise Exception('No permissions to edit this patch') - for (k, v) in params.iteritems(): + for (k, v) in params.items(): if k not in ok_params: continue @@ -789,7 +794,7 @@ def state_list(search_str=None, max_count=0): if max_count > 0: return map(state_to_dict, states)[:max_count] else: - return map(state_to_dict, states) + return list(map(state_to_dict, states)) except State.DoesNotExist: return [] @@ -915,9 +920,9 @@ def check_list(filt=None): checks = Check.objects.filter(**dfilter) if max_count > 0: - return map(check_to_dict, checks[:max_count]) + return list(map(check_to_dict, checks[:max_count])) else: - return map(check_to_dict, checks) + return list(map(check_to_dict, checks)) except Check.DoesNotExist: return []