From: Stephen Finucane Date: Sun, 29 Nov 2015 06:53:05 +0000 (+0000) Subject: py3: Resolve unicode issues X-Git-Tag: v1.1.0~89 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e8a50c8c302875a83ee6c3e6d23ab9ee3fdf0596;p=thirdparty%2Fpatchwork.git py3: Resolve unicode issues Python 3 is unicode only. While many of the issues with unicode, such as the now invalid 'u' prefix, have already been resolved, there are a few more issues. Many of these issues are related to HTTPResponse.content, which returns bytes and needs to be "decoded" in order to perform actions like concatenation with str objects (unicode). Where possible, make use of assertContains, per the Django documentation (http://bit.ly/1lRDYie), else fall back to including a 'decode' statement. Signed-off-by: Stephen Finucane --- diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py index 2bc080cb..43d733f3 100755 --- a/patchwork/bin/parsemail.py +++ b/patchwork/bin/parsemail.py @@ -68,7 +68,9 @@ def clean_header(header): (frag_str, frag_encoding) = fragment if frag_encoding: return frag_str.decode(frag_encoding) - return frag_str.decode() + elif isinstance(frag_str, six.binary_type): # python 2 + return frag_str.decode() + return frag_str fragments = list(map(decode, decode_header(header))) diff --git a/patchwork/parser.py b/patchwork/parser.py index 84f704a5..b65e50cd 100644 --- a/patchwork/parser.py +++ b/patchwork/parser.py @@ -230,7 +230,7 @@ def hash_patch(str): # other lines are ignored continue - hash.update(line.encode('utf-8') + '\n') + hash.update((line + '\n').encode('utf-8')) return hash diff --git a/patchwork/tests/test_bundles.py b/patchwork/tests/test_bundles.py index 4b982a5f..d5d5d2f3 100644 --- a/patchwork/tests/test_bundles.py +++ b/patchwork/tests/test_bundles.py @@ -109,7 +109,7 @@ class BundleViewTest(BundleTestBase): pos = 0 for patch in self.patches: - next_pos = response.content.find(patch.name) + next_pos = response.content.decode().find(patch.name) # ensure that this patch is after the previous self.failUnless(next_pos > pos) pos = next_pos @@ -126,7 +126,7 @@ class BundleViewTest(BundleTestBase): response = self.client.get(bundle_url(self.bundle)) pos = len(response.content) for patch in self.patches: - next_pos = response.content.find(patch.name) + next_pos = response.content.decode().find(patch.name) # ensure that this patch is now *before* the previous self.failUnless(next_pos < pos) pos = next_pos diff --git a/patchwork/tests/test_filters.py b/patchwork/tests/test_filters.py index ae1ff7c9..c92146ba 100644 --- a/patchwork/tests/test_filters.py +++ b/patchwork/tests/test_filters.py @@ -35,8 +35,8 @@ class FilterQueryStringTest(TestCase): url = '/project/%s/list/?submitter=a%%26b=c' % project.linkname response = self.client.get(url) self.failUnlessEqual(response.status_code, 200) - self.failIf('submitter=a&b=c' in response.content) - self.failIf('submitter=a&b=c' in response.content) + self.assertNotContains(response, 'submitter=a&b=c') + self.assertNotContains(response, 'submitter=a&b=c') def testUTF8QSHandling(self): """test that non-ascii characters can be handled by the filter diff --git a/patchwork/tests/test_list.py b/patchwork/tests/test_list.py index 0f23d131..62ff121f 100644 --- a/patchwork/tests/test_list.py +++ b/patchwork/tests/test_list.py @@ -77,7 +77,7 @@ class PatchOrderTest(TestCase): def _extract_patch_ids(self, response): id_re = re.compile('may' in response.content) + self.assertContains(response, 'may') optout_url = reverse('patchwork.views.mail.optout') - self.assertTrue(('action="%s"' % optout_url) in response.content) + self.assertContains(response, ('action="%s"' % optout_url)) def testMailSettingsPOSTOptedOut(self): email = u'foo@example.com' @@ -74,9 +74,9 @@ class MailSettingsTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/mail-settings.html') self.assertEquals(response.context['is_optout'], True) - self.assertTrue('may not' in response.content) + self.assertContains(response, 'may not') optin_url = reverse('patchwork.views.mail.optin') - self.assertTrue(('action="%s"' % optin_url) in response.content) + self.assertContains(response, ('action="%s"' % optin_url)) class OptoutRequestTest(TestCase): @@ -98,7 +98,7 @@ class OptoutRequestTest(TestCase): # check confirmation page self.assertEquals(response.status_code, 200) self.assertEquals(response.context['confirmation'], conf) - self.assertTrue(email in response.content) + self.assertContains(response, email) # check email url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key}) @@ -140,7 +140,7 @@ class OptoutTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/optout.html') - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check that we've got an optout in the list self.assertEquals(EmailOptout.objects.count(), 1) @@ -178,7 +178,7 @@ class OptinRequestTest(TestCase): # check confirmation page self.assertEquals(response.status_code, 200) self.assertEquals(response.context['confirmation'], conf) - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check email url = reverse('patchwork.views.confirm', kwargs = {'key': conf.key}) @@ -221,7 +221,7 @@ class OptinTest(TestCase): self.assertEquals(response.status_code, 200) self.assertTemplateUsed(response, 'patchwork/optin.html') - self.assertTrue(self.email in response.content) + self.assertContains(response, self.email) # check that there's no optout remaining self.assertEquals(EmailOptout.objects.count(), 0) @@ -243,7 +243,7 @@ class OptinWithoutOptoutTest(TestCase): # check for an error message self.assertEquals(response.status_code, 200) self.assertTrue(bool(response.context['error'])) - self.assertTrue('not on the patchwork opt-out list' in response.content) + self.assertContains(response, 'not on the patchwork opt-out list') class UserProfileOptoutFormTest(TestCase): """Test that the correct optin/optout forms appear on the user profile @@ -270,23 +270,22 @@ class UserProfileOptoutFormTest(TestCase): form_re = self._form_re(self.optout_url, self.user.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testMainEmailOptinForm(self): EmailOptout(email = self.user.email).save() form_re = self._form_re(self.optin_url, self.user.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testSecondaryEmailOptoutForm(self): p = Person(email = self.secondary_email, user = self.user) p.save() - form_re = self._form_re(self.optout_url, p.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) def testSecondaryEmailOptinForm(self): p = Person(email = self.secondary_email, user = self.user) @@ -296,4 +295,4 @@ class UserProfileOptoutFormTest(TestCase): form_re = self._form_re(self.optin_url, p.email) response = self.client.get(self.url) self.assertEquals(response.status_code, 200) - self.assertTrue(form_re.search(response.content) is not None) + self.assertTrue(form_re.search(response.content.decode()) is not None) diff --git a/patchwork/tests/test_mboxviews.py b/patchwork/tests/test_mboxviews.py index 8c98351f..37bab747 100644 --- a/patchwork/tests/test_mboxviews.py +++ b/patchwork/tests/test_mboxviews.py @@ -173,7 +173,7 @@ class MboxDateHeaderTest(TestCase): def testDateHeader(self): response = self.client.get('/patch/%d/mbox/' % self.patch.id) - mail = email.message_from_string(response.content) + mail = email.message_from_string(response.content.decode()) mail_date = dateutil.parser.parse(mail['Date']) # patch dates are all in UTC patch_date = self.patch.date.replace(tzinfo=dateutil.tz.tzutc(), @@ -190,7 +190,7 @@ class MboxDateHeaderTest(TestCase): self.patch.save() response = self.client.get('/patch/%d/mbox/' % self.patch.id) - mail = email.message_from_string(response.content) + mail = email.message_from_string(response.content.decode()) mail_date = dateutil.parser.parse(mail['Date']) self.assertEqual(mail_date, date) diff --git a/patchwork/tests/test_person.py b/patchwork/tests/test_person.py index ef35da64..9b91115d 100644 --- a/patchwork/tests/test_person.py +++ b/patchwork/tests/test_person.py @@ -39,14 +39,14 @@ class SubmitterCompletionTest(TestCase): def testNameComplete(self): response = self.client.get('/submitter/', {'q': 'name'}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 1) self.assertEquals(data[0]['name'], 'Test Name') def testEmailComplete(self): response = self.client.get('/submitter/', {'q': 'test2'}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 1) self.assertEquals(data[0]['email'], 'test2@example.com') @@ -56,5 +56,5 @@ class SubmitterCompletionTest(TestCase): person.save() response = self.client.get('/submitter/', {'q': 'test', 'l': 5}) self.assertEquals(response.status_code, 200) - data = json.loads(response.content) + data = json.loads(response.content.decode()) self.assertEquals(len(data), 5) diff --git a/patchwork/views/patch.py b/patchwork/views/patch.py index cb017fbd..7abb22f9 100644 --- a/patchwork/views/patch.py +++ b/patchwork/views/patch.py @@ -21,6 +21,7 @@ from __future__ import absolute_import from django.http import HttpResponse, HttpResponseForbidden from django.shortcuts import render_to_response, get_object_or_404 +from django.utils import six from patchwork.forms import PatchForm, CreateBundleForm from patchwork.models import Patch, Project, Bundle @@ -96,7 +97,11 @@ def content(request, patch_id): def mbox(request, patch_id): patch = get_object_or_404(Patch, id=patch_id) response = HttpResponse(content_type="text/plain") - response.write(patch_to_mbox(patch).as_string(True)) + # NOTE(stephenfin) http://stackoverflow.com/a/28584090/613428 + if six.PY3: + response.write(patch_to_mbox(patch).as_bytes(True).decode()) + else: + response.write(patch_to_mbox(patch).as_string(True)) response['Content-Disposition'] = 'attachment; filename=' + \ patch.filename().replace(';', '').replace('\n', '') return response