Convert some byte strings to unicode.

Using latin1 for HTTP headers, and utf8 elsewhere.

This gets all the tests passing under python 3.2 after running 2to3.

diff --git a/tornado/httpserver.py b/tornado/httpserver.py
index c3a6f8a3fad35fd221f4185538936069b58c0a54..89459b34cd1fef2bd11f6d41be3b282e265b5c4b 100644 (file)
--- a/tornado/httpserver.py
+++ b/tornado/httpserver.py
@@ -328,7 +328,8 @@ class HTTPConnection(object):
 
     def _on_headers(self, data):
         try:
-            eol = data.find(b("\r\n"))
+            data = data.decode('latin1')
+            eol = data.find("\r\n")
             start_line = data[:eol]
             try:
                 method, uri, version = start_line.split(" ")
@@ -365,6 +366,7 @@ class HTTPConnection(object):
             if content_type.startswith("application/x-www-form-urlencoded"):
                 arguments = cgi.parse_qs(self._request.body)
                 for name, values in arguments.iteritems():
+                    name = name.decode('utf-8')
                     values = [v for v in values if v]
                     if values:
                         self._request.arguments.setdefault(name, []).extend(
@@ -413,7 +415,7 @@ class HTTPConnection(object):
             if not name_values.get("name"):
                 logging.warning("multipart/form-data value missing name")
                 continue
-            name = name_values["name"]
+            name = name_values["name"].decode("utf-8")
             if name_values.get("filename"):
                 ctype = headers.get("Content-Type", "application/unknown")
                 self._request.files.setdefault(name, []).append(dict(

diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py
index c943284683aa9c19356b69f61e84662b9c0b6e61..ac4fb52657b1d179fa34b27e8e3bd605b5c9d1bf 100644 (file)
--- a/tornado/simple_httpclient.py
+++ b/tornado/simple_httpclient.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 from __future__ import with_statement
 
-from tornado.escape import utf8
+from tornado.escape import utf8, _unicode
 from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient
 from tornado.httputil import HTTPHeaders
 from tornado.ioloop import IOLoop
@@ -133,7 +133,7 @@ class _HTTPConnection(object):
         # Timeout handle returned by IOLoop.add_timeout
         self._timeout = None
         with stack_context.StackContext(self.cleanup):
-            parsed = urlparse.urlsplit(self.request.url)
+            parsed = urlparse.urlsplit(_unicode(self.request.url))
             if ":" in parsed.netloc:
                 host, _, port = parsed.netloc.partition(":")
                 port = int(port)
@@ -245,6 +245,7 @@ class _HTTPConnection(object):
                                   error=HTTPError(599, "Connection closed")))
 
     def _on_headers(self, data):
+        data = data.decode("latin1")
         first_line, _, header_data = data.partition("\r\n")
         match = re.match("HTTP/1.[01] ([0-9]+) .*", first_line)
         assert match

diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py
index 8941332d027b44603ec2982a572bd7978d1c4759..707ab01e088931e73aa64f16dbb3bb06554ab6cd 100644 (file)
--- a/tornado/test/web_test.py
+++ b/tornado/test/web_test.py
@@ -128,8 +128,8 @@ class RequestEncodingTest(AsyncHTTPTestCase, LogTrapTestCase):
 
     def test_question_mark(self):
         # Ensure that url-encoded question marks are handled properly
-        self.assertEqual(json_decode(self.fetch('/%3F').body),
+        self.assertEqual(json_decode(self.fetch('/%3F').body.decode('utf8')),
                          dict(path='?', args={}))
-        self.assertEqual(json_decode(self.fetch('/%3F?%3F=%3F').body),
+        self.assertEqual(json_decode(self.fetch('/%3F?%3F=%3F').body.decode('utf8')),
                          dict(path='?', args={'?': ['?']}))
 

diff --git a/tornado/web.py b/tornado/web.py
index 43721dab0f79fecdcb8753aeb42e3bdf4f777e79..83937001955052b2ae653d2d91fc2a28ae8f955b 100644 (file)
--- a/tornado/web.py
+++ b/tornado/web.py
@@ -1427,7 +1427,7 @@ class GZipContentEncoding(OutputTransform):
 
     def transform_first_chunk(self, headers, chunk, finishing):
         if self._gzipping:
-            ctype = headers.get("Content-Type", "").split(";")[0]
+            ctype = _unicode(headers.get("Content-Type", "")).split(";")[0]
             self._gzipping = (ctype in self.CONTENT_TYPES) and \
                 (not finishing or len(chunk) >= self.MIN_LENGTH) and \
                 (finishing or "Content-Length" not in headers) and \