From: Ben Darnell Date: Mon, 14 Jan 2013 02:15:45 +0000 (-0500) Subject: Checkpoint: tests can now be imported in python 3 without 2to3. X-Git-Tag: v3.0.0~180 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=051cd609e1e20ac0945266b8930d7ab79f0314e8;p=thirdparty%2Ftornado.git Checkpoint: tests can now be imported in python 3 without 2to3. The tests are not passing, but this fixes up all the import-time problems. The tests still work under 2to3 or python 2. --- diff --git a/tornado/auth.py b/tornado/auth.py index fd4728537..d1ddbc52a 100644 --- a/tornado/auth.py +++ b/tornado/auth.py @@ -52,7 +52,6 @@ import hashlib import hmac import time import urllib -import urlparse import uuid from tornado import httpclient @@ -61,6 +60,11 @@ from tornado.httputil import url_concat from tornado.log import gen_log from tornado.util import bytes_type, b, u +try: + import urlparse # py2 +except ImportError: + import urllib.parse as urlparse # py3 + class OpenIdMixin(object): """Abstract implementation of OpenID and Attribute Exchange. diff --git a/tornado/escape.py b/tornado/escape.py index ecf97a1df..4b94e95b0 100644 --- a/tornado/escape.py +++ b/tornado/escape.py @@ -22,24 +22,22 @@ have crept in over time. from __future__ import absolute_import, division, print_function, with_statement -import htmlentitydefs import re import sys import urllib -from tornado.util import u - -# Python3 compatibility: On python2.5, introduce the bytes alias from 2.6 -try: - bytes -except Exception: - bytes = str +from tornado.util import bytes_type, unicode_type, basestring_type, u try: from urlparse import parse_qs # Python 2.6+ except ImportError: from cgi import parse_qs +try: + import htmlentitydefs # py2 +except ImportError: + import html.entities as htmlentitydefs # py3 + # json module is in the standard library as of python 2.6; fall back to # simplejson if present for older versions. try: @@ -65,6 +63,10 @@ except Exception: "http://pypi.python.org/pypi/simplejson/") _json_encode = _json_decode +try: + unichr +except NameError: + unichr = chr _XHTML_ESCAPE_RE = re.compile('[&<>"]') _XHTML_ESCAPE_DICT = {'&': '&', '<': '<', '>': '>', '"': '"'} @@ -121,7 +123,7 @@ if sys.version_info[0] < 3: if encoding is None: return urllib.unquote_plus(utf8(value)) else: - return unicode(urllib.unquote_plus(utf8(value)), encoding) + return unicode_type(urllib.unquote_plus(utf8(value)), encoding) parse_qs_bytes = parse_qs else: @@ -156,7 +158,7 @@ else: return encoded -_UTF8_TYPES = (bytes, type(None)) +_UTF8_TYPES = (bytes_type, type(None)) def utf8(value): @@ -167,10 +169,10 @@ def utf8(value): """ if isinstance(value, _UTF8_TYPES): return value - assert isinstance(value, unicode) + assert isinstance(value, unicode_type) return value.encode("utf-8") -_TO_UNICODE_TYPES = (unicode, type(None)) +_TO_UNICODE_TYPES = (unicode_type, type(None)) def to_unicode(value): @@ -181,7 +183,7 @@ def to_unicode(value): """ if isinstance(value, _TO_UNICODE_TYPES): return value - assert isinstance(value, bytes) + assert isinstance(value, bytes_type) return value.decode("utf-8") # to_unicode was previously named _unicode not because it was private, @@ -190,12 +192,12 @@ _unicode = to_unicode # When dealing with the standard library across python 2 and 3 it is # sometimes useful to have a direct conversion to the native string type -if str is unicode: +if str is unicode_type: native_str = to_unicode else: native_str = utf8 -_BASESTRING_TYPES = (basestring, type(None)) +_BASESTRING_TYPES = (basestring_type, type(None)) def to_basestring(value): @@ -209,7 +211,7 @@ def to_basestring(value): """ if isinstance(value, _BASESTRING_TYPES): return value - assert isinstance(value, bytes) + assert isinstance(value, bytes_type) return value.decode("utf-8") @@ -224,7 +226,7 @@ def recursive_unicode(obj): return list(recursive_unicode(i) for i in obj) elif isinstance(obj, tuple): return tuple(recursive_unicode(i) for i in obj) - elif isinstance(obj, bytes): + elif isinstance(obj, bytes_type): return to_unicode(obj) else: return obj @@ -348,7 +350,7 @@ def _convert_entity(m): def _build_unicode_map(): unicode_map = {} - for name, value in htmlentitydefs.name2codepoint.iteritems(): + for name, value in htmlentitydefs.name2codepoint.items(): unicode_map[name] = unichr(value) return unicode_map diff --git a/tornado/httpclient.py b/tornado/httpclient.py index ec75515a6..31024c5ab 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -33,7 +33,6 @@ from __future__ import absolute_import, division, print_function, with_statement import calendar import email.utils -import httplib import time import weakref @@ -346,7 +345,7 @@ class HTTPResponse(object): time_info=None, reason=None): self.request = request self.code = code - self.reason = reason or httplib.responses.get(code, "Unknown") + self.reason = reason or httputil.responses.get(code, "Unknown") if headers is not None: self.headers = headers else: @@ -403,7 +402,7 @@ class HTTPError(Exception): """ def __init__(self, code, message=None, response=None): self.code = code - message = message or httplib.responses.get(code, "Unknown") + message = message or httputil.responses.get(code, "Unknown") self.response = response Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) diff --git a/tornado/httpserver.py b/tornado/httpserver.py index 003f18409..f2502586b 100644 --- a/tornado/httpserver.py +++ b/tornado/httpserver.py @@ -26,7 +26,6 @@ This module also defines the `HTTPRequest` class which is exposed via from __future__ import absolute_import, division, print_function, with_statement -import Cookie import socket import time @@ -43,6 +42,10 @@ try: except ImportError: ssl = None +try: + import Cookie # py2 +except ImportError: + import http.cookies as Cookie # py3 class HTTPServer(TCPServer): r"""A non-blocking, single-threaded HTTP server. diff --git a/tornado/httputil.py b/tornado/httputil.py index 30cefbb2c..8c7f684fa 100644 --- a/tornado/httputil.py +++ b/tornado/httputil.py @@ -25,6 +25,10 @@ from tornado.escape import native_str, parse_qs_bytes, utf8 from tornado.log import gen_log from tornado.util import b, ObjectDict +try: + from httplib import responses # py2 +except ImportError: + from http.client import responses # py3 class HTTPHeaders(dict): """A dictionary that maintains Http-Header-Case for all keys. diff --git a/tornado/ioloop.py b/tornado/ioloop.py index 9027ace46..d114107e3 100644 --- a/tornado/ioloop.py +++ b/tornado/ioloop.py @@ -36,7 +36,6 @@ import logging import os import select import sys -import thread import threading import time import traceback @@ -56,6 +55,11 @@ try: except ImportError: futures = None +try: + import thread # py2 +except ImportError: + import _thread as thread # py3 + from tornado.platform.auto import set_close_exec, Waker diff --git a/tornado/log.py b/tornado/log.py index 495fba91f..ff89d8490 100644 --- a/tornado/log.py +++ b/tornado/log.py @@ -36,6 +36,7 @@ import sys import time from tornado.escape import _unicode +from tornado.util import unicode_type, basestring_type try: import curses @@ -86,25 +87,25 @@ class LogFormatter(logging.Formatter): fg_color = (curses.tigetstr("setaf") or curses.tigetstr("setf") or "") if (3, 0) < sys.version_info < (3, 2, 3): - fg_color = unicode(fg_color, "ascii") + fg_color = unicode_type(fg_color, "ascii") self._colors = { - logging.DEBUG: unicode(curses.tparm(fg_color, 4), # Blue + logging.DEBUG: unicode_type(curses.tparm(fg_color, 4), # Blue "ascii"), - logging.INFO: unicode(curses.tparm(fg_color, 2), # Green + logging.INFO: unicode_type(curses.tparm(fg_color, 2), # Green "ascii"), - logging.WARNING: unicode(curses.tparm(fg_color, 3), # Yellow + logging.WARNING: unicode_type(curses.tparm(fg_color, 3), # Yellow "ascii"), - logging.ERROR: unicode(curses.tparm(fg_color, 1), # Red + logging.ERROR: unicode_type(curses.tparm(fg_color, 1), # Red "ascii"), } - self._normal = unicode(curses.tigetstr("sgr0"), "ascii") + self._normal = unicode_type(curses.tigetstr("sgr0"), "ascii") def format(self, record): try: record.message = record.getMessage() except Exception as e: record.message = "Bad message (%r): %r" % (e, record.__dict__) - assert isinstance(record.message, basestring) # guaranteed by logging + assert isinstance(record.message, basestring_type) # guaranteed by logging record.asctime = time.strftime( "%y%m%d %H:%M:%S", self.converter(record.created)) prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ diff --git a/tornado/netutil.py b/tornado/netutil.py index 8137945de..6321013f3 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -288,7 +288,7 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128, flags return sockets if hasattr(socket, 'AF_UNIX'): - def bind_unix_socket(file, mode=0600, backlog=128): + def bind_unix_socket(file, mode=int('600', 8), backlog=128): """Creates a listening unix socket. If a socket with the given name already exists, it will be deleted. diff --git a/tornado/options.py b/tornado/options.py index 64083a929..102a55467 100644 --- a/tornado/options.py +++ b/tornado/options.py @@ -68,6 +68,7 @@ import textwrap from tornado.escape import _unicode from tornado.log import define_logging_options from tornado import stack_context +from tornado.util import basestring_type class Error(Exception): @@ -171,7 +172,7 @@ class OptionParser(object): if args is None: args = sys.argv remaining = [] - for i in xrange(1, len(args)): + for i in range(1, len(args)): # All things after the last option are command line arguments if not args[i].startswith("-"): remaining = args[i:] @@ -301,7 +302,7 @@ class _Mockable(object): setattr(self._options, name, self._originals.pop(name)) class _Option(object): - def __init__(self, name, default=None, type=basestring, help=None, + def __init__(self, name, default=None, type=basestring_type, help=None, metavar=None, multiple=False, file_name=None, group_name=None, callback=None): if default is None and multiple: @@ -325,7 +326,7 @@ class _Option(object): datetime.datetime: self._parse_datetime, datetime.timedelta: self._parse_timedelta, bool: self._parse_bool, - basestring: self._parse_string, + basestring_type: self._parse_string, }.get(self.type, self.type) if self.multiple: self._value = [] diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index a289d3966..9ae0b5801 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -19,7 +19,6 @@ import re import socket import sys import time -import urlparse try: from io import BytesIO # python 3 @@ -31,6 +30,11 @@ try: except ImportError: ssl = None +try: + import urlparse # py2 +except ImportError: + import urllib.parse as urlparse # py3 + _DEFAULT_CA_CERTS = os.path.dirname(__file__) + '/ca-certificates.crt' diff --git a/tornado/template.py b/tornado/template.py index 2a31f3f21..78a523924 100644 --- a/tornado/template.py +++ b/tornado/template.py @@ -181,7 +181,6 @@ with ``{# ... #}``. from __future__ import absolute_import, division, print_function, with_statement -import cStringIO import datetime import linecache import os.path @@ -191,7 +190,12 @@ import threading from tornado import escape from tornado.log import app_log -from tornado.util import bytes_type, ObjectDict +from tornado.util import bytes_type, ObjectDict, exec_in + +try: + from cStringIO import StringIO # py2 +except ImportError: + from io import StringIO # py3 _DEFAULT_AUTOESCAPE = "xhtml_escape" _UNSET = object() @@ -254,7 +258,7 @@ class Template(object): } namespace.update(self.namespace) namespace.update(kwargs) - exec self.compiled in namespace + exec_in(self.compiled, namespace) execute = namespace["_execute"] # Clear the traceback module's cache of source data now that # we've generated a new template (mainly for this module's @@ -263,7 +267,7 @@ class Template(object): return execute() def _generate_python(self, loader, compress_whitespace): - buffer = cStringIO.StringIO() + buffer = StringIO() try: # named_blocks maps from names to _NamedBlock objects named_blocks = {} diff --git a/tornado/test/escape_test.py b/tornado/test/escape_test.py index ba40ccf8d..71a4fa1ba 100644 --- a/tornado/test/escape_test.py +++ b/tornado/test/escape_test.py @@ -123,11 +123,11 @@ linkify_tests = [ u('www.external-link.com')), ("www.external-link.com and www.internal-link.com/blogs extra", - {"extra_params": lambda(href):'class="internal"' if href.startswith("http://www.internal-link.com") else 'rel="nofollow" class="external"'}, + {"extra_params": lambda href:'class="internal"' if href.startswith("http://www.internal-link.com") else 'rel="nofollow" class="external"'}, u('www.external-link.com and www.internal-link.com/blogs extra')), ("www.external-link.com", - {"extra_params": lambda(href):' rel="nofollow" class="external" '}, + {"extra_params": lambda href:' rel="nofollow" class="external" '}, u('www.external-link.com')), ] diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py index d4bdaa10e..9018836f8 100644 --- a/tornado/test/twisted_test.py +++ b/tornado/test/twisted_test.py @@ -23,7 +23,6 @@ import os import shutil import signal import tempfile -import thread import threading try: @@ -41,6 +40,11 @@ try: except ImportError: have_twisted = False +try: + import thread # py2 +except ImportError: + import _thread as thread # py3 + from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop diff --git a/tornado/testing.py b/tornado/testing.py index 68de1a7db..63ececf94 100644 --- a/tornado/testing.py +++ b/tornado/testing.py @@ -20,7 +20,6 @@ information. from __future__ import absolute_import, division, print_function, with_statement -from cStringIO import StringIO try: from tornado.httpclient import AsyncHTTPClient from tornado.httpserver import HTTPServer @@ -46,6 +45,11 @@ import socket import sys import time +try: + from io import StringIO # py3 +except ImportError: + from cStringIO import StringIO # py2 + # Tornado's own test suite requires the updated unittest module # (either py27+ or unittest2) so tornado.test.util enforces # this requirement, but for other users of tornado.testing we want diff --git a/tornado/util.py b/tornado/util.py index 429c0c17c..3a7a0bfd2 100644 --- a/tornado/util.py +++ b/tornado/util.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function, with_statement +import sys import zlib @@ -74,37 +75,56 @@ def import_object(name): # unicode_literals" have other problems (see PEP 414). u() can be applied # to ascii strings that include \u escapes (but they must not contain # literal non-ascii characters). -if str is unicode: +if type('') is not type(b''): def b(s): return s.encode('latin1') def u(s): return s bytes_type = bytes + unicode_type = str + basestring_type = str else: def b(s): return s def u(s): return s.decode('unicode_escape') bytes_type = str - - + unicode_type = unicode + basestring_type = basestring + + +# def raise_exc_info(exc_info): +# """Re-raise an exception (with original traceback) from an exc_info tuple. + +# The argument is a ``(type, value, traceback)`` tuple as returned by +# `sys.exc_info`. +# """ +# # 2to3 isn't smart enough to convert three-argument raise +# # statements correctly in some cases. +# if isinstance(exc_info[1], exc_info[0]): +# raise exc_info[1], None, exc_info[2] +# # After 2to3: raise exc_info[1].with_traceback(exc_info[2]) +# else: +# # I think this branch is only taken for string exceptions, +# # which were removed in Python 2.6. +# raise exc_info[0], exc_info[1], exc_info[2] +# # After 2to3: raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) +if sys.version_info > (3,): + exec(""" def raise_exc_info(exc_info): - """Re-raise an exception (with original traceback) from an exc_info tuple. + raise exc_info[1].with_traceback(exc_info[2]) - The argument is a ``(type, value, traceback)`` tuple as returned by - `sys.exc_info`. - """ - # 2to3 isn't smart enough to convert three-argument raise - # statements correctly in some cases. - if isinstance(exc_info[1], exc_info[0]): - raise exc_info[1], None, exc_info[2] - # After 2to3: raise exc_info[1].with_traceback(exc_info[2]) - else: - # I think this branch is only taken for string exceptions, - # which were removed in Python 2.6. - raise exc_info[0], exc_info[1], exc_info[2] - # After 2to3: raise exc_info[0](exc_info[1]).with_traceback(exc_info[2]) +def exec_in(code, namespace): + exec(code, namespace) +""") +else: + exec(""" +def raise_exc_info(exc_info): + raise exc_info[0], exc_info[1], exc_info[2] +def exec_in(code, namespace): + exec code in namespace +""") class Configurable(object): """Base class for configurable interfaces. @@ -174,7 +194,7 @@ class Configurable(object): some parameters. """ base = cls.configurable_base() - if isinstance(impl, (unicode, bytes_type)): + if isinstance(impl, (unicode_type, bytes_type)): impl = import_object(impl) if impl is not None and not issubclass(impl, cls): raise ValueError("Invalid subclass of %s" % cls) diff --git a/tornado/web.py b/tornado/web.py index 17758919b..bc6d944f4 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -51,7 +51,6 @@ back to the main thread before finishing the request. from __future__ import absolute_import, division, print_function, with_statement -import Cookie import base64 import binascii import calendar @@ -61,7 +60,6 @@ import functools import gzip import hashlib import hmac -import httplib import itertools import mimetypes import os.path @@ -74,10 +72,10 @@ import tornado import traceback import types import urllib -import urlparse import uuid from tornado import escape +from tornado import httputil from tornado import locale from tornado.log import access_log, app_log, gen_log from tornado import stack_context @@ -90,6 +88,15 @@ try: except ImportError: from cStringIO import StringIO as BytesIO # python 2 +try: + import Cookie # py2 +except ImportError: + import http.cookies as Cookie # py3 + +try: + import urlparse # py2 +except ImportError: + import urllib.parse as urlparse # py3 class RequestHandler(object): """Subclass this class and define get() or post() to make a handler. @@ -233,7 +240,7 @@ class RequestHandler(object): self.set_header("Connection", "Keep-Alive") self._write_buffer = [] self._status_code = 200 - self._reason = httplib.responses[200] + self._reason = httputil.responses[200] def set_default_headers(self): """Override this to set HTTP headers at the beginning of the request. @@ -258,7 +265,7 @@ class RequestHandler(object): self._reason = escape.native_str(reason) else: try: - self._reason = httplib.responses[status_code] + self._reason = httputil.responses[status_code] except KeyError: raise ValueError("unknown status code %d", status_code) @@ -861,7 +868,7 @@ class RequestHandler(object): score = 1.0 locales.append((parts[0], score)) if locales: - locales.sort(key=lambda (l, s): s, reverse=True) + locales.sort(key=lambda pair: pair[1], reverse=True) codes = [l[0] for l in locales] return locale.get(*codes) return locale.get(default) @@ -1101,7 +1108,7 @@ class RequestHandler(object): format = "%d %s: " + e.log_message args = [e.status_code, self._request_summary()] + list(e.args) gen_log.warning(format, *args) - if e.status_code not in httplib.responses and not e.reason: + if e.status_code not in httputil.responses and not e.reason: gen_log.error("Bad HTTP status code: %d", e.status_code) self.send_error(500, exc_info=sys.exc_info()) else: @@ -1510,7 +1517,7 @@ class HTTPError(Exception): def __str__(self): message = "HTTP %d: %s" % ( self.status_code, - self.reason or httplib.responses.get(self.status_code, 'Unknown')) + self.reason or httputil.responses.get(self.status_code, 'Unknown')) if self.log_message: return message + " (" + (self.log_message % self.args) + ")" else: diff --git a/tornado/wsgi.py b/tornado/wsgi.py index 77163ea0c..3f38f67a3 100644 --- a/tornado/wsgi.py +++ b/tornado/wsgi.py @@ -31,8 +31,6 @@ provides WSGI support in two ways: from __future__ import absolute_import, division, print_function, with_statement -import Cookie -import httplib import sys import time import tornado @@ -43,19 +41,23 @@ from tornado import httputil from tornado.log import access_log from tornado import web from tornado.escape import native_str, utf8, parse_qs_bytes -from tornado.util import b, bytes_type +from tornado.util import b, bytes_type, unicode_type try: from io import BytesIO # python 3 except ImportError: from cStringIO import StringIO as BytesIO # python 2 +try: + import Cookie # py2 +except ImportError: + import http.cookies as Cookie # py3 # PEP 3333 specifies that WSGI on python 3 generally deals with byte strings # that are smuggled inside objects of type unicode (via the latin1 encoding). # These functions are like those in the tornado.escape module, but defined # here to minimize the temptation to use them in non-wsgi contexts. -if str is unicode: +if str is unicode_type: def to_wsgi_str(s): assert isinstance(s, bytes_type) return s.decode('latin1')