httpclient: Improve non-ascii characters in HTTP auth

- Use NFC as required by the spec
- Fix string-concat exceptions on py2
- Test both curl and simple http clients for basic auth
- Do the same for proxy auth in curl

diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py
index c4ddbbbb208e1609f0fbd0c6a3cdf96979dc118d..ef98225cb95c12b1c8f8efcb8733f834ee010972 100644 (file)
--- a/tornado/curl_httpclient.py
+++ b/tornado/curl_httpclient.py
@@ -348,8 +348,8 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             curl.setopt(pycurl.PROXY, request.proxy_host)
             curl.setopt(pycurl.PROXYPORT, request.proxy_port)
             if request.proxy_username:
-                credentials = '%s:%s' % (request.proxy_username,
-                                         request.proxy_password)
+                credentials = httputil.encode_username_password(request.proxy_username,
+                                                                request.proxy_password)
                 curl.setopt(pycurl.PROXYUSERPWD, credentials)
 
             if (request.proxy_auth_mode is None or
@@ -441,8 +441,6 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
                 curl.setopt(pycurl.INFILESIZE, len(request.body or ''))
 
         if request.auth_username is not None:
-            userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')
-
             if request.auth_mode is None or request.auth_mode == "basic":
                 curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
             elif request.auth_mode == "digest":
@@ -450,7 +448,9 @@ class CurlAsyncHTTPClient(AsyncHTTPClient):
             else:
                 raise ValueError("Unsupported auth_mode %s" % request.auth_mode)
 
-            curl.setopt(pycurl.USERPWD, utf8(userpwd))
+            userpwd = httputil.encode_username_password(request.auth_username,
+                                                        request.auth_password)
+            curl.setopt(pycurl.USERPWD, userpwd)
             curl_log.debug("%s %s (username: %r)", request.method, request.url,
                            request.auth_username)
         else:

diff --git a/tornado/httputil.py b/tornado/httputil.py
index 22a64c311bef6dff94926277859d39d861054f65..d1ace5a8636165bdbfd94dbcfcc3d5bcb6cb3353 100644 (file)
--- a/tornado/httputil.py
+++ b/tornado/httputil.py
@@ -29,11 +29,12 @@ import email.utils
 import numbers
 import re
 import time
+import unicodedata
 import warnings
 
 from tornado.escape import native_str, parse_qs_bytes, utf8
 from tornado.log import gen_log
-from tornado.util import ObjectDict, PY3
+from tornado.util import ObjectDict, PY3, unicode_type
 
 if PY3:
     import http.cookies as Cookie
@@ -949,6 +950,20 @@ def _encode_header(key, pdict):
     return '; '.join(out)
 
 
+def encode_username_password(username, password):
+    """Encodes a username/password pair in the format used by HTTP auth.
+
+    The return value is a byte string in the form ``username:password``.
+
+    .. versionadded:: 5.1
+    """
+    if isinstance(username, unicode_type):
+        username = unicodedata.normalize('NFC', username)
+    if isinstance(password, unicode_type):
+        password = unicodedata.normalize('NFC', password)
+    return utf8(username) + b":" + utf8(password)
+
+
 def doctests():
     import doctest
     return doctest.DocTestSuite()

diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py
index 4df4898a9705404aa6782fc5c0288975309912fa..35c719362de37df9a9ccb458701ef47d72498066 100644 (file)
--- a/tornado/simple_httpclient.py
+++ b/tornado/simple_httpclient.py
@@ -1,6 +1,6 @@
 from __future__ import absolute_import, division, print_function
 
-from tornado.escape import utf8, _unicode
+from tornado.escape import _unicode
 from tornado import gen
 from tornado.httpclient import HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
 from tornado import httputil
@@ -308,9 +308,9 @@ class _HTTPConnection(httputil.HTTPMessageDelegate):
                     if self.request.auth_mode not in (None, "basic"):
                         raise ValueError("unsupported auth_mode %s",
                                          self.request.auth_mode)
-                    auth = utf8(username) + b":" + utf8(password)
-                    self.request.headers["Authorization"] = (b"Basic " +
-                                                             base64.b64encode(auth))
+                    self.request.headers["Authorization"] = (
+                        b"Basic " + base64.b64encode(
+                            httputil.encode_username_password(username, password)))
                 if self.request.user_agent:
                     self.request.headers["User-Agent"] = self.request.user_agent
                 if not self.request.allow_nonstandard_methods:

diff --git a/tornado/test/httpclient_test.py b/tornado/test/httpclient_test.py
index 60c8f490d31e230036bb1cf841343a912c26be07..fb8b12d57f8e5bc22dc9eba9071f04faa332fe7d 100644 (file)
--- a/tornado/test/httpclient_test.py
+++ b/tornado/test/httpclient_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function
 
 import base64
@@ -8,6 +9,7 @@ import sys
 import threading
 import datetime
 from io import BytesIO
+import unicodedata
 
 from tornado.escape import utf8, native_str
 from tornado import gen
@@ -237,6 +239,7 @@ Transfer-Encoding: chunked
         self.assertIs(exc_info[0][0], ZeroDivisionError)
 
     def test_basic_auth(self):
+        # This test data appears in section 2 of RFC 7617.
         self.assertEqual(self.fetch("/auth", auth_username="Aladdin",
                                     auth_password="open sesame").body,
                          b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
@@ -247,6 +250,20 @@ Transfer-Encoding: chunked
                                     auth_mode="basic").body,
                          b"Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==")
 
+    def test_basic_auth_unicode(self):
+        # This test data appears in section 2.1 of RFC 7617.
+        self.assertEqual(self.fetch("/auth", auth_username="test",
+                                    auth_password="123£").body,
+                         b"Basic dGVzdDoxMjPCow==")
+
+        # The standard mandates NFC. Give it a decomposed username
+        # and ensure it is normalized to composed form.
+        username = unicodedata.normalize("NFD", u"josé")
+        self.assertEqual(self.fetch("/auth",
+                                    auth_username=username,
+                                    auth_password="səcrət").body,
+                         b"Basic am9zw6k6c8mZY3LJmXQ=")
+
     def test_unsupported_auth_mode(self):
         # curl and simple clients handle errors a bit differently; the
         # important thing is that they don't fall back to basic auth