Modify xhtml_unescape to handle hex numeric refs

While the existing code handles references like   properly,
it doesn't handle the valid case where the number is specified
in hex. For example 'foo bar', 'foo bar' and 'foo bar'
should all decode to 'foo bar'. The changes in escape.py check
for the 'x' and decode appropriately. Also added unit tests for
unescaping strings with numeric references.

diff --git a/tornado/escape.py b/tornado/escape.py
index 2852cf517bfa5de40c2bb15ff8fed31ff1cf7052..2f04b4683ae7cccecbbb2571d354c4a40eea3a9d 100644 (file)
--- a/tornado/escape.py
+++ b/tornado/escape.py
@@ -378,7 +378,10 @@ def linkify(text, shorten=False, extra_params="",
 def _convert_entity(m):
     if m.group(1) == "#":
         try:
-            return unichr(int(m.group(2)))
+            if m.group(2)[:1].lower() == 'x':
+                return unichr(int(m.group(2)[1:], 16))
+            else:
+                return unichr(int(m.group(2)))
         except ValueError:
             return "&#%s;" % m.group(2)
     try:

diff --git a/tornado/test/escape_test.py b/tornado/test/escape_test.py
index 98a23463890c0b8633bc966075253196df8ecadd..65765b68aa31fd90205d38fe22dcbf3b081ca381 100644 (file)
--- a/tornado/test/escape_test.py
+++ b/tornado/test/escape_test.py
@@ -154,6 +154,19 @@ class EscapeTestCase(unittest.TestCase):
             self.assertEqual(utf8(xhtml_escape(unescaped)), utf8(escaped))
             self.assertEqual(utf8(unescaped), utf8(xhtml_unescape(escaped)))
 
+    def test_xhtml_unescape_numeric(self):
+        tests = [
+            ('foo bar', 'foo bar'),
+            ('foo bar', 'foo bar'),
+            ('foo bar', 'foo bar'),
+            ('foo઼bar', u('foo\u0abcbar')),
+            ('foo&#xyz;bar', 'foo&#xyz;bar'),  # invalid encoding
+            ('foo&#;bar', 'foo&#;bar'),        # invalid encoding
+            ('foo&#x;bar', 'foo&#x;bar'),      # invalid encoding
+        ]
+        for escaped, unescaped in tests:
+            self.assertEqual(unescaped, xhtml_unescape(escaped))
+
     def test_url_escape_unicode(self):
         tests = [
             # byte strings are passed through as-is