gh-130197: pygettext: Test the --escape option (GH-131902)

diff --git a/Lib/test/test_tools/i18n_data/ascii-escapes.pot b/Lib/test/test_tools/i18n_data/ascii-escapes.pot
new file mode 100644 (file)
index 0000000..f8e0f53
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/ascii-escapes.pot
@@ -0,0 +1,51 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#. Special characters that are always escaped in the POT file
+#: escapes.py:5
+msgid ""
+"\"\t\n"
+"\r\\"
+msgstr ""
+
+#. All ascii characters 0-31
+#: escapes.py:8
+msgid ""
+"\000\001\002\003\004\005\006\007\010\t\n"
+"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
+msgstr ""
+
+#. All ascii characters 32-126
+#: escapes.py:13
+msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+msgstr ""
+
+#. ascii char 127
+#: escapes.py:17
+msgid "\177"
+msgstr ""
+
+#. some characters in the 128-255 range
+#: escapes.py:20
+msgid "\80   ÿ"
+msgstr ""
+
+#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
+#: escapes.py:23
+msgid "α ㄱ 𓂀"
+msgstr ""
+

diff --git a/Lib/test/test_tools/i18n_data/escapes.pot b/Lib/test/test_tools/i18n_data/escapes.pot
new file mode 100644 (file)
index 0000000..4dfac0f
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/escapes.pot
@@ -0,0 +1,51 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#. Special characters that are always escaped in the POT file
+#: escapes.py:5
+msgid ""
+"\"\t\n"
+"\r\\"
+msgstr ""
+
+#. All ascii characters 0-31
+#: escapes.py:8
+msgid ""
+"\000\001\002\003\004\005\006\007\010\t\n"
+"\013\014\r\016\017\020\021\022\023\024\025\026\027\030\031\032\033\034\035\036\037"
+msgstr ""
+
+#. All ascii characters 32-126
+#: escapes.py:13
+msgid " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
+msgstr ""
+
+#. ascii char 127
+#: escapes.py:17
+msgid "\177"
+msgstr ""
+
+#. some characters in the 128-255 range
+#: escapes.py:20
+msgid "\302\200 \302\240 \303\277"
+msgstr ""
+
+#. some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
+#: escapes.py:23
+msgid "\316\261 \343\204\261 \360\223\202\200"
+msgstr ""
+

diff --git a/Lib/test/test_tools/i18n_data/escapes.py b/Lib/test/test_tools/i18n_data/escapes.py
new file mode 100644 (file)
index 0000000..900bd97
--- /dev/null
+++ b/Lib/test/test_tools/i18n_data/escapes.py
@@ -0,0 +1,23 @@
+import gettext as _
+
+
+# Special characters that are always escaped in the POT file
+_('"\t\n\r\\')
+
+# All ascii characters 0-31
+_('\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n'
+  '\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15'
+  '\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f')
+
+# All ascii characters 32-126
+_(' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+  '[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+
+# ascii char 127
+_('\x7f')
+
+# some characters in the 128-255 range
+_('\x80 \xa0 ÿ')
+
+# some characters >= 256 encoded as 2, 3 and 4 bytes, respectively
+_('α ㄱ 𓂀')

diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py
index 2ba086109d30abf3805e031a2e745b02b2f267f5..66c3307742322920aad44260e9545b2c70f56b6b 100644 (file)
--- a/Lib/test/test_tools/test_i18n.py
+++ b/Lib/test/test_tools/test_i18n.py
@@ -526,11 +526,21 @@ def extract_from_snapshots():
         'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
                                '--keyword=pfoo:1c,2',
                                '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'),
+        # == Test character escaping
+        # Escape ascii and unicode:
+        'escapes.py': ('--escape', '--add-comments='),
+        # Escape only ascii and let unicode pass through:
+        ('escapes.py', 'ascii-escapes.pot'): ('--add-comments=',),
     }
 
     for filename, args in snapshots.items():
-        input_file = DATA_DIR / filename
-        output_file = input_file.with_suffix('.pot')
+        if isinstance(filename, tuple):
+            filename, output_file = filename
+            output_file = DATA_DIR / output_file
+            input_file = DATA_DIR / filename
+        else:
+            input_file = DATA_DIR / filename
+            output_file = input_file.with_suffix('.pot')
         contents = input_file.read_bytes()
         with temp_cwd(None):
             Path(input_file.name).write_bytes(contents)

diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py
index 0ec257020918b13ac70635753b57034e9e285b97..f0ee2ea386f18fa0c1a8538cb8c6bdda6023a8a7 100755 (executable)
--- a/Tools/i18n/pygettext.py
+++ b/Tools/i18n/pygettext.py
@@ -188,7 +188,7 @@ def make_escapes(pass_nonascii):
     global escapes, escape
     if pass_nonascii:
         # Allow non-ascii characters to pass through so that e.g. 'msgid
-        # "Höhe"' would result not result in 'msgid "H\366he"'.  Otherwise we
+        # "Höhe"' would not result in 'msgid "H\366he"'.  Otherwise we
         # escape any character outside the 32..126 range.
         mod = 128
         escape = escape_ascii