lib: unicode-data - Add IDNA code point data

diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index 52e9bc38f49d3df5a1348c12fdd6486046024dfd..a9dc13bedf31a10be5b4fb54334cdc40b50295a1 100644 (file)
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -30,7 +30,9 @@ UCD_FILES = \
        $(UCD_DIR)/PropList.txt \
        $(UCD_DIR)/SpecialCasing.txt \
        $(UCD_DIR)/UnicodeData.txt \
-       $(UCD_DIR)/WordBreakProperty.txt
+       $(UCD_DIR)/WordBreakProperty.txt \
+       $(UCD_DIR)/IdnaMappingTable.txt \
+       $(UCD_DIR)/IdnaTestV2.txt
 
 EXTRA_DIST = \
        unicode-data-tables.c \
@@ -87,6 +89,10 @@ $(UCD_DIR)/UnicodeData.txt:
        $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/UnicodeData.txt
 $(UCD_DIR)/WordBreakProperty.txt:
        $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/WordBreakProperty.txt
+$(UCD_DIR)/IdnaMappingTable.txt:
+       $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/IdnaMappingTable.txt
+$(UCD_DIR)/IdnaTestV2.txt:
+       $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/IdnaTestV2.txt
 
 $(srcdir)/unicode-data-tables.c $(srcdir)/unicode-data-tables.h \
        $(srcdir)/unicode-data-types.c $(srcdir)/unicode-data-types.h &: \

diff --git a/src/lib/test-unicode-data.c b/src/lib/test-unicode-data.c
index 1beda5d7c9d391d1e3ecf296aa7fb30bafed72aa..57ba11e039953b7d8b7b45b8ed9a5b6e295f77c5 100644 (file)
--- a/src/lib/test-unicode-data.c
+++ b/src/lib/test-unicode-data.c
@@ -17,6 +17,7 @@
 #define UCD_SPECIAL_CASING_TXT "SpecialCasing.txt"
 #define UCD_UNICODE_DATA_TXT "UnicodeData.txt"
 #define UCD_WORD_BREAK_PROPERTY_TXT "WordBreakProperty.txt"
+#define UCD_IDNA_MAPPING_TABLE_TXT "IdnaMappingTable.txt"
 
 static bool
 parse_prop_file_line(const char *line, const char *file, unsigned int line_num,
@@ -592,6 +593,70 @@ test_word_break_property_line(const char *line, unsigned int line_num)
        }
 }
 
+static void
+test_idna_mapping_table_line(const char *line, unsigned int line_num)
+{
+       if (*line == '\0')
+               return;
+
+       const char *const *columns = t_strsplit(line, ";");
+       size_t num_columns = str_array_length(columns);
+
+       if (num_columns == 0)
+               return;
+       if (num_columns < 2) {
+               test_failed(t_strdup_printf(
+                       "Invalid data at %s:%u",
+                       UCD_IDNA_MAPPING_TABLE_TXT, line_num));
+               return;
+       }
+
+       uint32_t cp_first, cp_last, cp;
+       const char *status, *value;
+
+       if (!parse_prop_file_line(line, UCD_IDNA_MAPPING_TABLE_TXT, line_num,
+                                 &cp_first, &cp_last, &status, &value))
+               return;
+
+       for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) {
+               const struct unicode_code_point_data *cp_data =
+                       unicode_code_point_get_data(cp);
+
+               switch (cp_data->idna_status) {
+               case UNICODE_IDNA_STATUS_DISALLOWED:
+                       test_assert_strcmp_idx(status, "disallowed", line_num);
+                       break;
+               case UNICODE_IDNA_STATUS_VALID:
+                       test_assert_strcmp_idx(status, "valid", line_num);
+                       break;
+               case UNICODE_IDNA_STATUS_IGNORED:
+                       test_assert_strcmp_idx(status, "ignored", line_num);
+                       break;
+               case UNICODE_IDNA_STATUS_MAPPED:
+                       test_assert_strcmp_idx(status, "mapped", line_num);
+                       break;
+               case UNICODE_IDNA_STATUS_DEVIATION:
+                       test_assert_strcmp_idx(status, "deviation", line_num);
+                       break;
+               }
+
+               if (strcmp(status, "mapping") != 0 &&
+                   strcmp(status, "deviation") != 0)
+                       continue;
+
+               const char *mapping = t_str_trim(value, " ");
+               const char *const *map = t_strsplit(mapping, " ");
+
+               /* Check data */
+
+               const uint32_t *idna_map =
+                       &unicode_idna_mappings[cp_data->idna_mapping_offset];
+               unsigned int idna_map_len = cp_data->idna_mapping_length;
+
+               test_case_mapping(cp, map, idna_map, idna_map_len);
+       }
+}
+
 static void
 test_ucd_file(const char *filename,
              void (*test_line)(const char *line, unsigned int line_num))
@@ -651,4 +716,6 @@ void test_unicode_data(void)
        test_ucd_file(UCD_UNICODE_DATA_TXT, test_unicode_data_line);
        test_ucd_file(UCD_WORD_BREAK_PROPERTY_TXT,
                      test_word_break_property_line);
+       test_ucd_file(UCD_IDNA_MAPPING_TABLE_TXT,
+                     test_idna_mapping_table_line);
 }

diff --git a/src/lib/unicode-data-static.h b/src/lib/unicode-data-static.h
index 9a2575c6230323354bcb03131f9e6656570988c1..481dc8ace6c628b954ab980e1d0fb0fcc4d45dea 100644 (file)
--- a/src/lib/unicode-data-static.h
+++ b/src/lib/unicode-data-static.h
@@ -122,8 +122,26 @@ enum unicode_indic_conjunct_break {
        UNICODE_INDIC_CONJUNCT_BREAK_EXTEND,
 };
 
+/* For each code point in Unicode, the IDNA Mapping Table provides one of the
+   following Status values: */
+enum unicode_idna_status {
+       /* disallowed: the code point is not allowed. */
+       UNICODE_IDNA_STATUS_DISALLOWED = 0,
+       /* valid: the code point is valid, and not modified. */
+       UNICODE_IDNA_STATUS_VALID,
+       /* ignored: the code point is removed:
+        * this is equivalent to mapping the code point to an empty string. */
+       UNICODE_IDNA_STATUS_IGNORED,
+       /* mapped: the code point is replaced in the string by the value for the
+          mapping. */
+       UNICODE_IDNA_STATUS_MAPPED,
+       /* deviation: the code point is either mapped or valid, depending on
+          whether the processing is transitional or not. */
+       UNICODE_IDNA_STATUS_DEVIATION,
+};
+
 struct unicode_code_point_data {
-       uint8_t general_category; // Not yet used
+       uint8_t general_category;
        uint8_t canonical_combining_class;
        uint8_t nf_quick_check;
 
@@ -138,6 +156,9 @@ struct unicode_code_point_data {
        uint8_t lowercase_mapping_length;
        uint8_t casefold_mapping_length;
 
+       uint8_t idna_status:3;
+       uint8_t idna_mapping_length:5;
+
        uint16_t decomposition_first_offset;
        uint16_t decomposition_full_offset;
        uint16_t decomposition_full_k_offset;
@@ -147,6 +168,8 @@ struct unicode_code_point_data {
        uint16_t lowercase_mapping_offset;
        uint16_t casefold_mapping_offset;
 
+       uint16_t idna_mapping_offset;
+
        uint32_t simple_titlecase_mapping;
 
        uint8_t indic_conjunct_break:3;

diff --git a/src/lib/unicode-data.h b/src/lib/unicode-data.h
index ed66c496714bd1c7daf10c805623680217fdacd4..7041f29bb24bcd1d101229221485d31a873fa3ec 100644 (file)
--- a/src/lib/unicode-data.h
+++ b/src/lib/unicode-data.h
@@ -102,6 +102,15 @@ static inline bool unicode_code_point_is_assigned(uint32_t cp)
        return unicode_code_point_data_is_assigned(cp_data);
 }
 
+static inline bool
+unicode_code_point_data_general_category_in(
+       const struct unicode_code_point_data *cp_data, uint8_t group)
+{
+       return (group ==
+               (cp_data->general_category &
+                UNICODE_GENERAL_CATEGORY_GROUP_MASK));
+}
+
 static inline size_t
 unicode_code_point_get_full_decomposition(uint32_t cp, bool canonical,
                                          const uint32_t **decomp_r)

diff --git a/src/lib/unicode-ucd-compile.py b/src/lib/unicode-ucd-compile.py
index 6c963ee668d3b8cb0fbd572c8c37e096b9676b62..3c39755224761b44cd3d3b10f8e4857a2fa30563 100755 (executable)
--- a/src/lib/unicode-ucd-compile.py
+++ b/src/lib/unicode-ucd-compile.py
@@ -45,6 +45,9 @@ ud_compositions_max_per_starter = 0
 ud_case_mappings = []
 ud_case_mapping_max_length = 0
 
+ud_idna_mappings = []
+ud_idna_mapping_max_length = 0
+
 
 class UCDFileOpen:
     def __init__(self, filename):
@@ -801,6 +804,59 @@ def read_ucd_files():
                 cpd.pb_wb_extendnumlet = True
                 CodePointRange(cprng[0], cprng[1], cpd)
 
+    # IdnaMappingTable.txt
+    with UCDFileOpen("IdnaMappingTable.txt") as ucd:
+        line_num = 0
+        for line in ucd.fd:
+            line_num = line_num + 1
+            data = line.split("#")
+            line = data[0].strip()
+            if len(line) == 0:
+                continue
+
+            cols = line.split(";")
+            if len(cols) < 2:
+                die(f"{ucd}:{line_num}: Missing columns")
+
+            cprng = parse_cp_range(cols[0])
+            if cprng is None:
+                continue
+
+            status_label = cols[1].strip()
+            status = None
+            mapping = ""
+            if len(cols) >= 3:
+                mapping = cols[2].strip()
+
+            if status_label == "disallowed":
+                continue
+            elif status_label == "valid":
+                status = "UNICODE_IDNA_STATUS_VALID"
+            elif status_label == "ignored":
+                status = "UNICODE_IDNA_STATUS_IGNORED"
+            elif status_label == "mapped":
+                status = "UNICODE_IDNA_STATUS_MAPPED"
+            elif status_label == "deviation":
+                status = "UNICODE_IDNA_STATUS_DEVIATION"
+            else:
+                continue
+
+            cpd = CodePointData()
+            cpd.idna_status = status
+
+            codes_hex = mapping.split(" ")
+            if len(mapping) > 0 and len(codes_hex) > 0:
+                first_code_hex = codes_hex[0].strip()
+                first_code = int(first_code_hex, 16)
+                if len(codes_hex) > 1 or first_code != cp:
+                    codes = []
+                    for code_hex in codes_hex:
+                        codes.append(int(code_hex, 16))
+
+                    cpd.idna_mapping = codes
+
+            CodePointRange(cprng[0], cprng[1], cpd)
+
 
 def resolve_case_mappings():
     global ud_codepoints
@@ -1088,6 +1144,29 @@ def derive_canonical_compositions():
         ud_composition_primaries = ud_composition_primaries + [p[1] for p in mp]
 
 
+def resolve_idna_mappings():
+    global ud_codepoints
+    global ud_idna_mappings
+    global ud_idna_mapping_max_length
+
+    for cpr in ud_codepoints:
+        if cpr.cp_last > cpr.cp_first:
+            # No mappings in ranges expected, ever
+            continue
+        cp = cpr.cp_first
+        cpd = cpr.data
+
+        idna_codes = []
+        if hasattr(cpd, "idna_mapping"):
+            idna_codes = cpd.idna_mapping
+        if len(idna_codes) > 0 and (len(idna_codes) > 1 or idna_codes[0] != cp):
+            cpd.idna_mapping_offset = len(ud_idna_mappings)
+            cpd.idna_mapping_length = len(idna_codes)
+            ud_idna_mappings = ud_idna_mappings + idna_codes
+        if len(idna_codes) > ud_idna_mapping_max_length:
+            ud_idna_mapping_max_length = len(idna_codes)
+
+
 def create_cp_range_index():
     global ud_codepoints
     global ud_codepoints_index
@@ -1281,6 +1360,7 @@ def write_tables_h():
     global ud_decomposition_max_length
     global ud_compositions_max_per_starter
     global ud_case_mapping_max_length
+    global ud_idna_mapping_max_length
 
     orig_stdout = sys.stdout
 
@@ -1301,6 +1381,7 @@ def write_tables_h():
             % ud_compositions_max_per_starter
         )
         print("#define UNICODE_CASE_MAPPING_MAX_LENGTH %s" % ud_case_mapping_max_length)
+        print("#define UNICODE_IDNA_MAX_MAPPING_LENGTH %s" % ud_idna_mapping_max_length)
         print("")
         print("extern const struct unicode_code_point_data unicode_code_points[];")
         print("")
@@ -1316,6 +1397,8 @@ def write_tables_h():
         print("")
         print("extern const uint32_t unicode_case_mappings[];")
         print("")
+        print("extern const uint32_t unicode_idna_mappings[];")
+        print("")
         print("#endif")
 
     sys.stdout = orig_stdout
@@ -1440,6 +1523,11 @@ def write_tables_c_cpd(cpd):
             "\t\t.simple_titlecase_mapping = 0x%04X,"
             % cpd.simple_titlecase_mapping
         )
+    if hasattr(cpd, "idna_status"):
+        print("\t\t.idna_status = %s," % cpd.idna_status)
+    if hasattr(cpd, "idna_mapping_length") and cpd.idna_mapping_length > 0:
+        print("\t\t.idna_mapping_length = %s," % cpd.idna_mapping_length)
+        print("\t\t.idna_mapping_offset = %s," % cpd.idna_mapping_offset)
     if hasattr(cpd, "indic_conjunct_break"):
         print(
             "\t\t.indic_conjunct_break = %s,"
@@ -1780,6 +1868,10 @@ def write_tables_c():
         print_list(ud_case_mappings)
         print(",")
         print("};")
+        print("")
+        print("const uint32_t unicode_idna_mappings[] = {")
+        print_list(ud_idna_mappings)
+        print("};")
 
     sys.stdout = orig_stdout
 
@@ -1890,6 +1982,7 @@ def main():
     resolve_case_mappings()
     expand_decompositions()
     derive_canonical_compositions()
+    resolve_idna_mappings()
 
     create_cp_index_tables()