From: Stephan Bosch Date: Mon, 24 Mar 2025 01:36:08 +0000 (+0100) Subject: lib: unicode-data - Add IDNA code point data X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8f3bc2eea4df8a17b7e6bd8945d5b84f0cf00406;p=thirdparty%2Fdovecot%2Fcore.git lib: unicode-data - Add IDNA code point data --- diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index 52e9bc38f4..a9dc13bedf 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -30,7 +30,9 @@ UCD_FILES = \ $(UCD_DIR)/PropList.txt \ $(UCD_DIR)/SpecialCasing.txt \ $(UCD_DIR)/UnicodeData.txt \ - $(UCD_DIR)/WordBreakProperty.txt + $(UCD_DIR)/WordBreakProperty.txt \ + $(UCD_DIR)/IdnaMappingTable.txt \ + $(UCD_DIR)/IdnaTestV2.txt EXTRA_DIST = \ unicode-data-tables.c \ @@ -87,6 +89,10 @@ $(UCD_DIR)/UnicodeData.txt: $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/UnicodeData.txt $(UCD_DIR)/WordBreakProperty.txt: $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/WordBreakProperty.txt +$(UCD_DIR)/IdnaMappingTable.txt: + $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/IdnaMappingTable.txt +$(UCD_DIR)/IdnaTestV2.txt: + $(AM_V_at)test -f $@ || $(WGET) -nv -O $@ $(UCD_URL)/IdnaTestV2.txt $(srcdir)/unicode-data-tables.c $(srcdir)/unicode-data-tables.h \ $(srcdir)/unicode-data-types.c $(srcdir)/unicode-data-types.h &: \ diff --git a/src/lib/test-unicode-data.c b/src/lib/test-unicode-data.c index 1beda5d7c9..57ba11e039 100644 --- a/src/lib/test-unicode-data.c +++ b/src/lib/test-unicode-data.c @@ -17,6 +17,7 @@ #define UCD_SPECIAL_CASING_TXT "SpecialCasing.txt" #define UCD_UNICODE_DATA_TXT "UnicodeData.txt" #define UCD_WORD_BREAK_PROPERTY_TXT "WordBreakProperty.txt" +#define UCD_IDNA_MAPPING_TABLE_TXT "IdnaMappingTable.txt" static bool parse_prop_file_line(const char *line, const char *file, unsigned int line_num, @@ -592,6 +593,70 @@ test_word_break_property_line(const char *line, unsigned int line_num) } } +static void +test_idna_mapping_table_line(const char *line, unsigned int line_num) +{ + if (*line == '\0') + return; + + const char *const *columns = t_strsplit(line, ";"); + size_t num_columns = str_array_length(columns); + + if (num_columns == 0) + return; + if (num_columns < 2) { + test_failed(t_strdup_printf( + "Invalid data at %s:%u", + UCD_IDNA_MAPPING_TABLE_TXT, line_num)); + return; + } + + uint32_t cp_first, cp_last, cp; + const char *status, *value; + + if (!parse_prop_file_line(line, UCD_IDNA_MAPPING_TABLE_TXT, line_num, + &cp_first, &cp_last, &status, &value)) + return; + + for (cp = cp_first; cp <= cp_last && !test_has_failed(); cp++) { + const struct unicode_code_point_data *cp_data = + unicode_code_point_get_data(cp); + + switch (cp_data->idna_status) { + case UNICODE_IDNA_STATUS_DISALLOWED: + test_assert_strcmp_idx(status, "disallowed", line_num); + break; + case UNICODE_IDNA_STATUS_VALID: + test_assert_strcmp_idx(status, "valid", line_num); + break; + case UNICODE_IDNA_STATUS_IGNORED: + test_assert_strcmp_idx(status, "ignored", line_num); + break; + case UNICODE_IDNA_STATUS_MAPPED: + test_assert_strcmp_idx(status, "mapped", line_num); + break; + case UNICODE_IDNA_STATUS_DEVIATION: + test_assert_strcmp_idx(status, "deviation", line_num); + break; + } + + if (strcmp(status, "mapping") != 0 && + strcmp(status, "deviation") != 0) + continue; + + const char *mapping = t_str_trim(value, " "); + const char *const *map = t_strsplit(mapping, " "); + + /* Check data */ + + const uint32_t *idna_map = + &unicode_idna_mappings[cp_data->idna_mapping_offset]; + unsigned int idna_map_len = cp_data->idna_mapping_length; + + test_case_mapping(cp, map, idna_map, idna_map_len); + } +} + static void test_ucd_file(const char *filename, void (*test_line)(const char *line, unsigned int line_num)) @@ -651,4 +716,6 @@ void test_unicode_data(void) test_ucd_file(UCD_UNICODE_DATA_TXT, test_unicode_data_line); test_ucd_file(UCD_WORD_BREAK_PROPERTY_TXT, test_word_break_property_line); + test_ucd_file(UCD_IDNA_MAPPING_TABLE_TXT, + test_idna_mapping_table_line); } diff --git a/src/lib/unicode-data-static.h b/src/lib/unicode-data-static.h index 9a2575c623..481dc8ace6 100644 --- a/src/lib/unicode-data-static.h +++ b/src/lib/unicode-data-static.h @@ -122,8 +122,26 @@ enum unicode_indic_conjunct_break { UNICODE_INDIC_CONJUNCT_BREAK_EXTEND, }; +/* For each code point in Unicode, the IDNA Mapping Table provides one of the + following Status values: */ +enum unicode_idna_status { + /* disallowed: the code point is not allowed. */ + UNICODE_IDNA_STATUS_DISALLOWED = 0, + /* valid: the code point is valid, and not modified. */ + UNICODE_IDNA_STATUS_VALID, + /* ignored: the code point is removed: + * this is equivalent to mapping the code point to an empty string. */ + UNICODE_IDNA_STATUS_IGNORED, + /* mapped: the code point is replaced in the string by the value for the + mapping. */ + UNICODE_IDNA_STATUS_MAPPED, + /* deviation: the code point is either mapped or valid, depending on + whether the processing is transitional or not. */ + UNICODE_IDNA_STATUS_DEVIATION, +}; + struct unicode_code_point_data { - uint8_t general_category; // Not yet used + uint8_t general_category; uint8_t canonical_combining_class; uint8_t nf_quick_check; @@ -138,6 +156,9 @@ struct unicode_code_point_data { uint8_t lowercase_mapping_length; uint8_t casefold_mapping_length; + uint8_t idna_status:3; + uint8_t idna_mapping_length:5; + uint16_t decomposition_first_offset; uint16_t decomposition_full_offset; uint16_t decomposition_full_k_offset; @@ -147,6 +168,8 @@ struct unicode_code_point_data { uint16_t lowercase_mapping_offset; uint16_t casefold_mapping_offset; + uint16_t idna_mapping_offset; + uint32_t simple_titlecase_mapping; uint8_t indic_conjunct_break:3; diff --git a/src/lib/unicode-data.h b/src/lib/unicode-data.h index ed66c49671..7041f29bb2 100644 --- a/src/lib/unicode-data.h +++ b/src/lib/unicode-data.h @@ -102,6 +102,15 @@ static inline bool unicode_code_point_is_assigned(uint32_t cp) return unicode_code_point_data_is_assigned(cp_data); } +static inline bool +unicode_code_point_data_general_category_in( + const struct unicode_code_point_data *cp_data, uint8_t group) +{ + return (group == + (cp_data->general_category & + UNICODE_GENERAL_CATEGORY_GROUP_MASK)); +} + static inline size_t unicode_code_point_get_full_decomposition(uint32_t cp, bool canonical, const uint32_t **decomp_r) diff --git a/src/lib/unicode-ucd-compile.py b/src/lib/unicode-ucd-compile.py index 6c963ee668..3c39755224 100755 --- a/src/lib/unicode-ucd-compile.py +++ b/src/lib/unicode-ucd-compile.py @@ -45,6 +45,9 @@ ud_compositions_max_per_starter = 0 ud_case_mappings = [] ud_case_mapping_max_length = 0 +ud_idna_mappings = [] +ud_idna_mapping_max_length = 0 + class UCDFileOpen: def __init__(self, filename): @@ -801,6 +804,59 @@ def read_ucd_files(): cpd.pb_wb_extendnumlet = True CodePointRange(cprng[0], cprng[1], cpd) + # IdnaMappingTable.txt + with UCDFileOpen("IdnaMappingTable.txt") as ucd: + line_num = 0 + for line in ucd.fd: + line_num = line_num + 1 + data = line.split("#") + line = data[0].strip() + if len(line) == 0: + continue + + cols = line.split(";") + if len(cols) < 2: + die(f"{ucd}:{line_num}: Missing columns") + + cprng = parse_cp_range(cols[0]) + if cprng is None: + continue + + status_label = cols[1].strip() + status = None + mapping = "" + if len(cols) >= 3: + mapping = cols[2].strip() + + if status_label == "disallowed": + continue + elif status_label == "valid": + status = "UNICODE_IDNA_STATUS_VALID" + elif status_label == "ignored": + status = "UNICODE_IDNA_STATUS_IGNORED" + elif status_label == "mapped": + status = "UNICODE_IDNA_STATUS_MAPPED" + elif status_label == "deviation": + status = "UNICODE_IDNA_STATUS_DEVIATION" + else: + continue + + cpd = CodePointData() + cpd.idna_status = status + + codes_hex = mapping.split(" ") + if len(mapping) > 0 and len(codes_hex) > 0: + first_code_hex = codes_hex[0].strip() + first_code = int(first_code_hex, 16) + if len(codes_hex) > 1 or first_code != cp: + codes = [] + for code_hex in codes_hex: + codes.append(int(code_hex, 16)) + + cpd.idna_mapping = codes + + CodePointRange(cprng[0], cprng[1], cpd) + def resolve_case_mappings(): global ud_codepoints @@ -1088,6 +1144,29 @@ def derive_canonical_compositions(): ud_composition_primaries = ud_composition_primaries + [p[1] for p in mp] +def resolve_idna_mappings(): + global ud_codepoints + global ud_idna_mappings + global ud_idna_mapping_max_length + + for cpr in ud_codepoints: + if cpr.cp_last > cpr.cp_first: + # No mappings in ranges expected, ever + continue + cp = cpr.cp_first + cpd = cpr.data + + idna_codes = [] + if hasattr(cpd, "idna_mapping"): + idna_codes = cpd.idna_mapping + if len(idna_codes) > 0 and (len(idna_codes) > 1 or idna_codes[0] != cp): + cpd.idna_mapping_offset = len(ud_idna_mappings) + cpd.idna_mapping_length = len(idna_codes) + ud_idna_mappings = ud_idna_mappings + idna_codes + if len(idna_codes) > ud_idna_mapping_max_length: + ud_idna_mapping_max_length = len(idna_codes) + + def create_cp_range_index(): global ud_codepoints global ud_codepoints_index @@ -1281,6 +1360,7 @@ def write_tables_h(): global ud_decomposition_max_length global ud_compositions_max_per_starter global ud_case_mapping_max_length + global ud_idna_mapping_max_length orig_stdout = sys.stdout @@ -1301,6 +1381,7 @@ def write_tables_h(): % ud_compositions_max_per_starter ) print("#define UNICODE_CASE_MAPPING_MAX_LENGTH %s" % ud_case_mapping_max_length) + print("#define UNICODE_IDNA_MAX_MAPPING_LENGTH %s" % ud_idna_mapping_max_length) print("") print("extern const struct unicode_code_point_data unicode_code_points[];") print("") @@ -1316,6 +1397,8 @@ def write_tables_h(): print("") print("extern const uint32_t unicode_case_mappings[];") print("") + print("extern const uint32_t unicode_idna_mappings[];") + print("") print("#endif") sys.stdout = orig_stdout @@ -1440,6 +1523,11 @@ def write_tables_c_cpd(cpd): "\t\t.simple_titlecase_mapping = 0x%04X," % cpd.simple_titlecase_mapping ) + if hasattr(cpd, "idna_status"): + print("\t\t.idna_status = %s," % cpd.idna_status) + if hasattr(cpd, "idna_mapping_length") and cpd.idna_mapping_length > 0: + print("\t\t.idna_mapping_length = %s," % cpd.idna_mapping_length) + print("\t\t.idna_mapping_offset = %s," % cpd.idna_mapping_offset) if hasattr(cpd, "indic_conjunct_break"): print( "\t\t.indic_conjunct_break = %s," @@ -1780,6 +1868,10 @@ def write_tables_c(): print_list(ud_case_mappings) print(",") print("};") + print("") + print("const uint32_t unicode_idna_mappings[] = {") + print_list(ud_idna_mappings) + print("};") sys.stdout = orig_stdout @@ -1890,6 +1982,7 @@ def main(): resolve_case_mappings() expand_decompositions() derive_canonical_compositions() + resolve_idna_mappings() create_cp_index_tables()