Impelment Unicode normalization Form D in libarchive itself in order not to rely...

It passes all NormalizationTest.txt released The Unicode Consortium(http://www.unicode.org).

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4d66b2c841c159fb684b2918a9e721e685807158..1940384e68a04bfe92369be6374f8ea72fc944ef 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1369,11 +1369,6 @@ IF(MSVC)
   ADD_DEFINITIONS(-D_CRT_SECURE_NO_DEPRECATE)
 ENDIF(MSVC)
 
-# We need CoreServices on Mac OS.
-IF(APPLE)
-  LIST(APPEND ADDITIONAL_LIBS "-framework CoreServices")
-ENDIF(APPLE)
-
 IF(ENABLE_TEST)
   ADD_CUSTOM_TARGET(run_all_tests)
 ENDIF(ENABLE_TEST)

diff --git a/build/utils/gen_archive_string_composition_h.sh b/build/utils/gen_archive_string_composition_h.sh
index 95dbe167d5e5152f86a58abf096b17de3c7024f6..925de5c85e784c35d2291b86d4d9550042a15a02 100755 (executable)
--- a/build/utils/gen_archive_string_composition_h.sh
+++ b/build/utils/gen_archive_string_composition_h.sh
@@ -1,10 +1,13 @@
 #!/bin/sh
 #
-# This needs http://unicode.org/Public/UNIDATA/UnicodeData.txt
+# This needs http://unicode.org/Public/6.0.0/ucd/UnicodeData.txt
 #
 inputfile="$1" # Expect UnicodeData.txt
 outfile=archive_string_composition.h
 pickout=/tmp/mk_unicode_composition_tbl$$.awk
+pickout2=/tmp/mk_unicode_composition_tbl2$$.awk
+#nfdtmp=/tmp/mk_unicode_decomposition_tmp$$.txt
+nfdtmp="nfdtmpx"
 #################################################################################
 #
 # Append the file header of "archive_string_composition.h"
@@ -14,7 +17,7 @@ append_copyright()
 {
 cat > ${outfile} <<CR_END
 /*-
- * Copyright (c) 2011 libarchive Project
+ * Copyright (c) 2011-2012 libarchive Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -44,7 +47,7 @@ cat > ${outfile} <<CR_END
 /*
  * ATTENTION!
  *  This file is generated by build/utils/gen_archive_string_composition_h.sh
- *  from http://unicode.org/Public/UNIDATA/UnicodeData.txt
+ *  from http://unicode.org/Public/6.0.0/ucd/UnicodeData.txt
  *
  *  See also http://unicode.org/report/tr15/
  */
@@ -76,6 +79,7 @@ BEGIN {
   min = "";
   max = "";
   cmd="sort | awk -F ' ' '{printf \"\\\\t{ 0x%s , 0x%s , 0x%s },\\\\n\",\$1,\$2,\$3}'"
+  nfdtbl="${nfdtmp}"
   print "static const struct unicode_composition_table u_composition_table[] = {"
 }
 END {
@@ -178,7 +182,6 @@ END {
   }
   print "};"
   print ""
-  print "#endif /* ARCHIVE_STRING_COMPOSITION_H_INCLUDED */"
 }
 #
 #
@@ -241,7 +244,7 @@ function hextoi(hex)
 #}
 #
 # Exclusion code points specified by  
-# http://unicode.org/Public/UNIDATA/CompositionExclusions.txt
+# http://unicode.org/Public/6.0.0/ucd/CompositionExclusions.txt
 ##
 # 1. Script Specifices
 ##
@@ -404,6 +407,35 @@ function hextoi(hex)
         print "0"cp[1], "0"cp[2], "0"\$1 | cmd
     else
         print cp[1], cp[2], \$1 | cmd
+    # NFC ==> NFD table.
+    if (length(\$1) == 4)
+        print "0"\$1, "0"cp[1], "0"cp[2] >>nfdtbl
+    else
+        print \$1, cp[1], cp[2] >>nfdtbl
+}
+AWK_END
+#################################################################################
+# awk script
+#
+#################################################################################
+cat > ${pickout2} <<AWK_END
+#
+BEGIN {
+  FS = " "
+  print "struct unicode_decomposition_table {"
+  print "\tuint32_t nfc;"
+  print "\tuint32_t cp1;"
+  print "\tuint32_t cp2;"
+  print "};"
+  print ""
+  print "static const struct unicode_decomposition_table u_decomposition_table[] = {"
+}
+END {
+  print "};"
+  print ""
+}
+{
+printf "\t{ 0x%s , 0x%s , 0x%s },\n", \$1, \$2, \$3;
 }
 AWK_END
 #################################################################################
@@ -413,6 +445,11 @@ AWK_END
 #################################################################################
 append_copyright
 awk -f ${pickout} ${inputfile} >> ${outfile}
+awk -f ${pickout2} ${nfdtmp} >> ${outfile}
+echo "#endif /* ARCHIVE_STRING_COMPOSITION_H_INCLUDED */" >> ${outfile}
+echo "" >> ${outfile}
 #
 # Remove awk the script.
 rm ${pickout}
+rm ${pickout2}
+rm ${nfdtmp}

diff --git a/configure.ac b/configure.ac
index c815c60496f5ac62d17106e54c1ffb4273ef1682..da4f42135576f617e4d21f6e8eea677cf0914b96 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -197,13 +197,6 @@ case $host in
   ;;
 esac
 
-# We need CoreServices on Mac OS.
-case $host in
-  *darwin* )
-  LIBS="${LIBS} -framework CoreServices"
-  ;;
-esac
-
 # Checks for header files.
 AC_HEADER_DIRENT
 AC_HEADER_SYS_WAIT

diff --git a/libarchive/archive_string.c b/libarchive/archive_string.c
index bf9f904bc2d5517bbc00086b5d2b9d799a3cb041..62206e8311c0ab937bd2a3eb2ebe850f47c91f76 100644 (file)
--- a/libarchive/archive_string.c
+++ b/libarchive/archive_string.c
@@ -61,9 +61,6 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_string.c 201095 2009-12-28 02:33
 #include <windows.h>
 #include <locale.h>
 #endif
-#if defined(__APPLE__)
-#include <CoreServices/CoreServices.h>
-#endif
 
 #include "archive_endian.h"
 #include "archive_private.h"
@@ -115,11 +112,6 @@ struct archive_string_conv {
 #endif
        /* A temporary buffer for normalization. */
        struct archive_string            utftmp;
-#if defined(__APPLE__)
-       UnicodeToTextInfo                uniInfo;
-       struct archive_string            utf16nfc;
-       struct archive_string            utf16nfd;
-#endif
        int (*converter[2])(struct archive_string *, const void *, size_t,
            struct archive_string_conv *);
        int                              nconverter;
@@ -201,10 +193,8 @@ static int strncat_from_utf8_to_utf8(struct archive_string *, const void *,
     size_t, struct archive_string_conv *);
 static int archive_string_normalize_C(struct archive_string *, const void *,
     size_t, struct archive_string_conv *);
-#if defined(__APPLE__)
 static int archive_string_normalize_D(struct archive_string *, const void *,
     size_t, struct archive_string_conv *);
-#endif
 static int archive_string_append_unicode(struct archive_string *,
     const void *, size_t, struct archive_string_conv *);
 
@@ -477,7 +467,8 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                        *ws++ = (wchar_t)*mp++;
                        count++;
                }
-       } else if (sc != NULL && (sc->flag & SCONV_NORMALIZATION_C)) {
+       } else if (sc != NULL &&
+           (sc->flag & (SCONV_NORMALIZATION_C | SCONV_NORMALIZATION_D))) {
                /*
                 * Normalize UTF-8 and UTF-16BE and convert it directly
                 * to UTF-16 as wchar_t.
@@ -493,18 +484,23 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
                if (sc->flag & SCONV_FROM_UTF16) {
                        /*
                         *  UTF-16BE/LE NFD ===> UTF-16 NFC
+                        *  UTF-16BE/LE NFC ===> UTF-16 NFD
                         */
                        count = utf16nbytes(s, length);
                } else {
                        /*
                         *  UTF-8 NFD ===> UTF-16 NFC
+                        *  UTF-8 NFC ===> UTF-16 NFD
                         */
                        count = mbsnbytes(s, length);
                }
                u16.s = (char *)dest->s;
                u16.length = dest->length << 1;;
                u16.buffer_length = dest->buffer_length;
-               ret = archive_string_normalize_C(&u16, s, count, sc);
+               if (sc->flag & SCONV_NORMALIZATION_C)
+                       ret = archive_string_normalize_C(&u16, s, count, sc);
+               else
+                       ret = archive_string_normalize_D(&u16, s, count, sc);
                dest->s = (wchar_t *)u16.s;
                dest->length = u16.length >> 1;
                dest->buffer_length = u16.buffer_length;
@@ -878,27 +874,6 @@ add_sconv_object(struct archive *a, struct archive_string_conv *sc)
        *psc = sc;
 }
 
-#if defined(__APPLE__)
-
-static int
-createUniInfo(struct archive_string_conv *sconv)
-{
-       UnicodeMapping map;
-       OSStatus err;
-
-       map.unicodeEncoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
-           kUnicodeNoSubset, kUnicode16BitFormat);
-       map.otherEncoding = CreateTextEncoding(kTextEncodingUnicodeDefault,
-           kUnicodeHFSPlusDecompVariant, kUnicode16BitFormat);
-       map.mappingVersion = kUnicodeUseLatestMapping;
-
-       sconv->uniInfo = NULL;
-       err = CreateUnicodeToTextInfo(&map, &(sconv->uniInfo));
-       return ((err == noErr)? 0: -1);
-}
-
-#endif /* __APPLE__ */
-
 static void
 add_converter(struct archive_string_conv *sc, int (*converter)
     (struct archive_string *, const void *, size_t,
@@ -975,12 +950,9 @@ setup_converter(struct archive_string_conv *sc)
                /*
                 * At least we should normalize a UTF-16BE string.
                 */
-#if defined(__APPLE__)
                if (sc->flag & SCONV_NORMALIZATION_D)
                        add_converter(sc,archive_string_normalize_D);
-               else
-#endif
-               if (sc->flag & SCONV_NORMALIZATION_C)
+               else if (sc->flag & SCONV_NORMALIZATION_C)
                        add_converter(sc, archive_string_normalize_C);
 
                if (sc->flag & SCONV_TO_UTF8) {
@@ -1028,12 +1000,9 @@ setup_converter(struct archive_string_conv *sc)
                /*
                 * At least we should normalize a UTF-8 string.
                 */
-#if defined(__APPLE__)
                if (sc->flag & SCONV_NORMALIZATION_D)
                        add_converter(sc,archive_string_normalize_D);
-               else
-#endif
-               if (sc->flag & SCONV_NORMALIZATION_C)
+               else if (sc->flag & SCONV_NORMALIZATION_C)
                        add_converter(sc, archive_string_normalize_C);
 
                /*
@@ -1146,10 +1115,6 @@ create_sconv_object(const char *fc, const char *tc,
                return (NULL);
        }
        archive_string_init(&sc->utftmp);
-#if defined(__APPLE__)
-       archive_string_init(&sc->utf16nfc);
-       archive_string_init(&sc->utf16nfd);
-#endif
 
        if (flag & SCONV_TO_CHARSET) {
                /*
@@ -1228,10 +1193,9 @@ create_sconv_object(const char *fc, const char *tc,
        if ((flag & SCONV_FROM_CHARSET) &&
            (flag & (SCONV_FROM_UTF16 | SCONV_FROM_UTF8))) {
 #if defined(__APPLE__)
-               if (flag & SCONV_TO_UTF8) {
-                       if (createUniInfo(sc) == 0)
-                               flag |= SCONV_NORMALIZATION_D;
-               } else
+               if (flag & SCONV_TO_UTF8)
+                       flag |= SCONV_NORMALIZATION_D;
+               else
 #endif
                        flag |= SCONV_NORMALIZATION_C;
        }
@@ -1342,12 +1306,6 @@ free_sconv_object(struct archive_string_conv *sc)
                iconv_close(sc->cd);
        if (sc->cd_w != (iconv_t)-1)
                iconv_close(sc->cd_w);
-#endif
-#if defined(__APPLE__)
-       archive_string_free(&sc->utf16nfc);
-       archive_string_free(&sc->utf16nfd);
-       if (sc->uniInfo != NULL)
-               DisposeUnicodeToTextInfo(&(sc->uniInfo));
 #endif
        free(sc);
 }
@@ -1893,6 +1851,22 @@ archive_string_conversion_set_opt(struct archive_string_conv *sc, int opt)
                }
 #endif
                break;
+       case SCONV_SET_OPT_NORMALIZATION_C:
+               if ((sc->flag & SCONV_NORMALIZATION_C) == 0) {
+                       sc->flag |= SCONV_NORMALIZATION_C;
+                       sc->flag &= ~SCONV_NORMALIZATION_D;
+                       /* Re-setup string converters. */
+                       setup_converter(sc);
+               }
+               break;
+       case SCONV_SET_OPT_NORMALIZATION_D:
+               if ((sc->flag & SCONV_NORMALIZATION_D) == 0) {
+                       sc->flag |= SCONV_NORMALIZATION_D;
+                       sc->flag &= ~SCONV_NORMALIZATION_C;
+                       /* Re-setup string converters. */
+                       setup_converter(sc);
+               }
+               break;
        default:
                break;
        }
@@ -3213,119 +3187,222 @@ archive_string_normalize_C(struct archive_string *as, const void *_p,
        return (ret);
 }
 
-#if defined(__APPLE__)
-
-/*
- * Normalize UTF-8 characters to Form D and copy the result.
- */
 static int
-archive_string_normalize_D(struct archive_string *as, const void *_p,
-    size_t len, struct archive_string_conv *sc)
+get_nfd(uint32_t *cp1, uint32_t *cp2, uint32_t uc)
 {
-       const UniChar *inp;
-       char *outp;
-       size_t newsize;
-       ByteCount inCount, outCount;
-       ByteCount inAvail, outAvail;
-       OSStatus err;
-       int ret, saved_flag;
+       int t, b;
 
        /*
-        * Convert the current string to UTF-16LE for normalization.
-        * The character-set of the current string must be UTF-16BE or
-        * UTF-8.
+        * These are not converted to NFD on Mac OS.
         */
-       archive_string_empty(&(sc->utf16nfc));
-       saved_flag = sc->flag;/* save a flag. */
-       sc->flag &= ~(SCONV_TO_UTF16BE | SCONV_TO_UTF8);
-       sc->flag |= SCONV_TO_UTF16LE;
-       ret = archive_string_append_unicode(&(sc->utf16nfc), _p, len, sc);
-       sc->flag = saved_flag;/* restore the saved flag */
-       if (archive_strlen(&(sc->utf16nfc)) == 0) {
-               if (archive_string_ensure(as, as->length + 1) == NULL)
-                       return (-1);
-               return (ret);
-       }
-
+       if ((uc >= 0x2000 && uc <= 0x2FFF) ||
+           (uc >= 0xF900 && uc <= 0xFAFF) ||
+           (uc >= 0x2F800 && uc <= 0x2FAFF))
+               return (0);
        /*
-        * Normalize an NFC string to be an NFD(HFS Plus version).
+        * Those code points are not converted to NFD on Mac OS.
+        * I do not know the reason because it is undocumented.
+        *   NFC        NFD
+        *   1109A  ==> 11099 110BA
+        *   1109C  ==> 1109B 110BA
+        *   110AB  ==> 110A5 110BA
         */
-       newsize = sc->utf16nfc.length + 2;
-       if (archive_string_ensure(&(sc->utf16nfd), newsize) == NULL)
-               return (-1);
+       if (uc == 0x1109A || uc == 0x1109C || uc == 0x110AB)
+               return (0);
 
-       inp = (UniChar *)sc->utf16nfc.s;
-       inAvail = archive_strlen(&(sc->utf16nfc));
-       sc->utf16nfd.length = 0;
-       outp = sc->utf16nfd.s;
-       outAvail = sc->utf16nfd.buffer_length -2;
+       t = 0;
+       b = sizeof(u_decomposition_table)/sizeof(u_decomposition_table[0]) -1;
+       while (b >= t) {
+               int m = (t + b) / 2;
+               if (u_decomposition_table[m].nfc < uc)
+                       t = m + 1;
+               else if (u_decomposition_table[m].nfc > uc)
+                       b = m - 1;
+               else {
+                       *cp1 = u_decomposition_table[m].cp1;
+                       *cp2 = u_decomposition_table[m].cp2;
+                       return (1);
+               }
+       }
+       return (0);
+}
 
-       do {
-               /* Reinitialize all state information. */
-               if (ResetUnicodeToTextInfo(sc->uniInfo) != noErr)
-                       goto return_no_changed_data;
-
-               inCount = outCount = 0;
-               err = ConvertFromUnicodeToText(sc->uniInfo,
-                   inAvail, inp,
-                   kUnicodeDefaultDirectionMask, 0, NULL, NULL, NULL,
-                   outAvail, &inCount, &outCount, outp);
-
-               if (err == noErr) {
-                       sc->utf16nfd.length = outCount;
-                       sc->utf16nfd.s[sc->utf16nfd.length] = 0;
-                       sc->utf16nfd.s[sc->utf16nfd.length+1] = 0;
-               } else if (err == kTECOutputBufferFullStatus) {
-                       newsize = inAvail - inCount;
-                       if (newsize > inAvail)
-                               newsize = inAvail;
-                       newsize += sc->utf16nfd.buffer_length + 2;
-                       if (archive_string_ensure(&(sc->utf16nfd), newsize)
-                           == NULL)
-                               return (-1);
-                       outp = sc->utf16nfd.s;
-                       outAvail = sc->utf16nfd.buffer_length -2;
-               } else
-                       goto return_no_changed_data;
-       } while (err == kTECOutputBufferFullStatus);
+#define REPLACE_UC_WITH(cp) do {               \
+       uc = cp;                                \
+       ucptr = NULL;                           \
+} while (0)
 
-       /*
-        * If there is a next-step conversion, we should convert
-        * a UTF-16LE(NFD) string back to the original Unicode type.
-        */
-       saved_flag = sc->flag;/* save a flag. */
-       if (!(sc->flag &
-           (SCONV_TO_UTF16BE | SCONV_TO_UTF16LE | SCONV_TO_UTF8))) {
+/*
+ * Normalize UTF-8 characters to Form D and copy the result.
+ */
+static int
+archive_string_normalize_D(struct archive_string *as, const void *_p,
+    size_t len, struct archive_string_conv *sc)
+{
+       const char *s = (const char *)_p;
+       char *p, *endp;
+       uint32_t uc, uc2;
+       size_t w;
+       int always_replace, n, n2, ret = 0, spair, ts, tm;
+       int (*parse)(uint32_t *, const char *, size_t);
+       size_t (*unparse)(char *, size_t, uint32_t);
+
+       always_replace = 1;
+       ts = 1;/* text size. */
+       if (sc->flag & SCONV_TO_UTF16BE) {
+               unparse = unicode_to_utf16be;
+               ts = 2;
+               if (sc->flag & SCONV_FROM_UTF16BE)
+                       always_replace = 0;
+       } else if (sc->flag & SCONV_TO_UTF16LE) {
+               unparse = unicode_to_utf16le;
+               ts = 2;
+               if (sc->flag & SCONV_FROM_UTF16LE)
+                       always_replace = 0;
+       } else if (sc->flag & SCONV_TO_UTF8) {
+               unparse = unicode_to_utf8;
+               if (sc->flag & SCONV_FROM_UTF8)
+                       always_replace = 0;
+       } else {
                /*
                 * This case is going to be converted to another
                 * character-set through iconv.
                 */
-               if (sc->flag & SCONV_FROM_UTF16BE)
-                       sc->flag |= SCONV_TO_UTF16BE;
-               else if (sc->flag & SCONV_FROM_UTF16LE)
-                       sc->flag |= SCONV_TO_UTF16LE;
+               always_replace = 0;
+               if (sc->flag & SCONV_FROM_UTF16BE) {
+                       unparse = unicode_to_utf16be;
+                       ts = 2;
+               } else if (sc->flag & SCONV_FROM_UTF16LE) {
+                       unparse = unicode_to_utf16le;
+                       ts = 2;
+               } else {
+                       unparse = unicode_to_utf8;
+               }
+       }
+
+       if (sc->flag & SCONV_FROM_UTF16BE) {
+               parse = utf16be_to_unicode;
+               tm = 1;
+               spair = 4;/* surrogate pair size in UTF-16. */
+       } else if (sc->flag & SCONV_FROM_UTF16LE) {
+               parse = utf16le_to_unicode;
+               tm = 1;
+               spair = 4;/* surrogate pair size in UTF-16. */
+       } else {
+               parse = cesu8_to_unicode;
+               tm = ts;
+               spair = 6;/* surrogate pair size in UTF-8. */
+       }
+
+       if (archive_string_ensure(as, as->length + len * tm + ts) == NULL)
+               return (-1);
+
+       p = as->s + as->length;
+       endp = as->s + as->buffer_length - ts;
+       while ((n = parse(&uc, s, len)) != 0) {
+               const char *ucptr;
+               uint32_t cp1, cp2;
+               int SIndex;
+               struct {
+                       uint32_t uc;
+                       int ccc;
+               } fdc[FDC_MAX];
+               int fdi, fdj;
+               int ccc;
+
+check_first_code:
+               if (n < 0) {
+                       /* Use a replaced unicode character. */
+                       UNPARSE(p, endp, uc);
+                       s += n*-1;
+                       len -= n*-1;
+                       ret = -1;
+                       continue;
+               } else if (n == spair || always_replace)
+                       /* uc is converted from a surrogate pair.
+                        * this should be treated as a changed code. */
+                       ucptr = NULL;
                else
-                       sc->flag |= SCONV_TO_UTF8;
+                       ucptr = s;
+               s += n;
+               len -= n;
+
+               /* Hangul Decomposition. */
+               if ((SIndex = uc - HC_SBASE) >= 0 && SIndex < HC_SCOUNT) {
+                       int L = HC_LBASE + SIndex / HC_NCOUNT;
+                       int V = HC_VBASE + (SIndex % HC_NCOUNT) / HC_TCOUNT;
+                       int T = HC_TBASE + SIndex % HC_TCOUNT;
+
+                       REPLACE_UC_WITH(L);
+                       WRITE_UC();
+                       REPLACE_UC_WITH(V);
+                       WRITE_UC();
+                       if (T != HC_TBASE) {
+                               REPLACE_UC_WITH(T);
+                               WRITE_UC();
+                       }
+                       continue;
+               }
+               if (IS_DECOMPOSABLE_BLOCK(uc) && CCC(uc) != 0) {
+                       WRITE_UC();
+                       continue;
+               }
+
+               fdi = 0;
+               while (get_nfd(&cp1, &cp2, uc) && fdi < FDC_MAX) {
+                       int k;
+
+                       for (k = fdi; k > 0; k--)
+                               fdc[k] = fdc[k-1];
+                       fdc[0].ccc = CCC(cp2);
+                       fdc[0].uc = cp2;
+                       fdi++;
+                       REPLACE_UC_WITH(cp1);
+               }
+
+               /* Read following code points. */
+               while ((n2 = parse(&uc2, s, len)) > 0 &&
+                   (ccc = CCC(uc2)) != 0 && fdi < FDC_MAX) {
+                       int j, k;
+
+                       s += n2;
+                       len -= n2;
+                       for (j = 0; j < fdi; j++) {
+                               if (fdc[j].ccc > ccc)
+                                       break;
+                       }
+                       if (j < fdi) {
+                               for (k = fdi; k > j; k--)
+                                       fdc[k] = fdc[k-1];
+                               fdc[j].ccc = ccc;
+                               fdc[j].uc = uc2;
+                       } else {
+                               fdc[fdi].ccc = ccc;
+                               fdc[fdi].uc = uc2;
+                       }
+                       fdi++;
+               }
+
+               WRITE_UC();
+               for (fdj = 0; fdj < fdi; fdj++) {
+                       REPLACE_UC_WITH(fdc[fdj].uc);
+                       WRITE_UC();
+               }
+
+               if (n2 == 0)
+                       break;
+               REPLACE_UC_WITH(uc2);
+               n = n2;
+               ucptr = s;
+               goto check_first_code;
        }
-       sc->flag &= ~(SCONV_FROM_UTF16BE | SCONV_FROM_UTF8);
-       sc->flag |= SCONV_FROM_UTF16LE;
-       if (archive_string_append_unicode(as, sc->utf16nfd.s,
-           sc->utf16nfd.length, sc) != 0)
-               ret = -1;
-       sc->flag = saved_flag;/* restore the saved flag */
+       as->length = p - as->s;
+       as->s[as->length] = '\0';
+       if (ts == 2)
+               as->s[as->length+1] = '\0';
        return (ret);
-
-return_no_changed_data:
-       /*
-        * Something conversion error happened, so we return a no normalized
-        * string with an error.
-        */
-       (void)archive_string_append_unicode(as, _p, len, sc);
-       return (-1);
 }
 
-#endif /* __APPLE__ */
-
 /*
  * libarchive 2.x made incorrect UTF-8 strings in the wrong assumption
  * that WCS is Unicode. It is true for several platforms but some are false.

diff --git a/libarchive/archive_string.h b/libarchive/archive_string.h
index 0e4c9b9041e3ef661fbd95a3f414a581254d9f0e..a5c0436fc4ab57394e703a815ded748e953a5d66 100644 (file)
--- a/libarchive/archive_string.h
+++ b/libarchive/archive_string.h
@@ -110,6 +110,8 @@ archive_string_conversion_charset_name(struct archive_string_conv *);
 void
 archive_string_conversion_set_opt(struct archive_string_conv *, int);
 #define SCONV_SET_OPT_UTF8_LIBARCHIVE2X        1
+#define SCONV_SET_OPT_NORMALIZATION_C  2
+#define SCONV_SET_OPT_NORMALIZATION_D  4
 
 
 /* Copy one archive_string to another in locale conversion.

diff --git a/libarchive/archive_string_composition.h b/libarchive/archive_string_composition.h
index cc4bf46c0e159567550d4886936d9329801c0341..be41e3365124f630e361bd29726d1df26688e64f 100644 (file)
--- a/libarchive/archive_string_composition.h
+++ b/libarchive/archive_string_composition.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011 libarchive Project
+ * Copyright (c) 2011-2012 libarchive Project
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -29,7 +29,7 @@
 /*
  * ATTENTION!
  *  This file is generated by build/utils/gen_archive_string_composition_h.sh
- *  from http://unicode.org/Public/UNIDATA/UnicodeData.txt
+ *  from http://unicode.org/Public/6.0.0/ucd/UnicodeData.txt
  *
  *  See also http://unicode.org/report/tr15/
  */
@@ -1348,4 +1348,945 @@ static const unsigned char ccc_index[] = {
   0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
   0, 0, 0, 0, 0, 0, 0, 0, 0,37,38,};
 
+struct unicode_decomposition_table {
+       uint32_t nfc;
+       uint32_t cp1;
+       uint32_t cp2;
+};
+
+static const struct unicode_decomposition_table u_decomposition_table[] = {
+       { 0x000C0 , 0x00041 , 0x00300 },
+       { 0x000C1 , 0x00041 , 0x00301 },
+       { 0x000C2 , 0x00041 , 0x00302 },
+       { 0x000C3 , 0x00041 , 0x00303 },
+       { 0x000C4 , 0x00041 , 0x00308 },
+       { 0x000C5 , 0x00041 , 0x0030A },
+       { 0x000C7 , 0x00043 , 0x00327 },
+       { 0x000C8 , 0x00045 , 0x00300 },
+       { 0x000C9 , 0x00045 , 0x00301 },
+       { 0x000CA , 0x00045 , 0x00302 },
+       { 0x000CB , 0x00045 , 0x00308 },
+       { 0x000CC , 0x00049 , 0x00300 },
+       { 0x000CD , 0x00049 , 0x00301 },
+       { 0x000CE , 0x00049 , 0x00302 },
+       { 0x000CF , 0x00049 , 0x00308 },
+       { 0x000D1 , 0x0004E , 0x00303 },
+       { 0x000D2 , 0x0004F , 0x00300 },
+       { 0x000D3 , 0x0004F , 0x00301 },
+       { 0x000D4 , 0x0004F , 0x00302 },
+       { 0x000D5 , 0x0004F , 0x00303 },
+       { 0x000D6 , 0x0004F , 0x00308 },
+       { 0x000D9 , 0x00055 , 0x00300 },
+       { 0x000DA , 0x00055 , 0x00301 },
+       { 0x000DB , 0x00055 , 0x00302 },
+       { 0x000DC , 0x00055 , 0x00308 },
+       { 0x000DD , 0x00059 , 0x00301 },
+       { 0x000E0 , 0x00061 , 0x00300 },
+       { 0x000E1 , 0x00061 , 0x00301 },
+       { 0x000E2 , 0x00061 , 0x00302 },
+       { 0x000E3 , 0x00061 , 0x00303 },
+       { 0x000E4 , 0x00061 , 0x00308 },
+       { 0x000E5 , 0x00061 , 0x0030A },
+       { 0x000E7 , 0x00063 , 0x00327 },
+       { 0x000E8 , 0x00065 , 0x00300 },
+       { 0x000E9 , 0x00065 , 0x00301 },
+       { 0x000EA , 0x00065 , 0x00302 },
+       { 0x000EB , 0x00065 , 0x00308 },
+       { 0x000EC , 0x00069 , 0x00300 },
+       { 0x000ED , 0x00069 , 0x00301 },
+       { 0x000EE , 0x00069 , 0x00302 },
+       { 0x000EF , 0x00069 , 0x00308 },
+       { 0x000F1 , 0x0006E , 0x00303 },
+       { 0x000F2 , 0x0006F , 0x00300 },
+       { 0x000F3 , 0x0006F , 0x00301 },
+       { 0x000F4 , 0x0006F , 0x00302 },
+       { 0x000F5 , 0x0006F , 0x00303 },
+       { 0x000F6 , 0x0006F , 0x00308 },
+       { 0x000F9 , 0x00075 , 0x00300 },
+       { 0x000FA , 0x00075 , 0x00301 },
+       { 0x000FB , 0x00075 , 0x00302 },
+       { 0x000FC , 0x00075 , 0x00308 },
+       { 0x000FD , 0x00079 , 0x00301 },
+       { 0x000FF , 0x00079 , 0x00308 },
+       { 0x00100 , 0x00041 , 0x00304 },
+       { 0x00101 , 0x00061 , 0x00304 },
+       { 0x00102 , 0x00041 , 0x00306 },
+       { 0x00103 , 0x00061 , 0x00306 },
+       { 0x00104 , 0x00041 , 0x00328 },
+       { 0x00105 , 0x00061 , 0x00328 },
+       { 0x00106 , 0x00043 , 0x00301 },
+       { 0x00107 , 0x00063 , 0x00301 },
+       { 0x00108 , 0x00043 , 0x00302 },
+       { 0x00109 , 0x00063 , 0x00302 },
+       { 0x0010A , 0x00043 , 0x00307 },
+       { 0x0010B , 0x00063 , 0x00307 },
+       { 0x0010C , 0x00043 , 0x0030C },
+       { 0x0010D , 0x00063 , 0x0030C },
+       { 0x0010E , 0x00044 , 0x0030C },
+       { 0x0010F , 0x00064 , 0x0030C },
+       { 0x00112 , 0x00045 , 0x00304 },
+       { 0x00113 , 0x00065 , 0x00304 },
+       { 0x00114 , 0x00045 , 0x00306 },
+       { 0x00115 , 0x00065 , 0x00306 },
+       { 0x00116 , 0x00045 , 0x00307 },
+       { 0x00117 , 0x00065 , 0x00307 },
+       { 0x00118 , 0x00045 , 0x00328 },
+       { 0x00119 , 0x00065 , 0x00328 },
+       { 0x0011A , 0x00045 , 0x0030C },
+       { 0x0011B , 0x00065 , 0x0030C },
+       { 0x0011C , 0x00047 , 0x00302 },
+       { 0x0011D , 0x00067 , 0x00302 },
+       { 0x0011E , 0x00047 , 0x00306 },
+       { 0x0011F , 0x00067 , 0x00306 },
+       { 0x00120 , 0x00047 , 0x00307 },
+       { 0x00121 , 0x00067 , 0x00307 },
+       { 0x00122 , 0x00047 , 0x00327 },
+       { 0x00123 , 0x00067 , 0x00327 },
+       { 0x00124 , 0x00048 , 0x00302 },
+       { 0x00125 , 0x00068 , 0x00302 },
+       { 0x00128 , 0x00049 , 0x00303 },
+       { 0x00129 , 0x00069 , 0x00303 },
+       { 0x0012A , 0x00049 , 0x00304 },
+       { 0x0012B , 0x00069 , 0x00304 },
+       { 0x0012C , 0x00049 , 0x00306 },
+       { 0x0012D , 0x00069 , 0x00306 },
+       { 0x0012E , 0x00049 , 0x00328 },
+       { 0x0012F , 0x00069 , 0x00328 },
+       { 0x00130 , 0x00049 , 0x00307 },
+       { 0x00134 , 0x0004A , 0x00302 },
+       { 0x00135 , 0x0006A , 0x00302 },
+       { 0x00136 , 0x0004B , 0x00327 },
+       { 0x00137 , 0x0006B , 0x00327 },
+       { 0x00139 , 0x0004C , 0x00301 },
+       { 0x0013A , 0x0006C , 0x00301 },
+       { 0x0013B , 0x0004C , 0x00327 },
+       { 0x0013C , 0x0006C , 0x00327 },
+       { 0x0013D , 0x0004C , 0x0030C },
+       { 0x0013E , 0x0006C , 0x0030C },
+       { 0x00143 , 0x0004E , 0x00301 },
+       { 0x00144 , 0x0006E , 0x00301 },
+       { 0x00145 , 0x0004E , 0x00327 },
+       { 0x00146 , 0x0006E , 0x00327 },
+       { 0x00147 , 0x0004E , 0x0030C },
+       { 0x00148 , 0x0006E , 0x0030C },
+       { 0x0014C , 0x0004F , 0x00304 },
+       { 0x0014D , 0x0006F , 0x00304 },
+       { 0x0014E , 0x0004F , 0x00306 },
+       { 0x0014F , 0x0006F , 0x00306 },
+       { 0x00150 , 0x0004F , 0x0030B },
+       { 0x00151 , 0x0006F , 0x0030B },
+       { 0x00154 , 0x00052 , 0x00301 },
+       { 0x00155 , 0x00072 , 0x00301 },
+       { 0x00156 , 0x00052 , 0x00327 },
+       { 0x00157 , 0x00072 , 0x00327 },
+       { 0x00158 , 0x00052 , 0x0030C },
+       { 0x00159 , 0x00072 , 0x0030C },
+       { 0x0015A , 0x00053 , 0x00301 },
+       { 0x0015B , 0x00073 , 0x00301 },
+       { 0x0015C , 0x00053 , 0x00302 },
+       { 0x0015D , 0x00073 , 0x00302 },
+       { 0x0015E , 0x00053 , 0x00327 },
+       { 0x0015F , 0x00073 , 0x00327 },
+       { 0x00160 , 0x00053 , 0x0030C },
+       { 0x00161 , 0x00073 , 0x0030C },
+       { 0x00162 , 0x00054 , 0x00327 },
+       { 0x00163 , 0x00074 , 0x00327 },
+       { 0x00164 , 0x00054 , 0x0030C },
+       { 0x00165 , 0x00074 , 0x0030C },
+       { 0x00168 , 0x00055 , 0x00303 },
+       { 0x00169 , 0x00075 , 0x00303 },
+       { 0x0016A , 0x00055 , 0x00304 },
+       { 0x0016B , 0x00075 , 0x00304 },
+       { 0x0016C , 0x00055 , 0x00306 },
+       { 0x0016D , 0x00075 , 0x00306 },
+       { 0x0016E , 0x00055 , 0x0030A },
+       { 0x0016F , 0x00075 , 0x0030A },
+       { 0x00170 , 0x00055 , 0x0030B },
+       { 0x00171 , 0x00075 , 0x0030B },
+       { 0x00172 , 0x00055 , 0x00328 },
+       { 0x00173 , 0x00075 , 0x00328 },
+       { 0x00174 , 0x00057 , 0x00302 },
+       { 0x00175 , 0x00077 , 0x00302 },
+       { 0x00176 , 0x00059 , 0x00302 },
+       { 0x00177 , 0x00079 , 0x00302 },
+       { 0x00178 , 0x00059 , 0x00308 },
+       { 0x00179 , 0x0005A , 0x00301 },
+       { 0x0017A , 0x0007A , 0x00301 },
+       { 0x0017B , 0x0005A , 0x00307 },
+       { 0x0017C , 0x0007A , 0x00307 },
+       { 0x0017D , 0x0005A , 0x0030C },
+       { 0x0017E , 0x0007A , 0x0030C },
+       { 0x001A0 , 0x0004F , 0x0031B },
+       { 0x001A1 , 0x0006F , 0x0031B },
+       { 0x001AF , 0x00055 , 0x0031B },
+       { 0x001B0 , 0x00075 , 0x0031B },
+       { 0x001CD , 0x00041 , 0x0030C },
+       { 0x001CE , 0x00061 , 0x0030C },
+       { 0x001CF , 0x00049 , 0x0030C },
+       { 0x001D0 , 0x00069 , 0x0030C },
+       { 0x001D1 , 0x0004F , 0x0030C },
+       { 0x001D2 , 0x0006F , 0x0030C },
+       { 0x001D3 , 0x00055 , 0x0030C },
+       { 0x001D4 , 0x00075 , 0x0030C },
+       { 0x001D5 , 0x000DC , 0x00304 },
+       { 0x001D6 , 0x000FC , 0x00304 },
+       { 0x001D7 , 0x000DC , 0x00301 },
+       { 0x001D8 , 0x000FC , 0x00301 },
+       { 0x001D9 , 0x000DC , 0x0030C },
+       { 0x001DA , 0x000FC , 0x0030C },
+       { 0x001DB , 0x000DC , 0x00300 },
+       { 0x001DC , 0x000FC , 0x00300 },
+       { 0x001DE , 0x000C4 , 0x00304 },
+       { 0x001DF , 0x000E4 , 0x00304 },
+       { 0x001E0 , 0x00226 , 0x00304 },
+       { 0x001E1 , 0x00227 , 0x00304 },
+       { 0x001E2 , 0x000C6 , 0x00304 },
+       { 0x001E3 , 0x000E6 , 0x00304 },
+       { 0x001E6 , 0x00047 , 0x0030C },
+       { 0x001E7 , 0x00067 , 0x0030C },
+       { 0x001E8 , 0x0004B , 0x0030C },
+       { 0x001E9 , 0x0006B , 0x0030C },
+       { 0x001EA , 0x0004F , 0x00328 },
+       { 0x001EB , 0x0006F , 0x00328 },
+       { 0x001EC , 0x001EA , 0x00304 },
+       { 0x001ED , 0x001EB , 0x00304 },
+       { 0x001EE , 0x001B7 , 0x0030C },
+       { 0x001EF , 0x00292 , 0x0030C },
+       { 0x001F0 , 0x0006A , 0x0030C },
+       { 0x001F4 , 0x00047 , 0x00301 },
+       { 0x001F5 , 0x00067 , 0x00301 },
+       { 0x001F8 , 0x0004E , 0x00300 },
+       { 0x001F9 , 0x0006E , 0x00300 },
+       { 0x001FA , 0x000C5 , 0x00301 },
+       { 0x001FB , 0x000E5 , 0x00301 },
+       { 0x001FC , 0x000C6 , 0x00301 },
+       { 0x001FD , 0x000E6 , 0x00301 },
+       { 0x001FE , 0x000D8 , 0x00301 },
+       { 0x001FF , 0x000F8 , 0x00301 },
+       { 0x00200 , 0x00041 , 0x0030F },
+       { 0x00201 , 0x00061 , 0x0030F },
+       { 0x00202 , 0x00041 , 0x00311 },
+       { 0x00203 , 0x00061 , 0x00311 },
+       { 0x00204 , 0x00045 , 0x0030F },
+       { 0x00205 , 0x00065 , 0x0030F },
+       { 0x00206 , 0x00045 , 0x00311 },
+       { 0x00207 , 0x00065 , 0x00311 },
+       { 0x00208 , 0x00049 , 0x0030F },
+       { 0x00209 , 0x00069 , 0x0030F },
+       { 0x0020A , 0x00049 , 0x00311 },
+       { 0x0020B , 0x00069 , 0x00311 },
+       { 0x0020C , 0x0004F , 0x0030F },
+       { 0x0020D , 0x0006F , 0x0030F },
+       { 0x0020E , 0x0004F , 0x00311 },
+       { 0x0020F , 0x0006F , 0x00311 },
+       { 0x00210 , 0x00052 , 0x0030F },
+       { 0x00211 , 0x00072 , 0x0030F },
+       { 0x00212 , 0x00052 , 0x00311 },
+       { 0x00213 , 0x00072 , 0x00311 },
+       { 0x00214 , 0x00055 , 0x0030F },
+       { 0x00215 , 0x00075 , 0x0030F },
+       { 0x00216 , 0x00055 , 0x00311 },
+       { 0x00217 , 0x00075 , 0x00311 },
+       { 0x00218 , 0x00053 , 0x00326 },
+       { 0x00219 , 0x00073 , 0x00326 },
+       { 0x0021A , 0x00054 , 0x00326 },
+       { 0x0021B , 0x00074 , 0x00326 },
+       { 0x0021E , 0x00048 , 0x0030C },
+       { 0x0021F , 0x00068 , 0x0030C },
+       { 0x00226 , 0x00041 , 0x00307 },
+       { 0x00227 , 0x00061 , 0x00307 },
+       { 0x00228 , 0x00045 , 0x00327 },
+       { 0x00229 , 0x00065 , 0x00327 },
+       { 0x0022A , 0x000D6 , 0x00304 },
+       { 0x0022B , 0x000F6 , 0x00304 },
+       { 0x0022C , 0x000D5 , 0x00304 },
+       { 0x0022D , 0x000F5 , 0x00304 },
+       { 0x0022E , 0x0004F , 0x00307 },
+       { 0x0022F , 0x0006F , 0x00307 },
+       { 0x00230 , 0x0022E , 0x00304 },
+       { 0x00231 , 0x0022F , 0x00304 },
+       { 0x00232 , 0x00059 , 0x00304 },
+       { 0x00233 , 0x00079 , 0x00304 },
+       { 0x00385 , 0x000A8 , 0x00301 },
+       { 0x00386 , 0x00391 , 0x00301 },
+       { 0x00388 , 0x00395 , 0x00301 },
+       { 0x00389 , 0x00397 , 0x00301 },
+       { 0x0038A , 0x00399 , 0x00301 },
+       { 0x0038C , 0x0039F , 0x00301 },
+       { 0x0038E , 0x003A5 , 0x00301 },
+       { 0x0038F , 0x003A9 , 0x00301 },
+       { 0x00390 , 0x003CA , 0x00301 },
+       { 0x003AA , 0x00399 , 0x00308 },
+       { 0x003AB , 0x003A5 , 0x00308 },
+       { 0x003AC , 0x003B1 , 0x00301 },
+       { 0x003AD , 0x003B5 , 0x00301 },
+       { 0x003AE , 0x003B7 , 0x00301 },
+       { 0x003AF , 0x003B9 , 0x00301 },
+       { 0x003B0 , 0x003CB , 0x00301 },
+       { 0x003CA , 0x003B9 , 0x00308 },
+       { 0x003CB , 0x003C5 , 0x00308 },
+       { 0x003CC , 0x003BF , 0x00301 },
+       { 0x003CD , 0x003C5 , 0x00301 },
+       { 0x003CE , 0x003C9 , 0x00301 },
+       { 0x003D3 , 0x003D2 , 0x00301 },
+       { 0x003D4 , 0x003D2 , 0x00308 },
+       { 0x00400 , 0x00415 , 0x00300 },
+       { 0x00401 , 0x00415 , 0x00308 },
+       { 0x00403 , 0x00413 , 0x00301 },
+       { 0x00407 , 0x00406 , 0x00308 },
+       { 0x0040C , 0x0041A , 0x00301 },
+       { 0x0040D , 0x00418 , 0x00300 },
+       { 0x0040E , 0x00423 , 0x00306 },
+       { 0x00419 , 0x00418 , 0x00306 },
+       { 0x00439 , 0x00438 , 0x00306 },
+       { 0x00450 , 0x00435 , 0x00300 },
+       { 0x00451 , 0x00435 , 0x00308 },
+       { 0x00453 , 0x00433 , 0x00301 },
+       { 0x00457 , 0x00456 , 0x00308 },
+       { 0x0045C , 0x0043A , 0x00301 },
+       { 0x0045D , 0x00438 , 0x00300 },
+       { 0x0045E , 0x00443 , 0x00306 },
+       { 0x00476 , 0x00474 , 0x0030F },
+       { 0x00477 , 0x00475 , 0x0030F },
+       { 0x004C1 , 0x00416 , 0x00306 },
+       { 0x004C2 , 0x00436 , 0x00306 },
+       { 0x004D0 , 0x00410 , 0x00306 },
+       { 0x004D1 , 0x00430 , 0x00306 },
+       { 0x004D2 , 0x00410 , 0x00308 },
+       { 0x004D3 , 0x00430 , 0x00308 },
+       { 0x004D6 , 0x00415 , 0x00306 },
+       { 0x004D7 , 0x00435 , 0x00306 },
+       { 0x004DA , 0x004D8 , 0x00308 },
+       { 0x004DB , 0x004D9 , 0x00308 },
+       { 0x004DC , 0x00416 , 0x00308 },
+       { 0x004DD , 0x00436 , 0x00308 },
+       { 0x004DE , 0x00417 , 0x00308 },
+       { 0x004DF , 0x00437 , 0x00308 },
+       { 0x004E2 , 0x00418 , 0x00304 },
+       { 0x004E3 , 0x00438 , 0x00304 },
+       { 0x004E4 , 0x00418 , 0x00308 },
+       { 0x004E5 , 0x00438 , 0x00308 },
+       { 0x004E6 , 0x0041E , 0x00308 },
+       { 0x004E7 , 0x0043E , 0x00308 },
+       { 0x004EA , 0x004E8 , 0x00308 },
+       { 0x004EB , 0x004E9 , 0x00308 },
+       { 0x004EC , 0x0042D , 0x00308 },
+       { 0x004ED , 0x0044D , 0x00308 },
+       { 0x004EE , 0x00423 , 0x00304 },
+       { 0x004EF , 0x00443 , 0x00304 },
+       { 0x004F0 , 0x00423 , 0x00308 },
+       { 0x004F1 , 0x00443 , 0x00308 },
+       { 0x004F2 , 0x00423 , 0x0030B },
+       { 0x004F3 , 0x00443 , 0x0030B },
+       { 0x004F4 , 0x00427 , 0x00308 },
+       { 0x004F5 , 0x00447 , 0x00308 },
+       { 0x004F8 , 0x0042B , 0x00308 },
+       { 0x004F9 , 0x0044B , 0x00308 },
+       { 0x00622 , 0x00627 , 0x00653 },
+       { 0x00623 , 0x00627 , 0x00654 },
+       { 0x00624 , 0x00648 , 0x00654 },
+       { 0x00625 , 0x00627 , 0x00655 },
+       { 0x00626 , 0x0064A , 0x00654 },
+       { 0x006C0 , 0x006D5 , 0x00654 },
+       { 0x006C2 , 0x006C1 , 0x00654 },
+       { 0x006D3 , 0x006D2 , 0x00654 },
+       { 0x00929 , 0x00928 , 0x0093C },
+       { 0x00931 , 0x00930 , 0x0093C },
+       { 0x00934 , 0x00933 , 0x0093C },
+       { 0x009CB , 0x009C7 , 0x009BE },
+       { 0x009CC , 0x009C7 , 0x009D7 },
+       { 0x00B48 , 0x00B47 , 0x00B56 },
+       { 0x00B4B , 0x00B47 , 0x00B3E },
+       { 0x00B4C , 0x00B47 , 0x00B57 },
+       { 0x00B94 , 0x00B92 , 0x00BD7 },
+       { 0x00BCA , 0x00BC6 , 0x00BBE },
+       { 0x00BCB , 0x00BC7 , 0x00BBE },
+       { 0x00BCC , 0x00BC6 , 0x00BD7 },
+       { 0x00C48 , 0x00C46 , 0x00C56 },
+       { 0x00CC0 , 0x00CBF , 0x00CD5 },
+       { 0x00CC7 , 0x00CC6 , 0x00CD5 },
+       { 0x00CC8 , 0x00CC6 , 0x00CD6 },
+       { 0x00CCA , 0x00CC6 , 0x00CC2 },
+       { 0x00CCB , 0x00CCA , 0x00CD5 },
+       { 0x00D4A , 0x00D46 , 0x00D3E },
+       { 0x00D4B , 0x00D47 , 0x00D3E },
+       { 0x00D4C , 0x00D46 , 0x00D57 },
+       { 0x00DDA , 0x00DD9 , 0x00DCA },
+       { 0x00DDC , 0x00DD9 , 0x00DCF },
+       { 0x00DDD , 0x00DDC , 0x00DCA },
+       { 0x00DDE , 0x00DD9 , 0x00DDF },
+       { 0x01026 , 0x01025 , 0x0102E },
+       { 0x01B06 , 0x01B05 , 0x01B35 },
+       { 0x01B08 , 0x01B07 , 0x01B35 },
+       { 0x01B0A , 0x01B09 , 0x01B35 },
+       { 0x01B0C , 0x01B0B , 0x01B35 },
+       { 0x01B0E , 0x01B0D , 0x01B35 },
+       { 0x01B12 , 0x01B11 , 0x01B35 },
+       { 0x01B3B , 0x01B3A , 0x01B35 },
+       { 0x01B3D , 0x01B3C , 0x01B35 },
+       { 0x01B40 , 0x01B3E , 0x01B35 },
+       { 0x01B41 , 0x01B3F , 0x01B35 },
+       { 0x01B43 , 0x01B42 , 0x01B35 },
+       { 0x01E00 , 0x00041 , 0x00325 },
+       { 0x01E01 , 0x00061 , 0x00325 },
+       { 0x01E02 , 0x00042 , 0x00307 },
+       { 0x01E03 , 0x00062 , 0x00307 },
+       { 0x01E04 , 0x00042 , 0x00323 },
+       { 0x01E05 , 0x00062 , 0x00323 },
+       { 0x01E06 , 0x00042 , 0x00331 },
+       { 0x01E07 , 0x00062 , 0x00331 },
+       { 0x01E08 , 0x000C7 , 0x00301 },
+       { 0x01E09 , 0x000E7 , 0x00301 },
+       { 0x01E0A , 0x00044 , 0x00307 },
+       { 0x01E0B , 0x00064 , 0x00307 },
+       { 0x01E0C , 0x00044 , 0x00323 },
+       { 0x01E0D , 0x00064 , 0x00323 },
+       { 0x01E0E , 0x00044 , 0x00331 },
+       { 0x01E0F , 0x00064 , 0x00331 },
+       { 0x01E10 , 0x00044 , 0x00327 },
+       { 0x01E11 , 0x00064 , 0x00327 },
+       { 0x01E12 , 0x00044 , 0x0032D },
+       { 0x01E13 , 0x00064 , 0x0032D },
+       { 0x01E14 , 0x00112 , 0x00300 },
+       { 0x01E15 , 0x00113 , 0x00300 },
+       { 0x01E16 , 0x00112 , 0x00301 },
+       { 0x01E17 , 0x00113 , 0x00301 },
+       { 0x01E18 , 0x00045 , 0x0032D },
+       { 0x01E19 , 0x00065 , 0x0032D },
+       { 0x01E1A , 0x00045 , 0x00330 },
+       { 0x01E1B , 0x00065 , 0x00330 },
+       { 0x01E1C , 0x00228 , 0x00306 },
+       { 0x01E1D , 0x00229 , 0x00306 },
+       { 0x01E1E , 0x00046 , 0x00307 },
+       { 0x01E1F , 0x00066 , 0x00307 },
+       { 0x01E20 , 0x00047 , 0x00304 },
+       { 0x01E21 , 0x00067 , 0x00304 },
+       { 0x01E22 , 0x00048 , 0x00307 },
+       { 0x01E23 , 0x00068 , 0x00307 },
+       { 0x01E24 , 0x00048 , 0x00323 },
+       { 0x01E25 , 0x00068 , 0x00323 },
+       { 0x01E26 , 0x00048 , 0x00308 },
+       { 0x01E27 , 0x00068 , 0x00308 },
+       { 0x01E28 , 0x00048 , 0x00327 },
+       { 0x01E29 , 0x00068 , 0x00327 },
+       { 0x01E2A , 0x00048 , 0x0032E },
+       { 0x01E2B , 0x00068 , 0x0032E },
+       { 0x01E2C , 0x00049 , 0x00330 },
+       { 0x01E2D , 0x00069 , 0x00330 },
+       { 0x01E2E , 0x000CF , 0x00301 },
+       { 0x01E2F , 0x000EF , 0x00301 },
+       { 0x01E30 , 0x0004B , 0x00301 },
+       { 0x01E31 , 0x0006B , 0x00301 },
+       { 0x01E32 , 0x0004B , 0x00323 },
+       { 0x01E33 , 0x0006B , 0x00323 },
+       { 0x01E34 , 0x0004B , 0x00331 },
+       { 0x01E35 , 0x0006B , 0x00331 },
+       { 0x01E36 , 0x0004C , 0x00323 },
+       { 0x01E37 , 0x0006C , 0x00323 },
+       { 0x01E38 , 0x01E36 , 0x00304 },
+       { 0x01E39 , 0x01E37 , 0x00304 },
+       { 0x01E3A , 0x0004C , 0x00331 },
+       { 0x01E3B , 0x0006C , 0x00331 },
+       { 0x01E3C , 0x0004C , 0x0032D },
+       { 0x01E3D , 0x0006C , 0x0032D },
+       { 0x01E3E , 0x0004D , 0x00301 },
+       { 0x01E3F , 0x0006D , 0x00301 },
+       { 0x01E40 , 0x0004D , 0x00307 },
+       { 0x01E41 , 0x0006D , 0x00307 },
+       { 0x01E42 , 0x0004D , 0x00323 },
+       { 0x01E43 , 0x0006D , 0x00323 },
+       { 0x01E44 , 0x0004E , 0x00307 },
+       { 0x01E45 , 0x0006E , 0x00307 },
+       { 0x01E46 , 0x0004E , 0x00323 },
+       { 0x01E47 , 0x0006E , 0x00323 },
+       { 0x01E48 , 0x0004E , 0x00331 },
+       { 0x01E49 , 0x0006E , 0x00331 },
+       { 0x01E4A , 0x0004E , 0x0032D },
+       { 0x01E4B , 0x0006E , 0x0032D },
+       { 0x01E4C , 0x000D5 , 0x00301 },
+       { 0x01E4D , 0x000F5 , 0x00301 },
+       { 0x01E4E , 0x000D5 , 0x00308 },
+       { 0x01E4F , 0x000F5 , 0x00308 },
+       { 0x01E50 , 0x0014C , 0x00300 },
+       { 0x01E51 , 0x0014D , 0x00300 },
+       { 0x01E52 , 0x0014C , 0x00301 },
+       { 0x01E53 , 0x0014D , 0x00301 },
+       { 0x01E54 , 0x00050 , 0x00301 },
+       { 0x01E55 , 0x00070 , 0x00301 },
+       { 0x01E56 , 0x00050 , 0x00307 },
+       { 0x01E57 , 0x00070 , 0x00307 },
+       { 0x01E58 , 0x00052 , 0x00307 },
+       { 0x01E59 , 0x00072 , 0x00307 },
+       { 0x01E5A , 0x00052 , 0x00323 },
+       { 0x01E5B , 0x00072 , 0x00323 },
+       { 0x01E5C , 0x01E5A , 0x00304 },
+       { 0x01E5D , 0x01E5B , 0x00304 },
+       { 0x01E5E , 0x00052 , 0x00331 },
+       { 0x01E5F , 0x00072 , 0x00331 },
+       { 0x01E60 , 0x00053 , 0x00307 },
+       { 0x01E61 , 0x00073 , 0x00307 },
+       { 0x01E62 , 0x00053 , 0x00323 },
+       { 0x01E63 , 0x00073 , 0x00323 },
+       { 0x01E64 , 0x0015A , 0x00307 },
+       { 0x01E65 , 0x0015B , 0x00307 },
+       { 0x01E66 , 0x00160 , 0x00307 },
+       { 0x01E67 , 0x00161 , 0x00307 },
+       { 0x01E68 , 0x01E62 , 0x00307 },
+       { 0x01E69 , 0x01E63 , 0x00307 },
+       { 0x01E6A , 0x00054 , 0x00307 },
+       { 0x01E6B , 0x00074 , 0x00307 },
+       { 0x01E6C , 0x00054 , 0x00323 },
+       { 0x01E6D , 0x00074 , 0x00323 },
+       { 0x01E6E , 0x00054 , 0x00331 },
+       { 0x01E6F , 0x00074 , 0x00331 },
+       { 0x01E70 , 0x00054 , 0x0032D },
+       { 0x01E71 , 0x00074 , 0x0032D },
+       { 0x01E72 , 0x00055 , 0x00324 },
+       { 0x01E73 , 0x00075 , 0x00324 },
+       { 0x01E74 , 0x00055 , 0x00330 },
+       { 0x01E75 , 0x00075 , 0x00330 },
+       { 0x01E76 , 0x00055 , 0x0032D },
+       { 0x01E77 , 0x00075 , 0x0032D },
+       { 0x01E78 , 0x00168 , 0x00301 },
+       { 0x01E79 , 0x00169 , 0x00301 },
+       { 0x01E7A , 0x0016A , 0x00308 },
+       { 0x01E7B , 0x0016B , 0x00308 },
+       { 0x01E7C , 0x00056 , 0x00303 },
+       { 0x01E7D , 0x00076 , 0x00303 },
+       { 0x01E7E , 0x00056 , 0x00323 },
+       { 0x01E7F , 0x00076 , 0x00323 },
+       { 0x01E80 , 0x00057 , 0x00300 },
+       { 0x01E81 , 0x00077 , 0x00300 },
+       { 0x01E82 , 0x00057 , 0x00301 },
+       { 0x01E83 , 0x00077 , 0x00301 },
+       { 0x01E84 , 0x00057 , 0x00308 },
+       { 0x01E85 , 0x00077 , 0x00308 },
+       { 0x01E86 , 0x00057 , 0x00307 },
+       { 0x01E87 , 0x00077 , 0x00307 },
+       { 0x01E88 , 0x00057 , 0x00323 },
+       { 0x01E89 , 0x00077 , 0x00323 },
+       { 0x01E8A , 0x00058 , 0x00307 },
+       { 0x01E8B , 0x00078 , 0x00307 },
+       { 0x01E8C , 0x00058 , 0x00308 },
+       { 0x01E8D , 0x00078 , 0x00308 },
+       { 0x01E8E , 0x00059 , 0x00307 },
+       { 0x01E8F , 0x00079 , 0x00307 },
+       { 0x01E90 , 0x0005A , 0x00302 },
+       { 0x01E91 , 0x0007A , 0x00302 },
+       { 0x01E92 , 0x0005A , 0x00323 },
+       { 0x01E93 , 0x0007A , 0x00323 },
+       { 0x01E94 , 0x0005A , 0x00331 },
+       { 0x01E95 , 0x0007A , 0x00331 },
+       { 0x01E96 , 0x00068 , 0x00331 },
+       { 0x01E97 , 0x00074 , 0x00308 },
+       { 0x01E98 , 0x00077 , 0x0030A },
+       { 0x01E99 , 0x00079 , 0x0030A },
+       { 0x01E9B , 0x0017F , 0x00307 },
+       { 0x01EA0 , 0x00041 , 0x00323 },
+       { 0x01EA1 , 0x00061 , 0x00323 },
+       { 0x01EA2 , 0x00041 , 0x00309 },
+       { 0x01EA3 , 0x00061 , 0x00309 },
+       { 0x01EA4 , 0x000C2 , 0x00301 },
+       { 0x01EA5 , 0x000E2 , 0x00301 },
+       { 0x01EA6 , 0x000C2 , 0x00300 },
+       { 0x01EA7 , 0x000E2 , 0x00300 },
+       { 0x01EA8 , 0x000C2 , 0x00309 },
+       { 0x01EA9 , 0x000E2 , 0x00309 },
+       { 0x01EAA , 0x000C2 , 0x00303 },
+       { 0x01EAB , 0x000E2 , 0x00303 },
+       { 0x01EAC , 0x01EA0 , 0x00302 },
+       { 0x01EAD , 0x01EA1 , 0x00302 },
+       { 0x01EAE , 0x00102 , 0x00301 },
+       { 0x01EAF , 0x00103 , 0x00301 },
+       { 0x01EB0 , 0x00102 , 0x00300 },
+       { 0x01EB1 , 0x00103 , 0x00300 },
+       { 0x01EB2 , 0x00102 , 0x00309 },
+       { 0x01EB3 , 0x00103 , 0x00309 },
+       { 0x01EB4 , 0x00102 , 0x00303 },
+       { 0x01EB5 , 0x00103 , 0x00303 },
+       { 0x01EB6 , 0x01EA0 , 0x00306 },
+       { 0x01EB7 , 0x01EA1 , 0x00306 },
+       { 0x01EB8 , 0x00045 , 0x00323 },
+       { 0x01EB9 , 0x00065 , 0x00323 },
+       { 0x01EBA , 0x00045 , 0x00309 },
+       { 0x01EBB , 0x00065 , 0x00309 },
+       { 0x01EBC , 0x00045 , 0x00303 },
+       { 0x01EBD , 0x00065 , 0x00303 },
+       { 0x01EBE , 0x000CA , 0x00301 },
+       { 0x01EBF , 0x000EA , 0x00301 },
+       { 0x01EC0 , 0x000CA , 0x00300 },
+       { 0x01EC1 , 0x000EA , 0x00300 },
+       { 0x01EC2 , 0x000CA , 0x00309 },
+       { 0x01EC3 , 0x000EA , 0x00309 },
+       { 0x01EC4 , 0x000CA , 0x00303 },
+       { 0x01EC5 , 0x000EA , 0x00303 },
+       { 0x01EC6 , 0x01EB8 , 0x00302 },
+       { 0x01EC7 , 0x01EB9 , 0x00302 },
+       { 0x01EC8 , 0x00049 , 0x00309 },
+       { 0x01EC9 , 0x00069 , 0x00309 },
+       { 0x01ECA , 0x00049 , 0x00323 },
+       { 0x01ECB , 0x00069 , 0x00323 },
+       { 0x01ECC , 0x0004F , 0x00323 },
+       { 0x01ECD , 0x0006F , 0x00323 },
+       { 0x01ECE , 0x0004F , 0x00309 },
+       { 0x01ECF , 0x0006F , 0x00309 },
+       { 0x01ED0 , 0x000D4 , 0x00301 },
+       { 0x01ED1 , 0x000F4 , 0x00301 },
+       { 0x01ED2 , 0x000D4 , 0x00300 },
+       { 0x01ED3 , 0x000F4 , 0x00300 },
+       { 0x01ED4 , 0x000D4 , 0x00309 },
+       { 0x01ED5 , 0x000F4 , 0x00309 },
+       { 0x01ED6 , 0x000D4 , 0x00303 },
+       { 0x01ED7 , 0x000F4 , 0x00303 },
+       { 0x01ED8 , 0x01ECC , 0x00302 },
+       { 0x01ED9 , 0x01ECD , 0x00302 },
+       { 0x01EDA , 0x001A0 , 0x00301 },
+       { 0x01EDB , 0x001A1 , 0x00301 },
+       { 0x01EDC , 0x001A0 , 0x00300 },
+       { 0x01EDD , 0x001A1 , 0x00300 },
+       { 0x01EDE , 0x001A0 , 0x00309 },
+       { 0x01EDF , 0x001A1 , 0x00309 },
+       { 0x01EE0 , 0x001A0 , 0x00303 },
+       { 0x01EE1 , 0x001A1 , 0x00303 },
+       { 0x01EE2 , 0x001A0 , 0x00323 },
+       { 0x01EE3 , 0x001A1 , 0x00323 },
+       { 0x01EE4 , 0x00055 , 0x00323 },
+       { 0x01EE5 , 0x00075 , 0x00323 },
+       { 0x01EE6 , 0x00055 , 0x00309 },
+       { 0x01EE7 , 0x00075 , 0x00309 },
+       { 0x01EE8 , 0x001AF , 0x00301 },
+       { 0x01EE9 , 0x001B0 , 0x00301 },
+       { 0x01EEA , 0x001AF , 0x00300 },
+       { 0x01EEB , 0x001B0 , 0x00300 },
+       { 0x01EEC , 0x001AF , 0x00309 },
+       { 0x01EED , 0x001B0 , 0x00309 },
+       { 0x01EEE , 0x001AF , 0x00303 },
+       { 0x01EEF , 0x001B0 , 0x00303 },
+       { 0x01EF0 , 0x001AF , 0x00323 },
+       { 0x01EF1 , 0x001B0 , 0x00323 },
+       { 0x01EF2 , 0x00059 , 0x00300 },
+       { 0x01EF3 , 0x00079 , 0x00300 },
+       { 0x01EF4 , 0x00059 , 0x00323 },
+       { 0x01EF5 , 0x00079 , 0x00323 },
+       { 0x01EF6 , 0x00059 , 0x00309 },
+       { 0x01EF7 , 0x00079 , 0x00309 },
+       { 0x01EF8 , 0x00059 , 0x00303 },
+       { 0x01EF9 , 0x00079 , 0x00303 },
+       { 0x01F00 , 0x003B1 , 0x00313 },
+       { 0x01F01 , 0x003B1 , 0x00314 },
+       { 0x01F02 , 0x01F00 , 0x00300 },
+       { 0x01F03 , 0x01F01 , 0x00300 },
+       { 0x01F04 , 0x01F00 , 0x00301 },
+       { 0x01F05 , 0x01F01 , 0x00301 },
+       { 0x01F06 , 0x01F00 , 0x00342 },
+       { 0x01F07 , 0x01F01 , 0x00342 },
+       { 0x01F08 , 0x00391 , 0x00313 },
+       { 0x01F09 , 0x00391 , 0x00314 },
+       { 0x01F0A , 0x01F08 , 0x00300 },
+       { 0x01F0B , 0x01F09 , 0x00300 },
+       { 0x01F0C , 0x01F08 , 0x00301 },
+       { 0x01F0D , 0x01F09 , 0x00301 },
+       { 0x01F0E , 0x01F08 , 0x00342 },
+       { 0x01F0F , 0x01F09 , 0x00342 },
+       { 0x01F10 , 0x003B5 , 0x00313 },
+       { 0x01F11 , 0x003B5 , 0x00314 },
+       { 0x01F12 , 0x01F10 , 0x00300 },
+       { 0x01F13 , 0x01F11 , 0x00300 },
+       { 0x01F14 , 0x01F10 , 0x00301 },
+       { 0x01F15 , 0x01F11 , 0x00301 },
+       { 0x01F18 , 0x00395 , 0x00313 },
+       { 0x01F19 , 0x00395 , 0x00314 },
+       { 0x01F1A , 0x01F18 , 0x00300 },
+       { 0x01F1B , 0x01F19 , 0x00300 },
+       { 0x01F1C , 0x01F18 , 0x00301 },
+       { 0x01F1D , 0x01F19 , 0x00301 },
+       { 0x01F20 , 0x003B7 , 0x00313 },
+       { 0x01F21 , 0x003B7 , 0x00314 },
+       { 0x01F22 , 0x01F20 , 0x00300 },
+       { 0x01F23 , 0x01F21 , 0x00300 },
+       { 0x01F24 , 0x01F20 , 0x00301 },
+       { 0x01F25 , 0x01F21 , 0x00301 },
+       { 0x01F26 , 0x01F20 , 0x00342 },
+       { 0x01F27 , 0x01F21 , 0x00342 },
+       { 0x01F28 , 0x00397 , 0x00313 },
+       { 0x01F29 , 0x00397 , 0x00314 },
+       { 0x01F2A , 0x01F28 , 0x00300 },
+       { 0x01F2B , 0x01F29 , 0x00300 },
+       { 0x01F2C , 0x01F28 , 0x00301 },
+       { 0x01F2D , 0x01F29 , 0x00301 },
+       { 0x01F2E , 0x01F28 , 0x00342 },
+       { 0x01F2F , 0x01F29 , 0x00342 },
+       { 0x01F30 , 0x003B9 , 0x00313 },
+       { 0x01F31 , 0x003B9 , 0x00314 },
+       { 0x01F32 , 0x01F30 , 0x00300 },
+       { 0x01F33 , 0x01F31 , 0x00300 },
+       { 0x01F34 , 0x01F30 , 0x00301 },
+       { 0x01F35 , 0x01F31 , 0x00301 },
+       { 0x01F36 , 0x01F30 , 0x00342 },
+       { 0x01F37 , 0x01F31 , 0x00342 },
+       { 0x01F38 , 0x00399 , 0x00313 },
+       { 0x01F39 , 0x00399 , 0x00314 },
+       { 0x01F3A , 0x01F38 , 0x00300 },
+       { 0x01F3B , 0x01F39 , 0x00300 },
+       { 0x01F3C , 0x01F38 , 0x00301 },
+       { 0x01F3D , 0x01F39 , 0x00301 },
+       { 0x01F3E , 0x01F38 , 0x00342 },
+       { 0x01F3F , 0x01F39 , 0x00342 },
+       { 0x01F40 , 0x003BF , 0x00313 },
+       { 0x01F41 , 0x003BF , 0x00314 },
+       { 0x01F42 , 0x01F40 , 0x00300 },
+       { 0x01F43 , 0x01F41 , 0x00300 },
+       { 0x01F44 , 0x01F40 , 0x00301 },
+       { 0x01F45 , 0x01F41 , 0x00301 },
+       { 0x01F48 , 0x0039F , 0x00313 },
+       { 0x01F49 , 0x0039F , 0x00314 },
+       { 0x01F4A , 0x01F48 , 0x00300 },
+       { 0x01F4B , 0x01F49 , 0x00300 },
+       { 0x01F4C , 0x01F48 , 0x00301 },
+       { 0x01F4D , 0x01F49 , 0x00301 },
+       { 0x01F50 , 0x003C5 , 0x00313 },
+       { 0x01F51 , 0x003C5 , 0x00314 },
+       { 0x01F52 , 0x01F50 , 0x00300 },
+       { 0x01F53 , 0x01F51 , 0x00300 },
+       { 0x01F54 , 0x01F50 , 0x00301 },
+       { 0x01F55 , 0x01F51 , 0x00301 },
+       { 0x01F56 , 0x01F50 , 0x00342 },
+       { 0x01F57 , 0x01F51 , 0x00342 },
+       { 0x01F59 , 0x003A5 , 0x00314 },
+       { 0x01F5B , 0x01F59 , 0x00300 },
+       { 0x01F5D , 0x01F59 , 0x00301 },
+       { 0x01F5F , 0x01F59 , 0x00342 },
+       { 0x01F60 , 0x003C9 , 0x00313 },
+       { 0x01F61 , 0x003C9 , 0x00314 },
+       { 0x01F62 , 0x01F60 , 0x00300 },
+       { 0x01F63 , 0x01F61 , 0x00300 },
+       { 0x01F64 , 0x01F60 , 0x00301 },
+       { 0x01F65 , 0x01F61 , 0x00301 },
+       { 0x01F66 , 0x01F60 , 0x00342 },
+       { 0x01F67 , 0x01F61 , 0x00342 },
+       { 0x01F68 , 0x003A9 , 0x00313 },
+       { 0x01F69 , 0x003A9 , 0x00314 },
+       { 0x01F6A , 0x01F68 , 0x00300 },
+       { 0x01F6B , 0x01F69 , 0x00300 },
+       { 0x01F6C , 0x01F68 , 0x00301 },
+       { 0x01F6D , 0x01F69 , 0x00301 },
+       { 0x01F6E , 0x01F68 , 0x00342 },
+       { 0x01F6F , 0x01F69 , 0x00342 },
+       { 0x01F70 , 0x003B1 , 0x00300 },
+       { 0x01F72 , 0x003B5 , 0x00300 },
+       { 0x01F74 , 0x003B7 , 0x00300 },
+       { 0x01F76 , 0x003B9 , 0x00300 },
+       { 0x01F78 , 0x003BF , 0x00300 },
+       { 0x01F7A , 0x003C5 , 0x00300 },
+       { 0x01F7C , 0x003C9 , 0x00300 },
+       { 0x01F80 , 0x01F00 , 0x00345 },
+       { 0x01F81 , 0x01F01 , 0x00345 },
+       { 0x01F82 , 0x01F02 , 0x00345 },
+       { 0x01F83 , 0x01F03 , 0x00345 },
+       { 0x01F84 , 0x01F04 , 0x00345 },
+       { 0x01F85 , 0x01F05 , 0x00345 },
+       { 0x01F86 , 0x01F06 , 0x00345 },
+       { 0x01F87 , 0x01F07 , 0x00345 },
+       { 0x01F88 , 0x01F08 , 0x00345 },
+       { 0x01F89 , 0x01F09 , 0x00345 },
+       { 0x01F8A , 0x01F0A , 0x00345 },
+       { 0x01F8B , 0x01F0B , 0x00345 },
+       { 0x01F8C , 0x01F0C , 0x00345 },
+       { 0x01F8D , 0x01F0D , 0x00345 },
+       { 0x01F8E , 0x01F0E , 0x00345 },
+       { 0x01F8F , 0x01F0F , 0x00345 },
+       { 0x01F90 , 0x01F20 , 0x00345 },
+       { 0x01F91 , 0x01F21 , 0x00345 },
+       { 0x01F92 , 0x01F22 , 0x00345 },
+       { 0x01F93 , 0x01F23 , 0x00345 },
+       { 0x01F94 , 0x01F24 , 0x00345 },
+       { 0x01F95 , 0x01F25 , 0x00345 },
+       { 0x01F96 , 0x01F26 , 0x00345 },
+       { 0x01F97 , 0x01F27 , 0x00345 },
+       { 0x01F98 , 0x01F28 , 0x00345 },
+       { 0x01F99 , 0x01F29 , 0x00345 },
+       { 0x01F9A , 0x01F2A , 0x00345 },
+       { 0x01F9B , 0x01F2B , 0x00345 },
+       { 0x01F9C , 0x01F2C , 0x00345 },
+       { 0x01F9D , 0x01F2D , 0x00345 },
+       { 0x01F9E , 0x01F2E , 0x00345 },
+       { 0x01F9F , 0x01F2F , 0x00345 },
+       { 0x01FA0 , 0x01F60 , 0x00345 },
+       { 0x01FA1 , 0x01F61 , 0x00345 },
+       { 0x01FA2 , 0x01F62 , 0x00345 },
+       { 0x01FA3 , 0x01F63 , 0x00345 },
+       { 0x01FA4 , 0x01F64 , 0x00345 },
+       { 0x01FA5 , 0x01F65 , 0x00345 },
+       { 0x01FA6 , 0x01F66 , 0x00345 },
+       { 0x01FA7 , 0x01F67 , 0x00345 },
+       { 0x01FA8 , 0x01F68 , 0x00345 },
+       { 0x01FA9 , 0x01F69 , 0x00345 },
+       { 0x01FAA , 0x01F6A , 0x00345 },
+       { 0x01FAB , 0x01F6B , 0x00345 },
+       { 0x01FAC , 0x01F6C , 0x00345 },
+       { 0x01FAD , 0x01F6D , 0x00345 },
+       { 0x01FAE , 0x01F6E , 0x00345 },
+       { 0x01FAF , 0x01F6F , 0x00345 },
+       { 0x01FB0 , 0x003B1 , 0x00306 },
+       { 0x01FB1 , 0x003B1 , 0x00304 },
+       { 0x01FB2 , 0x01F70 , 0x00345 },
+       { 0x01FB3 , 0x003B1 , 0x00345 },
+       { 0x01FB4 , 0x003AC , 0x00345 },
+       { 0x01FB6 , 0x003B1 , 0x00342 },
+       { 0x01FB7 , 0x01FB6 , 0x00345 },
+       { 0x01FB8 , 0x00391 , 0x00306 },
+       { 0x01FB9 , 0x00391 , 0x00304 },
+       { 0x01FBA , 0x00391 , 0x00300 },
+       { 0x01FBC , 0x00391 , 0x00345 },
+       { 0x01FC1 , 0x000A8 , 0x00342 },
+       { 0x01FC2 , 0x01F74 , 0x00345 },
+       { 0x01FC3 , 0x003B7 , 0x00345 },
+       { 0x01FC4 , 0x003AE , 0x00345 },
+       { 0x01FC6 , 0x003B7 , 0x00342 },
+       { 0x01FC7 , 0x01FC6 , 0x00345 },
+       { 0x01FC8 , 0x00395 , 0x00300 },
+       { 0x01FCA , 0x00397 , 0x00300 },
+       { 0x01FCC , 0x00397 , 0x00345 },
+       { 0x01FCD , 0x01FBF , 0x00300 },
+       { 0x01FCE , 0x01FBF , 0x00301 },
+       { 0x01FCF , 0x01FBF , 0x00342 },
+       { 0x01FD0 , 0x003B9 , 0x00306 },
+       { 0x01FD1 , 0x003B9 , 0x00304 },
+       { 0x01FD2 , 0x003CA , 0x00300 },
+       { 0x01FD6 , 0x003B9 , 0x00342 },
+       { 0x01FD7 , 0x003CA , 0x00342 },
+       { 0x01FD8 , 0x00399 , 0x00306 },
+       { 0x01FD9 , 0x00399 , 0x00304 },
+       { 0x01FDA , 0x00399 , 0x00300 },
+       { 0x01FDD , 0x01FFE , 0x00300 },
+       { 0x01FDE , 0x01FFE , 0x00301 },
+       { 0x01FDF , 0x01FFE , 0x00342 },
+       { 0x01FE0 , 0x003C5 , 0x00306 },
+       { 0x01FE1 , 0x003C5 , 0x00304 },
+       { 0x01FE2 , 0x003CB , 0x00300 },
+       { 0x01FE4 , 0x003C1 , 0x00313 },
+       { 0x01FE5 , 0x003C1 , 0x00314 },
+       { 0x01FE6 , 0x003C5 , 0x00342 },
+       { 0x01FE7 , 0x003CB , 0x00342 },
+       { 0x01FE8 , 0x003A5 , 0x00306 },
+       { 0x01FE9 , 0x003A5 , 0x00304 },
+       { 0x01FEA , 0x003A5 , 0x00300 },
+       { 0x01FEC , 0x003A1 , 0x00314 },
+       { 0x01FED , 0x000A8 , 0x00300 },
+       { 0x01FF2 , 0x01F7C , 0x00345 },
+       { 0x01FF3 , 0x003C9 , 0x00345 },
+       { 0x01FF4 , 0x003CE , 0x00345 },
+       { 0x01FF6 , 0x003C9 , 0x00342 },
+       { 0x01FF7 , 0x01FF6 , 0x00345 },
+       { 0x01FF8 , 0x0039F , 0x00300 },
+       { 0x01FFA , 0x003A9 , 0x00300 },
+       { 0x01FFC , 0x003A9 , 0x00345 },
+       { 0x0219A , 0x02190 , 0x00338 },
+       { 0x0219B , 0x02192 , 0x00338 },
+       { 0x021AE , 0x02194 , 0x00338 },
+       { 0x021CD , 0x021D0 , 0x00338 },
+       { 0x021CE , 0x021D4 , 0x00338 },
+       { 0x021CF , 0x021D2 , 0x00338 },
+       { 0x02204 , 0x02203 , 0x00338 },
+       { 0x02209 , 0x02208 , 0x00338 },
+       { 0x0220C , 0x0220B , 0x00338 },
+       { 0x02224 , 0x02223 , 0x00338 },
+       { 0x02226 , 0x02225 , 0x00338 },
+       { 0x02241 , 0x0223C , 0x00338 },
+       { 0x02244 , 0x02243 , 0x00338 },
+       { 0x02247 , 0x02245 , 0x00338 },
+       { 0x02249 , 0x02248 , 0x00338 },
+       { 0x02260 , 0x0003D , 0x00338 },
+       { 0x02262 , 0x02261 , 0x00338 },
+       { 0x0226D , 0x0224D , 0x00338 },
+       { 0x0226E , 0x0003C , 0x00338 },
+       { 0x0226F , 0x0003E , 0x00338 },
+       { 0x02270 , 0x02264 , 0x00338 },
+       { 0x02271 , 0x02265 , 0x00338 },
+       { 0x02274 , 0x02272 , 0x00338 },
+       { 0x02275 , 0x02273 , 0x00338 },
+       { 0x02278 , 0x02276 , 0x00338 },
+       { 0x02279 , 0x02277 , 0x00338 },
+       { 0x02280 , 0x0227A , 0x00338 },
+       { 0x02281 , 0x0227B , 0x00338 },
+       { 0x02284 , 0x02282 , 0x00338 },
+       { 0x02285 , 0x02283 , 0x00338 },
+       { 0x02288 , 0x02286 , 0x00338 },
+       { 0x02289 , 0x02287 , 0x00338 },
+       { 0x022AC , 0x022A2 , 0x00338 },
+       { 0x022AD , 0x022A8 , 0x00338 },
+       { 0x022AE , 0x022A9 , 0x00338 },
+       { 0x022AF , 0x022AB , 0x00338 },
+       { 0x022E0 , 0x0227C , 0x00338 },
+       { 0x022E1 , 0x0227D , 0x00338 },
+       { 0x022E2 , 0x02291 , 0x00338 },
+       { 0x022E3 , 0x02292 , 0x00338 },
+       { 0x022EA , 0x022B2 , 0x00338 },
+       { 0x022EB , 0x022B3 , 0x00338 },
+       { 0x022EC , 0x022B4 , 0x00338 },
+       { 0x022ED , 0x022B5 , 0x00338 },
+       { 0x0304C , 0x0304B , 0x03099 },
+       { 0x0304E , 0x0304D , 0x03099 },
+       { 0x03050 , 0x0304F , 0x03099 },
+       { 0x03052 , 0x03051 , 0x03099 },
+       { 0x03054 , 0x03053 , 0x03099 },
+       { 0x03056 , 0x03055 , 0x03099 },
+       { 0x03058 , 0x03057 , 0x03099 },
+       { 0x0305A , 0x03059 , 0x03099 },
+       { 0x0305C , 0x0305B , 0x03099 },
+       { 0x0305E , 0x0305D , 0x03099 },
+       { 0x03060 , 0x0305F , 0x03099 },
+       { 0x03062 , 0x03061 , 0x03099 },
+       { 0x03065 , 0x03064 , 0x03099 },
+       { 0x03067 , 0x03066 , 0x03099 },
+       { 0x03069 , 0x03068 , 0x03099 },
+       { 0x03070 , 0x0306F , 0x03099 },
+       { 0x03071 , 0x0306F , 0x0309A },
+       { 0x03073 , 0x03072 , 0x03099 },
+       { 0x03074 , 0x03072 , 0x0309A },
+       { 0x03076 , 0x03075 , 0x03099 },
+       { 0x03077 , 0x03075 , 0x0309A },
+       { 0x03079 , 0x03078 , 0x03099 },
+       { 0x0307A , 0x03078 , 0x0309A },
+       { 0x0307C , 0x0307B , 0x03099 },
+       { 0x0307D , 0x0307B , 0x0309A },
+       { 0x03094 , 0x03046 , 0x03099 },
+       { 0x0309E , 0x0309D , 0x03099 },
+       { 0x030AC , 0x030AB , 0x03099 },
+       { 0x030AE , 0x030AD , 0x03099 },
+       { 0x030B0 , 0x030AF , 0x03099 },
+       { 0x030B2 , 0x030B1 , 0x03099 },
+       { 0x030B4 , 0x030B3 , 0x03099 },
+       { 0x030B6 , 0x030B5 , 0x03099 },
+       { 0x030B8 , 0x030B7 , 0x03099 },
+       { 0x030BA , 0x030B9 , 0x03099 },
+       { 0x030BC , 0x030BB , 0x03099 },
+       { 0x030BE , 0x030BD , 0x03099 },
+       { 0x030C0 , 0x030BF , 0x03099 },
+       { 0x030C2 , 0x030C1 , 0x03099 },
+       { 0x030C5 , 0x030C4 , 0x03099 },
+       { 0x030C7 , 0x030C6 , 0x03099 },
+       { 0x030C9 , 0x030C8 , 0x03099 },
+       { 0x030D0 , 0x030CF , 0x03099 },
+       { 0x030D1 , 0x030CF , 0x0309A },
+       { 0x030D3 , 0x030D2 , 0x03099 },
+       { 0x030D4 , 0x030D2 , 0x0309A },
+       { 0x030D6 , 0x030D5 , 0x03099 },
+       { 0x030D7 , 0x030D5 , 0x0309A },
+       { 0x030D9 , 0x030D8 , 0x03099 },
+       { 0x030DA , 0x030D8 , 0x0309A },
+       { 0x030DC , 0x030DB , 0x03099 },
+       { 0x030DD , 0x030DB , 0x0309A },
+       { 0x030F4 , 0x030A6 , 0x03099 },
+       { 0x030F7 , 0x030EF , 0x03099 },
+       { 0x030F8 , 0x030F0 , 0x03099 },
+       { 0x030F9 , 0x030F1 , 0x03099 },
+       { 0x030FA , 0x030F2 , 0x03099 },
+       { 0x030FE , 0x030FD , 0x03099 },
+       { 0x1109A , 0x11099 , 0x110BA },
+       { 0x1109C , 0x1109B , 0x110BA },
+       { 0x110AB , 0x110A5 , 0x110BA },
+};
+
 #endif /* ARCHIVE_STRING_COMPOSITION_H_INCLUDED */
+

diff --git a/libarchive/test/test_archive_string_conversion.c b/libarchive/test/test_archive_string_conversion.c
index 8b833ea6f705d18f294addb8b306c4086a3cb374..3d217a10f996dcfbaa11f04b3fe133478d770219 100644 (file)
--- a/libarchive/test/test_archive_string_conversion.c
+++ b/libarchive/test/test_archive_string_conversion.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011 Michihiro NAKAJIMA
+ * Copyright (c) 2011-2012 Michihiro NAKAJIMA
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,7 +34,7 @@ __FBSDID("$FreeBSD$");
 Execute the following to rebuild the data for this program:
    tail -n +36 test_archive_string_conversion.c | /bin/sh
 #
-# This requires http://unicode.org/Public/UNIDATA/NormalizationTest.txt
+# This requires http://unicode.org/Public/6.0.0/ucd/NormalizationTest.txt
 #
 if="NormalizationTest.txt"
 if [ ! -f ${if} ]; then
@@ -158,7 +158,7 @@ unicode_to_wc(wchar_t *wp, uint32_t uc)
  */
 static int
 scan_unicode_pattern(char *out, wchar_t *wout, char *u16be, char *u16le,
-    const char *pattern, int exclude_mac_nfd)
+    const char *pattern, int mac_nfd)
 {
        unsigned uc = 0;
        const char *p = pattern;
@@ -166,6 +166,7 @@ scan_unicode_pattern(char *out, wchar_t *wout, char *u16be, char *u16le,
        wchar_t *owp = wout;
        char *op16be = u16be;
        char *op16le = u16le;
+       int ret = 0;
 
        for (;;) {
                if (*p >= '0' && *p <= '9')
@@ -173,14 +174,31 @@ scan_unicode_pattern(char *out, wchar_t *wout, char *u16be, char *u16le,
                else if (*p >= 'A' && *p <= 'F')
                        uc = (uc << 4) + (*p - 'A' + 0x0a);
                else {
-                       if (exclude_mac_nfd) {
+                       if (mac_nfd && op == out) {
                                /*
                                 * These are not converted to NFD on Mac OS.
+                                * U+2000 - U+2FFF
+                                * U+F900 - U+FAFF
+                                * U+2F800 - U+2FAFF
                                 */
-                               if ((uc >= 0x2000 && uc <= 0x2FFF) ||
-                                   (uc >= 0xF900 && uc <= 0xFAFF) ||
-                                   (uc >= 0x2F800 && uc <= 0x2FAFF))
-                                       return (-1);
+                               switch (uc) {
+                               case 0x2194: case 0x219A: case 0x219B:
+                               case 0x21AE: case 0x21CD: case 0x21CE:
+                               case 0x21CF: case 0x2204: case 0x2209:
+                               case 0x220C: case 0x2224: case 0x2226:
+                               case 0x2241: case 0x2244: case 0x2247:
+                               case 0x2249: case 0x2260: case 0x2262:
+                               case 0x226D: case 0x226E: case 0x226F:
+                               case 0x2270: case 0x2271: case 0x2274:
+                               case 0x2275: case 0x2276: case 0x2278:
+                               case 0x2279: case 0x227A: case 0x227B:
+                               case 0x2280: case 0x2281: case 0x2284:
+                               case 0x2285: case 0x2288: case 0x2289:
+                               case 0x22AC: case 0x22AD: case 0x22AE:
+                               case 0x22AF: case 0x22E0: case 0x22E1:
+                               case 0x22E2: case 0x22E3: case 0x22EA:
+                               case 0x22EB: case 0x22EC: case 0x22ED:
+                               
                                /*
                                 * Those code points are not converted to
                                 * NFD on Mac OS. I do not know the reason
@@ -190,9 +208,10 @@ scan_unicode_pattern(char *out, wchar_t *wout, char *u16be, char *u16le,
                                 *   1109C  ==> 1109B 110BA
                                 *   110AB  ==> 110A5 110BA
                                 */
-                               if (uc == 0x1109A || uc == 0x1109C ||
-                                   uc == 0x110AB)
-                                       return (-1);
+                               case 0x1109A: case 0x1109C: case 0x110AB:
+                                       ret = 1;
+                                       break;
+                               }
                        }
                        op16be += unicode_to_utf16be(op16be, uc);
                        op16le += unicode_to_utf16le(op16le, uc);
@@ -211,7 +230,7 @@ scan_unicode_pattern(char *out, wchar_t *wout, char *u16be, char *u16le,
                }
                p++;
        }
-       return (0);
+       return (ret);
 }
 
 static int
@@ -230,7 +249,7 @@ is_wc_unicode(void)
  * On other platforms, the characters to be Form C.
  */
 static void
-test_archive_string_normalization(void)
+test_archive_string_normalization(int mac_nfd)
 {
        struct archive *a, *a2;
        struct archive_entry *ae;
@@ -244,6 +263,7 @@ test_archive_string_normalization(void)
        ssize_t size;
        int line = 0;
        int locale_is_utf8, wc_is_unicode;
+       int sconv_opt = 0;
 
        locale_is_utf8 = (NULL != setlocale(LC_ALL, "en_US.UTF-8"));
        wc_is_unicode = is_wc_unicode();
@@ -257,6 +277,11 @@ test_archive_string_normalization(void)
        archive_string_init(&utf8);
        memset(&mstr, 0, sizeof(mstr));
 
+       if (mac_nfd)
+               sconv_opt = SCONV_SET_OPT_NORMALIZATION_D;
+       else
+               sconv_opt = SCONV_SET_OPT_NORMALIZATION_C;
+
        /*
         * Extract a test pattern file.
         */
@@ -296,6 +321,10 @@ test_archive_string_normalization(void)
                assertEqualInt(ARCHIVE_OK, archive_read_free(a));
                return;
        }
+       archive_string_conversion_set_opt(f_sconv8, sconv_opt);
+       archive_string_conversion_set_opt(f_sconv16be, sconv_opt);
+       archive_string_conversion_set_opt(f_sconv16le, sconv_opt);
+       archive_string_conversion_set_opt(t_sconv8, sconv_opt);
 
        /*
         * Read test data.
@@ -311,6 +340,10 @@ test_archive_string_normalization(void)
                char utf16le_nfc[80], utf16le_nfd[80];
                wchar_t wc_nfc[40], wc_nfd[40];
                char *e, *p;
+               const wchar_t *wp;
+               const char *mp;
+               size_t mplen;
+               int should_be_nfc;
 
                line++;
                if (buff[0] == '#')
@@ -332,33 +365,33 @@ test_archive_string_normalization(void)
                nfd[sizeof(nfd)-1] = '\0';
 
                /*
-                * Convert an NFC pattern to UTF-8 bytes.
+                * Get an NFC patterns.
                 */
-#if defined(__APPLE__)
-               if (scan_unicode_pattern(utf8_nfc, wc_nfc, utf16be_nfc, utf16le_nfc,
-                   nfc, 1) != 0)
-                       continue;
-#else
-               scan_unicode_pattern(utf8_nfc, wc_nfc, utf16be_nfc, utf16le_nfc,
-                   nfc, 0);
-#endif
+               should_be_nfc = scan_unicode_pattern(utf8_nfc, wc_nfc,
+                       utf16be_nfc, utf16le_nfc, nfc, mac_nfd);
 
                /*
-                * Convert an NFD pattern to UTF-8 bytes.
+                * Get an NFD patterns.
                 */
                scan_unicode_pattern(utf8_nfd, wc_nfd, utf16be_nfd, utf16le_nfd,
                    nfd, 0);
 
-               if (locale_is_utf8) {
-#if defined(__APPLE__)
+               if (locale_is_utf8 && mac_nfd) {
                        /*
                         * Normalize an NFC string for import.
                         */
                        assertEqualInt(0, archive_strcpy_in_locale(
                            &utf8, utf8_nfc, f_sconv8));
-                       failure("NFC(%s) should be converted to NFD(%s):%d",
-                           nfc, nfd, line);
-                       assertEqualUTF8String(utf8_nfd, utf8.s);
+                       if (should_be_nfc) {
+                               failure("NFC(%s) should not be converted to"
+                                   " NFD(%s):%d",
+                                   nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfc, utf8.s);
+                       } else {
+                               failure("NFC(%s) should be converted to"
+                                   " NFD(%s):%d", nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfd, utf8.s);
+                       }
 
                        /*
                         * Normalize an NFD string for import.
@@ -383,19 +416,32 @@ test_archive_string_normalization(void)
                         */
                        assertEqualInt(0, archive_strncpy_in_locale(
                            &utf8, utf16be_nfc, 100000, f_sconv16be));
-                       failure("NFC(%s) should be converted to NFD(%s):%d",
-                           nfc, nfd, line);
-                       assertEqualUTF8String(utf8_nfd, utf8.s);
+                       if (should_be_nfc) {
+                               failure("NFC(%s) should not be converted to"
+                                   " NFD(%s):%d", nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfc, utf8.s);
+                       } else {
+                               failure("NFC(%s) should be converted to"
+                                   " NFD(%s):%d", nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfd, utf8.s);
+                       }
 
                        /*
                         * Normalize an NFC string in UTF-16LE for import.
                         */
                        assertEqualInt(0, archive_strncpy_in_locale(
                            &utf8, utf16le_nfc, 100000, f_sconv16le));
-                       failure("NFC(%s) should be converted to NFD(%s):%d",
-                           nfc, nfd, line);
-                       assertEqualUTF8String(utf8_nfd, utf8.s);
-#else
+                       if (should_be_nfc) {
+                               failure("NFC(%s) should not be converted to"
+                                   " NFD(%s):%d", nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfc, utf8.s);
+                       } else {
+                               failure("NFC(%s) should be converted to"
+                                   " NFD(%s):%d", nfc, nfd, line);
+                               assertEqualUTF8String(utf8_nfd, utf8.s);
+                       }
+               }
+               if (locale_is_utf8 && !mac_nfd) {
                        /*
                         * Normalize an NFD string for import.
                         */
@@ -440,7 +486,6 @@ test_archive_string_normalization(void)
                        failure("NFD(%s) should be converted to NFC(%s):%d",
                            nfd, nfc, line);
                        assertEqualUTF8String(utf8_nfc, utf8.s);
-#endif
                }
 
                /*
@@ -450,12 +495,7 @@ test_archive_string_normalization(void)
                 * current locale since windows platform cannot make
                 * locale UTF-8.
                 */
-               if (locale_is_utf8 || wc_is_unicode) {
-                       const wchar_t *wp;
-                       const char *mp;
-                       size_t mplen;
-
-#if defined(__APPLE__)
+               if ((locale_is_utf8 || wc_is_unicode) && mac_nfd) {
                        /*
                         * Normalize an NFD string in UTF-8 for import.
                         */
@@ -463,9 +503,15 @@ test_archive_string_normalization(void)
                            &mstr, utf8_nfc, 100000, f_sconv8));
                        assertEqualInt(0,
                            archive_mstring_get_wcs(a, &mstr, &wp));
-                       failure("UTF-8 NFC(%s) should be converted "
-                           "to WCS NFD(%s):%d", nfc, nfd, line);
-                       assertEqualWString(wc_nfd, wp);
+                       if (should_be_nfc) {
+                               failure("UTF-8 NFC(%s) should not be converted "
+                                   "to WCS NFD(%s):%d", nfc, nfd, line);
+                               assertEqualWString(wc_nfc, wp);
+                       } else {
+                               failure("UTF-8 NFC(%s) should be converted "
+                                   "to WCS NFD(%s):%d", nfc, nfd, line);
+                               assertEqualWString(wc_nfd, wp);
+                       }
 
                        /*
                         * Normalize an NFD string in UTF-16BE for import.
@@ -474,9 +520,16 @@ test_archive_string_normalization(void)
                            &mstr, utf16be_nfc, 100000, f_sconv16be));
                        assertEqualInt(0,
                            archive_mstring_get_wcs(a, &mstr, &wp));
-                       failure("UTF-16BE NFC(%s) should be converted "
-                           "to WCS NFD(%s):%d", nfc, nfd, line);
-                       assertEqualWString(wc_nfd, wp);
+                       if (should_be_nfc) {
+                               failure("UTF-16BE NFC(%s) should not be "
+                                   "converted to WCS NFD(%s):%d",
+                                   nfc, nfd, line);
+                               assertEqualWString(wc_nfc, wp);
+                       } else {
+                               failure("UTF-16BE NFC(%s) should be converted "
+                                   "to WCS NFD(%s):%d", nfc, nfd, line);
+                               assertEqualWString(wc_nfd, wp);
+                       }
 
                        /*
                         * Normalize an NFD string in UTF-16LE for import.
@@ -485,9 +538,16 @@ test_archive_string_normalization(void)
                            &mstr, utf16le_nfc, 100000, f_sconv16le));
                        assertEqualInt(0,
                            archive_mstring_get_wcs(a, &mstr, &wp));
-                       failure("UTF-16LE NFC(%s) should be converted "
-                           "to WCS NFD(%s):%d", nfc, nfd, line);
-                       assertEqualWString(wc_nfd, wp);
+                       if (should_be_nfc) {
+                               failure("UTF-16LE NFC(%s) should not be "
+                                   "converted to WCS NFD(%s):%d",
+                                   nfc, nfd, line);
+                               assertEqualWString(wc_nfc, wp);
+                       } else {
+                               failure("UTF-16LE NFC(%s) should be converted "
+                                   "to WCS NFD(%s):%d", nfc, nfd, line);
+                               assertEqualWString(wc_nfd, wp);
+                       }
 
                        /*
                         * Copy an NFD wide-string for export.
@@ -499,7 +559,8 @@ test_archive_string_normalization(void)
                        failure("WCS NFD(%s) should be UTF-8 NFD:%d"
                            ,nfd, line);
                        assertEqualUTF8String(utf8_nfd, mp);
-#else
+               }
+               if ((locale_is_utf8 || wc_is_unicode) && !mac_nfd) {
                        /*
                         * Normalize an NFD string in UTF-8 for import.
                         */
@@ -543,7 +604,6 @@ test_archive_string_normalization(void)
                        failure("WCS NFC(%s) should be UTF-8 NFC:%d"
                            ,nfc, line);
                        assertEqualUTF8String(utf8_nfc, mp);
-#endif
                }
        }
 
@@ -624,6 +684,7 @@ test_archive_string_canonicalization(void)
 
 DEFINE_TEST(test_archive_string_conversion)
 {
-       test_archive_string_normalization();
+       test_archive_string_normalization(0);
+       test_archive_string_normalization(1);
        test_archive_string_canonicalization();
 }