liblib: Fix Unicode decomposition

diff --git a/src/lib/test-unichar.c b/src/lib/test-unichar.c
index abb6357dfe9b297884bfb0d224a4959e30729183..b7bb2581dd7b1b9420822fc863543c207df8b2ad 100644 (file)
--- a/src/lib/test-unichar.c
+++ b/src/lib/test-unichar.c
@@ -2,11 +2,15 @@
 
 #include "test-lib.h"
 #include "str.h"
+#include "buffer.h"
 #include "unichar.h"
 
 void test_unichar(void)
 {
-       static const char *overlong_utf8 = "\xf8\x80\x95\x81\xa1";
+       static const char overlong_utf8[] = "\xf8\x80\x95\x81\xa1";
+       static const char collate_in[] = "\xc3\xbc \xc2\xb3";
+       static const char collate_exp[] = "U\xcc\x88 3";
+       buffer_t *collate_out;
        unichar_t chr, chr2;
        string_t *str = t_str_new(16);
 
@@ -18,6 +22,13 @@ void test_unichar(void)
                test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0);
                test_assert(chr2 == chr);
        }
+
+       collate_out = buffer_create_dynamic(default_pool, 32);
+       uni_utf8_to_decomposed_titlecase(collate_in, sizeof(collate_in),
+                                        collate_out);
+       test_assert(!strcmp(collate_out->data, collate_exp));
+       buffer_free(&collate_out);
+
        test_assert(!uni_utf8_str_is_valid(overlong_utf8));
        test_assert(uni_utf8_get_char(overlong_utf8, &chr2) < 0);
        test_end();

diff --git a/src/lib/unichar.c b/src/lib/unichar.c
index b0c50f5b60168212e8862e432418c06c5d507595..97b0aafcd77dcb71b087bd072cabee8ce633464a 100644 (file)
--- a/src/lib/unichar.c
+++ b/src/lib/unichar.c
@@ -287,7 +287,7 @@ static void uni_ucs4_decompose_hangul_utf8(unichar_t chr, buffer_t *output)
 
 static bool uni_ucs4_decompose_multi_utf8(unichar_t chr, buffer_t *output)
 {
-       const uint16_t *value;
+       const uint32_t *value;
        unsigned int idx;
 
        if (chr < multidecomp_keys[0] || chr > 0xffff)

diff --git a/src/lib/unicodemap.pl b/src/lib/unicodemap.pl
index c18c273df26316d4db6f1101830c2eae9cbadd95..2c1bf7ab0ce1b4a6e843139fcb273cc46b1225e9 100755 (executable)
--- a/src/lib/unicodemap.pl
+++ b/src/lib/unicodemap.pl
@@ -30,14 +30,14 @@ while (<>) {
       push @titlecase32_keys, $code;
       push @titlecase32_values, $value;
     }
-  } elsif ($decomp =~ /\<[^>]*> (.+)/) {
+  } elsif ($decomp =~ /(?:\<[^>]*> )?(.+)/) {
     # decompositions
     my $decomp_codes = $1;
     if ($decomp_codes =~ /^([0-9A-Z]*)$/i) {
       # unicharacter decomposition. use separate lists for this
       my $value = eval("0x$1");
-      if ($value > 0xffff) {
-       print STDERR "Error: We've assumed decomposition codes are max. 16bit\n";
+      if ($value > 0xffffffff) {
+       print STDERR "Error: We've assumed decomposition codes are max. 32bit\n";
        exit 1;
       }
       if ($code <= 0xff) {
@@ -61,8 +61,8 @@ while (<>) {
 
       foreach my $dcode (split(" ", $decomp_codes)) {
        my $value = eval("0x$dcode");
-       if ($value > 0xffff) {
-         print STDERR "Error: We've assumed decomposition codes are max. 16bit\n";
+       if ($value > 0xffffffff) {
+         print STDERR "Error: We've assumed decomposition codes are max. 32bit\n";
          exit 1;
        }
        push @multidecomp_values, $value;
@@ -78,7 +78,7 @@ sub print_list {
   my $last = $#list;
   my $n = 0;
   foreach my $key (@list) {
-    printf("0x%04x", $key);
+    printf("0x%05x", $key);
     last if ($n == $last);
     print ",";
     
@@ -137,7 +137,7 @@ print "static const uint16_t uni16_decomp_keys[] = {\n\t";
 print_list(\@uni16_decomp_keys);
 print "\n};\n";
 
-print "static const uint16_t uni16_decomp_values[] = {\n\t";
+print "static const uint32_t uni16_decomp_values[] = {\n\t";
 print_list(\@uni16_decomp_values);
 print "\n};\n";
 
@@ -145,7 +145,7 @@ print "static const uint32_t uni32_decomp_keys[] = {\n\t";
 print_list(\@uni32_decomp_keys);
 print "\n};\n";
 
-print "static const uint16_t uni32_decomp_values[] = {\n\t";
+print "static const uint32_t uni32_decomp_values[] = {\n\t";
 print_list(\@uni32_decomp_values);
 print "\n};\n";
 
@@ -157,6 +157,6 @@ print "static const uint16_t multidecomp_offsets[] = {\n\t";
 print_list(\@multidecomp_offsets);
 print "\n};\n";
 
-print "static const uint16_t multidecomp_values[] = {\n\t";
+print "static const uint32_t multidecomp_values[] = {\n\t";
 print_list(\@multidecomp_values);
 print "\n};\n";