2 * Copyright (c) 2014 SGI.
3 * Copyright (c) 2018 Collabora Ltd.
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it would be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
18 * This code is adapted from the Linux Kernel. We have a
19 * userspace version here such that the hashes will match that
34 /* Encoding a unicode version number as a single unsigned int. */
35 #define UNICODE_MAJ_SHIFT (16)
36 #define UNICODE_MIN_SHIFT (8)
38 #define UNICODE_AGE(MAJ, MIN, REV) \
39 (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \
40 ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \
41 ((unsigned int)(REV)))
43 /* Needed in struct utf8cursor below. */
44 #define UTF8HANGULLEAF (12)
47 * Cursor structure used by the normalizer.
50 const struct utf8data
*data
;
59 unsigned char hangul
[UTF8HANGULLEAF
];
63 * Initialize a utf8cursor to normalize a string.
64 * Returns 0 on success.
65 * Returns -1 on failure.
67 // extern int utf8cursor(struct utf8cursor *u8c, const struct utf8data *data,
69 // extern int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data,
70 // const char *s, size_t len);
73 * Get the next byte in the normalization.
74 * Returns a value > 0 && < 256 on success.
75 * Returns 0 when the end of the normalization is reached.
76 * Returns -1 if the string being normalized is not valid UTF-8.
78 // extern int utf8byte(struct utf8cursor *u8c);
86 #define __INCLUDED_FROM_UTF8NORM_C__
88 #undef __INCLUDED_FROM_UTF8NORM_C__
90 #define ARRAY_SIZE(array) \
91 (sizeof(array) / sizeof(array[0]))
94 /* Highest unicode version supported by the data tables. */
95 static int utf8version_is_supported(uint8_t maj
, uint8_t min
, uint8_t rev
)
97 int i
= ARRAY_SIZE(utf8agetab
) - 1;
98 unsigned int sb_utf8version
= UNICODE_AGE(maj
, min
, rev
);
100 while (i
>= 0 && utf8agetab
[i
] != 0) {
101 if (sb_utf8version
== utf8agetab
[i
])
110 static int utf8version_latest(void)
117 * UTF-8 valid ranges.
119 * The UTF-8 encoding spreads the bits of a 32bit word over several
120 * bytes. This table gives the ranges that can be held and how they'd
123 * 0x00000000 0x0000007F: 0xxxxxxx
124 * 0x00000000 0x000007FF: 110xxxxx 10xxxxxx
125 * 0x00000000 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
126 * 0x00000000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
127 * 0x00000000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
128 * 0x00000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
130 * There is an additional requirement on UTF-8, in that only the
131 * shortest representation of a 32bit value is to be used. A decoder
132 * must not decode sequences that do not satisfy this requirement.
133 * Thus the allowed ranges have a lower bound.
135 * 0x00000000 0x0000007F: 0xxxxxxx
136 * 0x00000080 0x000007FF: 110xxxxx 10xxxxxx
137 * 0x00000800 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx
138 * 0x00010000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
139 * 0x00200000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
140 * 0x04000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
142 * Actual unicode characters are limited to the range 0x0 - 0x10FFFF,
143 * 17 planes of 65536 values. This limits the sequences actually seen
144 * even more, to just the following.
147 * 0x80 - 0x7FF: 0xC2 0x80 - 0xDF 0xBF
148 * 0x800 - 0xFFFF: 0xE0 0xA0 0x80 - 0xEF 0xBF 0xBF
149 * 0x10000 - 0x10FFFF: 0xF0 0x90 0x80 0x80 - 0xF4 0x8F 0xBF 0xBF
151 * Within those ranges the surrogates 0xD800 - 0xDFFF are not allowed.
153 * Note that the longest sequence seen with valid usage is 4 bytes,
154 * the same a single UTF-32 character. This makes the UTF-8
155 * representation of Unicode strictly smaller than UTF-32.
157 * The shortest sequence requirement was introduced by:
158 * Corrigendum #1: UTF-8 Shortest Form
159 * It can be found here:
160 * http://www.unicode.org/versions/corrigendum1.html
165 * Return the number of bytes used by the current UTF-8 sequence.
166 * Assumes the input points to the first byte of a valid UTF-8
169 static inline int utf8clen(const char *s
)
171 unsigned char c
= *s
;
173 return 1 + (c
>= 0xC0) + (c
>= 0xE0) + (c
>= 0xF0);
177 * Decode a 3-byte UTF-8 sequence.
180 utf8decode3(const char *str
)
194 * Encode a 3-byte UTF-8 sequence.
197 utf8encode3(char *str
, unsigned int val
)
199 str
[2] = (val
& 0x3F) | 0x80;
201 str
[1] = (val
& 0x3F) | 0x80;
211 * A compact binary tree, used to decode UTF-8 characters.
213 * Internal nodes are one byte for the node itself, and up to three
214 * bytes for an offset into the tree. The first byte contains the
215 * following information:
216 * NEXTBYTE - flag - advance to next byte if set
217 * BITNUM - 3 bit field - the bit number to tested
218 * OFFLEN - 2 bit field - number of bytes in the offset
219 * if offlen == 0 (non-branching node)
220 * RIGHTPATH - 1 bit field - set if the following node is for the
221 * right-hand path (tested bit is set)
222 * TRIENODE - 1 bit field - set if the following node is an internal
223 * node, otherwise it is a leaf node
224 * if offlen != 0 (branching node)
225 * LEFTNODE - 1 bit field - set if the left-hand node is internal
226 * RIGHTNODE - 1 bit field - set if the right-hand node is internal
228 * Due to the way utf8 works, there cannot be branching nodes with
229 * NEXTBYTE set, and moreover those nodes always have a righthand
232 typedef const unsigned char utf8trie_t
;
234 #define NEXTBYTE 0x08
236 #define OFFLEN_SHIFT 4
237 #define RIGHTPATH 0x40
238 #define TRIENODE 0x80
239 #define RIGHTNODE 0x40
240 #define LEFTNODE 0x80
245 * The leaves of the trie are embedded in the trie, and so the same
246 * underlying datatype: unsigned char.
248 * leaf[0]: The unicode version, stored as a generation number that is
249 * an index into utf8agetab[]. With this we can filter code
250 * points based on the unicode version in which they were
251 * defined. The CCC of a non-defined code point is 0.
252 * leaf[1]: Canonical Combining Class. During normalization, we need
253 * to do a stable sort into ascending order of all characters
254 * with a non-zero CCC that occur between two characters with
255 * a CCC of 0, or at the begin or end of a string.
256 * The unicode standard guarantees that all CCC values are
257 * between 0 and 254 inclusive, which leaves 255 available as
259 * Code points with CCC 0 are known as stoppers.
260 * leaf[2]: Decomposition. If leaf[1] == 255, then leaf[2] is the
261 * start of a NUL-terminated string that is the decomposition
263 * The CCC of a decomposable character is the same as the CCC
264 * of the first character of its decomposition.
265 * Some characters decompose as the empty string: these are
266 * characters with the Default_Ignorable_Code_Point property.
267 * These do affect normalization, as they all have CCC 0.
269 * The decompositions in the trie have been fully expanded, with the
270 * exception of Hangul syllables, which are decomposed algorithmically.
272 * Casefolding, if applicable, is also done using decompositions.
274 * The trie is constructed in such a way that leaves exist for all
275 * UTF-8 sequences that match the criteria from the "UTF-8 valid
276 * ranges" comment above, and only for those sequences. Therefore a
277 * lookup in the trie can be used to validate the UTF-8 input.
279 typedef const unsigned char utf8leaf_t
;
281 #define LEAF_GEN(LEAF) ((LEAF)[0])
282 #define LEAF_CCC(LEAF) ((LEAF)[1])
283 #define LEAF_STR(LEAF) ((const char *)((LEAF) + 2))
288 #define DECOMPOSE (255)
290 /* Marker for hangul syllable decomposition. */
291 #define HANGUL ((char)(255))
292 /* Size of the synthesized leaf used for Hangul syllable decomposition. */
293 #define UTF8HANGULLEAF (12)
296 * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0)
298 * AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;
299 * D7A3;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;
308 * NCount = 588 (VCount * TCount)
309 * SCount = 11172 (LCount * NCount)
314 * LV (Canonical/Full)
315 * LIndex = SIndex / NCount
316 * VIndex = (Sindex % NCount) / TCount
317 * LPart = LBase + LIndex
318 * VPart = VBase + VIndex
321 * LVIndex = (SIndex / TCount) * TCount
322 * TIndex = (Sindex % TCount)
323 * LVPart = SBase + LVIndex
324 * TPart = TBase + TIndex
327 * LIndex = SIndex / NCount
328 * VIndex = (Sindex % NCount) / TCount
329 * TIndex = (Sindex % TCount)
330 * LPart = LBase + LIndex
331 * VPart = VBase + VIndex
335 * TPart = TBase + TIndex
336 * d = <LPart, TPart, VPart>
351 /* Algorithmic decomposition of hangul syllable. */
353 utf8hangul(const char *str
, unsigned char *hangul
)
361 /* Calculate the SI, LI, VI, and TI values. */
362 si
= utf8decode3(str
) - SB
;
367 /* Fill in base of leaf. */
370 LEAF_CCC(h
) = DECOMPOSE
;
373 /* Add LPart, a 3-byte UTF-8 sequence. */
374 h
+= utf8encode3((char *)h
, li
+ LB
);
376 /* Add VPart, a 3-byte UTF-8 sequence. */
377 h
+= utf8encode3((char *)h
, vi
+ VB
);
379 /* Add TPart if required, also a 3-byte UTF-8 sequence. */
381 h
+= utf8encode3((char *)h
, ti
+ TB
);
383 /* Terminate string. */
390 * Use trie to scan s, touching at most len bytes.
391 * Returns the leaf if one exists, NULL otherwise.
393 * A non-NULL return guarantees that the UTF-8 sequence starting at s
394 * is well-formed and corresponds to a known unicode code point. The
395 * shorthand for this will be "is valid UTF-8 unicode".
397 static utf8leaf_t
*utf8nlookup(const struct utf8data
*data
,
398 unsigned char *hangul
, const char *s
, size_t len
)
411 trie
= utf8data
+ data
->offset
;
414 offlen
= (*trie
& OFFLEN
) >> OFFLEN_SHIFT
;
415 if (*trie
& NEXTBYTE
) {
420 mask
= 1 << (*trie
& BITNUM
);
424 /* Right node at offset of trie */
425 node
= (*trie
& RIGHTNODE
);
426 offset
= trie
[offlen
];
429 offset
|= trie
[offlen
];
432 } else if (*trie
& RIGHTPATH
) {
433 /* Right node after this node */
434 node
= (*trie
& TRIENODE
);
443 /* Left node after this node. */
444 node
= (*trie
& LEFTNODE
);
446 } else if (*trie
& RIGHTPATH
) {
450 /* Left node after this node */
451 node
= (*trie
& TRIENODE
);
457 * Hangul decomposition is done algorithmically. These are the
458 * codepoints >= 0xAC00 and <= 0xD7A3. Their UTF-8 encoding is
459 * always 3 bytes long, so s has been advanced twice, and the
460 * start of the sequence is at s-2.
462 if (LEAF_CCC(trie
) == DECOMPOSE
&& LEAF_STR(trie
)[0] == HANGUL
)
463 trie
= utf8hangul(s
- 2, hangul
);
468 * Use trie to scan s.
469 * Returns the leaf if one exists, NULL otherwise.
471 * Forwards to utf8nlookup().
473 static utf8leaf_t
*utf8lookup(const struct utf8data
*data
,
474 unsigned char *hangul
, const char *s
)
476 return utf8nlookup(data
, hangul
, s
, (size_t)-1);
481 * Maximum age of any character in s.
482 * Return -1 if s is not valid UTF-8 unicode.
483 * Return 0 if only non-assigned code points are used.
485 static int utf8agemax(const struct utf8data
*data
, const char *s
)
490 unsigned char hangul
[UTF8HANGULLEAF
];
496 leaf
= utf8lookup(data
, hangul
, s
);
500 leaf_age
= utf8agetab
[LEAF_GEN(leaf
)];
501 if (leaf_age
<= data
->maxage
&& leaf_age
> age
)
511 * Minimum age of any character in s.
512 * Return -1 if s is not valid UTF-8 unicode.
513 * Return 0 if non-assigned code points are used.
515 static int utf8agemin(const struct utf8data
*data
, const char *s
)
520 unsigned char hangul
[UTF8HANGULLEAF
];
526 leaf
= utf8lookup(data
, hangul
, s
);
529 leaf_age
= utf8agetab
[LEAF_GEN(leaf
)];
530 if (leaf_age
<= data
->maxage
&& leaf_age
< age
)
540 * Maximum age of any character in s, touch at most len bytes.
541 * Return -1 if s is not valid UTF-8 unicode.
543 static int utf8nagemax(const struct utf8data
*data
, const char *s
, size_t len
)
548 unsigned char hangul
[UTF8HANGULLEAF
];
554 leaf
= utf8nlookup(data
, hangul
, s
, len
);
557 leaf_age
= utf8agetab
[LEAF_GEN(leaf
)];
558 if (leaf_age
<= data
->maxage
&& leaf_age
> age
)
569 * Maximum age of any character in s, touch at most len bytes.
570 * Return -1 if s is not valid UTF-8 unicode.
572 static int utf8nagemin(const struct utf8data
*data
, const char *s
, size_t len
)
577 unsigned char hangul
[UTF8HANGULLEAF
];
583 leaf
= utf8nlookup(data
, hangul
, s
, len
);
586 leaf_age
= utf8agetab
[LEAF_GEN(leaf
)];
587 if (leaf_age
<= data
->maxage
&& leaf_age
< age
)
598 * Length of the normalization of s.
599 * Return -1 if s is not valid UTF-8 unicode.
601 * A string of Default_Ignorable_Code_Point has length 0.
603 static ssize_t
utf8len(const struct utf8data
*data
, const char *s
)
607 unsigned char hangul
[UTF8HANGULLEAF
];
612 leaf
= utf8lookup(data
, hangul
, s
);
615 if (utf8agetab
[LEAF_GEN(leaf
)] > data
->maxage
)
617 else if (LEAF_CCC(leaf
) == DECOMPOSE
)
618 ret
+= strlen(LEAF_STR(leaf
));
629 * Length of the normalization of s, touch at most len bytes.
630 * Return -1 if s is not valid UTF-8 unicode.
632 static ssize_t
utf8nlen(const struct utf8data
*data
, const char *s
, size_t len
)
636 unsigned char hangul
[UTF8HANGULLEAF
];
641 leaf
= utf8nlookup(data
, hangul
, s
, len
);
644 if (utf8agetab
[LEAF_GEN(leaf
)] > data
->maxage
)
646 else if (LEAF_CCC(leaf
) == DECOMPOSE
)
647 ret
+= strlen(LEAF_STR(leaf
));
658 * Set up an utf8cursor for use by utf8byte().
660 * u8c : pointer to cursor.
661 * data : const struct utf8data to use for normalization.
665 * Returns -1 on error, 0 on success.
667 static int utf8ncursor(struct utf8cursor
*u8c
, const struct utf8data
*data
,
668 const char *s
, size_t len
)
683 /* Check we didn't clobber the maximum length. */
686 /* The first byte of s may not be an utf8 continuation. */
687 if (len
> 0 && (*s
& 0xC0) == 0x80)
694 * Set up an utf8cursor for use by utf8byte().
696 * u8c : pointer to cursor.
697 * data : const struct utf8data to use for normalization.
698 * s : NUL-terminated string.
700 * Returns -1 on error, 0 on success.
702 static int utf8cursor(struct utf8cursor
*u8c
, const struct utf8data
*data
,
705 return utf8ncursor(u8c
, data
, s
, (unsigned int)-1);
710 * Get one byte from the normalized form of the string described by u8c.
712 * Returns the byte cast to an unsigned char on succes, and -1 on failure.
714 * The cursor keeps track of the location in the string in u8c->s.
715 * When a character is decomposed, the current location is stored in
716 * u8c->p, and u8c->s is set to the start of the decomposition. Note
717 * that bytes from a decomposition do not count against u8c->len.
719 * Characters are emitted if they match the current CCC in u8c->ccc.
720 * Hitting end-of-string while u8c->ccc == STOPPER means we're done,
721 * and the function returns 0 in that case.
723 * Sorting by CCC is done by repeatedly scanning the string. The
724 * values of u8c->s and u8c->p are stored in u8c->ss and u8c->sp at
725 * the start of the scan. The first pass finds the lowest CCC to be
726 * emitted and stores it in u8c->nccc, the second pass emits the
727 * characters with this CCC and finds the next lowest CCC. This limits
728 * the number of passes to 1 + the number of different CCCs in the
729 * sequence being scanned.
732 * u8c->p != NULL -> a decomposition is being scanned.
733 * u8c->ss != NULL -> this is a repeating scan.
734 * u8c->ccc == -1 -> this is the first scan of a repeating scan.
736 static int utf8byte(struct utf8cursor
*u8c
)
742 /* Check for the end of a decomposed character. */
743 if (u8c
->p
&& *u8c
->s
== '\0') {
748 /* Check for end-of-string. */
749 if (!u8c
->p
&& (u8c
->len
== 0 || *u8c
->s
== '\0')) {
750 /* There is no next byte. */
751 if (u8c
->ccc
== STOPPER
)
753 /* End-of-string during a scan counts as a stopper. */
756 } else if ((*u8c
->s
& 0xC0) == 0x80) {
757 /* This is a continuation of the current character. */
760 return (unsigned char)*u8c
->s
++;
763 /* Look up the data for the current character. */
765 leaf
= utf8lookup(u8c
->data
, u8c
->hangul
, u8c
->s
);
767 leaf
= utf8nlookup(u8c
->data
, u8c
->hangul
,
771 /* No leaf found implies that the input is a binary blob. */
775 ccc
= LEAF_CCC(leaf
);
776 /* Characters that are too new have CCC 0. */
777 if (utf8agetab
[LEAF_GEN(leaf
)] > u8c
->data
->maxage
) {
779 } else if (ccc
== DECOMPOSE
) {
780 u8c
->len
-= utf8clen(u8c
->s
);
781 u8c
->p
= u8c
->s
+ utf8clen(u8c
->s
);
782 u8c
->s
= LEAF_STR(leaf
);
783 /* Empty decomposition implies CCC 0. */
784 if (*u8c
->s
== '\0') {
785 if (u8c
->ccc
== STOPPER
)
791 leaf
= utf8lookup(u8c
->data
, u8c
->hangul
, u8c
->s
);
792 ccc
= LEAF_CCC(leaf
);
796 * If this is not a stopper, then see if it updates
797 * the next canonical class to be emitted.
799 if (ccc
!= STOPPER
&& u8c
->ccc
< ccc
&& ccc
< u8c
->nccc
)
803 * Return the current byte if this is the current
806 if (ccc
== u8c
->ccc
) {
809 return (unsigned char)*u8c
->s
++;
812 /* Current combining class mismatch. */
814 if (u8c
->nccc
== STOPPER
) {
816 * Scan forward for the first canonical class
817 * to be emitted. Save the position from
820 u8c
->ccc
= MINCCC
- 1;
824 u8c
->slen
= u8c
->len
;
826 u8c
->len
-= utf8clen(u8c
->s
);
827 u8c
->s
+= utf8clen(u8c
->s
);
828 } else if (ccc
!= STOPPER
) {
829 /* Not a stopper, and not the ccc we're emitting. */
831 u8c
->len
-= utf8clen(u8c
->s
);
832 u8c
->s
+= utf8clen(u8c
->s
);
833 } else if (u8c
->nccc
!= MAXCCC
+ 1) {
834 /* At a stopper, restart for next ccc. */
835 u8c
->ccc
= u8c
->nccc
;
836 u8c
->nccc
= MAXCCC
+ 1;
839 u8c
->len
= u8c
->slen
;
841 /* All done, proceed from here. */
853 * Look for the correct const struct utf8data for a unicode version.
854 * Returns NULL if the version requested is too new.
856 * Two normalization forms are supported: nfdi and nfdicf.
859 * - Apply unicode normalization form NFD.
860 * - Remove any Default_Ignorable_Code_Point.
863 * - Apply unicode normalization form NFD.
864 * - Remove any Default_Ignorable_Code_Point.
865 * - Apply a full casefold (C + F).
867 static const struct utf8data
*utf8nfdi(unsigned int maxage
)
869 int i
= ARRAY_SIZE(utf8nfdidata
) - 1;
871 while (maxage
< utf8nfdidata
[i
].maxage
)
873 if (maxage
> utf8nfdidata
[i
].maxage
)
875 return &utf8nfdidata
[i
];
879 static const struct utf8data
*utf8nfdicf(unsigned int maxage
)
881 int i
= ARRAY_SIZE(utf8nfdicfdata
) - 1;
883 while (maxage
< utf8nfdicfdata
[i
].maxage
)
885 if (maxage
> utf8nfdicfdata
[i
].maxage
)
887 return &utf8nfdicfdata
[i
];
890 static int utf8_casefold(const struct ext2fs_nls_table
*table
,
891 const unsigned char *str
, size_t len
,
892 unsigned char *dest
, size_t dlen
)
894 const struct utf8data
*data
= utf8nfdicf(table
->version
);
895 struct utf8cursor cur
;
898 if (utf8ncursor(&cur
, data
, (const char *) str
, len
) < 0)
901 for (nlen
= 0; nlen
< dlen
; nlen
++) {
902 int c
= utf8byte(&cur
);
911 return -ENAMETOOLONG
;
915 return -ENAMETOOLONG
;
917 /* Signal invalid sequence */
921 static const struct ext2fs_nls_ops utf8_ops
= {
922 .casefold
= utf8_casefold
,
925 static const struct ext2fs_nls_table nls_utf8
= {
927 .version
= UNICODE_AGE(12, 1, 0),
930 const struct ext2fs_nls_table
*ext2fs_load_nls_table(int encoding
)
932 if (encoding
== EXT4_ENC_UTF8_12_1
)