1 /* SPDX-License-Identifier: LGPL-2.1+ */
3 This file is part of systemd.
5 Copyright 2008-2011 Kay Sievers
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
22 /* Parts of this file are based on the GLIB utf8 validation functions. The
23 * original license text follows. */
25 /* gutf8.c - Operations on UTF-8 strings.
27 * Copyright (C) 1999 Tom Tromey
28 * Copyright (C) 2000 Red Hat, Inc.
30 * This library is free software; you can redistribute it and/or
31 * modify it under the terms of the GNU Library General Public
32 * License as published by the Free Software Foundation; either
33 * version 2 of the License, or (at your option) any later version.
35 * This library is distributed in the hope that it will be useful,
36 * but WITHOUT ANY WARRANTY; without even the implied warranty of
37 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
38 * Library General Public License for more details.
40 * You should have received a copy of the GNU Library General Public
41 * License along with this library; if not, write to the Free Software
42 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
50 #include "alloc-util.h"
51 #include "hexdecoct.h"
55 bool unichar_is_valid(char32_t ch
) {
57 if (ch
>= 0x110000) /* End of unicode space */
59 if ((ch
& 0xFFFFF800) == 0xD800) /* Reserved area for UTF-16 */
61 if ((ch
>= 0xFDD0) && (ch
<= 0xFDEF)) /* Reserved */
63 if ((ch
& 0xFFFE) == 0xFFFE) /* BOM (Byte Order Mark) */
69 static bool unichar_is_control(char32_t ch
) {
72 0 to ' '-1 is the C0 range.
73 DEL=0x7F, and DEL+1 to 0x9F is C1 range.
74 '\t' is in C0 range, but more or less harmless and commonly used.
77 return (ch
< ' ' && !IN_SET(ch
, '\t', '\n')) ||
78 (0x7F <= ch
&& ch
<= 0x9F);
81 /* count of characters used to encode one unicode char */
82 static int utf8_encoded_expected_len(const char *str
) {
87 c
= (unsigned char) str
[0];
90 if ((c
& 0xe0) == 0xc0)
92 if ((c
& 0xf0) == 0xe0)
94 if ((c
& 0xf8) == 0xf0)
96 if ((c
& 0xfc) == 0xf8)
98 if ((c
& 0xfe) == 0xfc)
104 /* decode one unicode char */
105 int utf8_encoded_to_unichar(const char *str
, char32_t
*ret_unichar
) {
111 len
= utf8_encoded_expected_len(str
);
115 *ret_unichar
= (char32_t
)str
[0];
118 unichar
= str
[0] & 0x1f;
121 unichar
= (char32_t
)str
[0] & 0x0f;
124 unichar
= (char32_t
)str
[0] & 0x07;
127 unichar
= (char32_t
)str
[0] & 0x03;
130 unichar
= (char32_t
)str
[0] & 0x01;
136 for (i
= 1; i
< len
; i
++) {
137 if (((char32_t
)str
[i
] & 0xc0) != 0x80)
140 unichar
|= (char32_t
)str
[i
] & 0x3f;
143 *ret_unichar
= unichar
;
148 bool utf8_is_printable_newline(const char* str
, size_t length
, bool newline
) {
153 for (p
= str
; length
;) {
157 encoded_len
= utf8_encoded_valid_unichar(p
);
158 if (encoded_len
< 0 ||
159 (size_t) encoded_len
> length
)
162 r
= utf8_encoded_to_unichar(p
, &val
);
164 unichar_is_control(val
) ||
165 (!newline
&& val
== '\n'))
168 length
-= encoded_len
;
175 const char *utf8_is_valid(const char *str
) {
180 for (p
= (const uint8_t*) str
; *p
; ) {
183 len
= utf8_encoded_valid_unichar((const char *)p
);
193 char *utf8_escape_invalid(const char *str
) {
198 p
= s
= malloc(strlen(str
) * 4 + 1);
205 len
= utf8_encoded_valid_unichar(str
);
207 s
= mempcpy(s
, str
, len
);
210 s
= stpcpy(s
, UTF8_REPLACEMENT_CHARACTER
);
220 char *utf8_escape_non_printable(const char *str
) {
225 p
= s
= malloc(strlen(str
) * 4 + 1);
232 len
= utf8_encoded_valid_unichar(str
);
234 if (utf8_is_printable(str
, len
)) {
235 s
= mempcpy(s
, str
, len
);
241 *(s
++) = hexchar((int) *str
>> 4);
242 *(s
++) = hexchar((int) *str
);
249 s
= stpcpy(s
, UTF8_REPLACEMENT_CHARACTER
);
259 char *ascii_is_valid(const char *str
) {
264 for (p
= str
; *p
; p
++)
265 if ((unsigned char) *p
>= 128)
272 * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
273 * @out_utf8: output buffer of at least 4 bytes or NULL
274 * @g: UCS-4 character to encode
276 * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
277 * The length of the character is returned. It is not zero-terminated! If the
278 * output buffer is NULL, only the length is returned.
280 * Returns: The length in bytes that the UTF-8 representation does or would
283 size_t utf8_encode_unichar(char *out_utf8
, char32_t g
) {
287 out_utf8
[0] = g
& 0x7f;
289 } else if (g
< (1 << 11)) {
291 out_utf8
[0] = 0xc0 | ((g
>> 6) & 0x1f);
292 out_utf8
[1] = 0x80 | (g
& 0x3f);
295 } else if (g
< (1 << 16)) {
297 out_utf8
[0] = 0xe0 | ((g
>> 12) & 0x0f);
298 out_utf8
[1] = 0x80 | ((g
>> 6) & 0x3f);
299 out_utf8
[2] = 0x80 | (g
& 0x3f);
302 } else if (g
< (1 << 21)) {
304 out_utf8
[0] = 0xf0 | ((g
>> 18) & 0x07);
305 out_utf8
[1] = 0x80 | ((g
>> 12) & 0x3f);
306 out_utf8
[2] = 0x80 | ((g
>> 6) & 0x3f);
307 out_utf8
[3] = 0x80 | (g
& 0x3f);
315 char *utf16_to_utf8(const void *s
, size_t length
) {
319 r
= new(char, (length
* 4 + 1) / 2 + 1);
326 while (f
< (const uint8_t*) s
+ length
) {
329 /* see RFC 2781 section 2.2 */
331 w1
= f
[1] << 8 | f
[0];
334 if (!utf16_is_surrogate(w1
)) {
335 t
+= utf8_encode_unichar(t
, w1
);
340 if (utf16_is_trailing_surrogate(w1
))
342 else if (f
>= (const uint8_t*) s
+ length
)
345 w2
= f
[1] << 8 | f
[0];
348 if (!utf16_is_trailing_surrogate(w2
)) {
353 t
+= utf8_encode_unichar(t
, utf16_surrogate_pair_to_unichar(w1
, w2
));
360 /* expected size used to encode one unicode char */
361 static int utf8_unichar_to_encoded_len(char32_t unichar
) {
367 if (unichar
< 0x10000)
369 if (unichar
< 0x200000)
371 if (unichar
< 0x4000000)
377 /* validate one encoded unicode char and return its length */
378 int utf8_encoded_valid_unichar(const char *str
) {
384 len
= utf8_encoded_expected_len(str
);
392 /* check if expected encoded chars are available */
393 for (i
= 0; i
< len
; i
++)
394 if ((str
[i
] & 0x80) != 0x80)
397 r
= utf8_encoded_to_unichar(str
, &unichar
);
401 /* check if encoded length matches encoded value */
402 if (utf8_unichar_to_encoded_len(unichar
) != len
)
405 /* check if value has valid range */
406 if (!unichar_is_valid(unichar
))