src/basic/utf8.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2008-2011 Kay Sievers
   7   Copyright 2012 Lennart Poettering
   8
   9   systemd is free software; you can redistribute it and/or modify it
  10   under the terms of the GNU Lesser General Public License as published by
  11   the Free Software Foundation; either version 2.1 of the License, or
  12   (at your option) any later version.
  13
  14   systemd is distributed in the hope that it will be useful, but
  15   WITHOUT ANY WARRANTY; without even the implied warranty of
  16   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17   Lesser General Public License for more details.
  18
  19   You should have received a copy of the GNU Lesser General Public License
  20   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  21 ***/
  22
  23 /* Parts of this file are based on the GLIB utf8 validation functions. The
  24  * original license text follows. */
  25
  26 /* gutf8.c - Operations on UTF-8 strings.
  27  *
  28  * Copyright (C) 1999 Tom Tromey
  29  * Copyright (C) 2000 Red Hat, Inc.
  30  *
  31  * This library is free software; you can redistribute it and/or
  32  * modify it under the terms of the GNU Library General Public
  33  * License as published by the Free Software Foundation; either
  34  * version 2 of the License, or (at your option) any later version.
  35  *
  36  * This library is distributed in the hope that it will be useful,
  37  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  38  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  39  * Library General Public License for more details.
  40  *
  41  * You should have received a copy of the GNU Library General Public
  42  * License along with this library; if not, write to the Free Software
  43  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  44  */
  45
  46 #include <errno.h>
  47 #include <stdbool.h>
  48 #include <stdlib.h>
  49 #include <string.h>
  50
  51 #include "alloc-util.h"
  52 #include "hexdecoct.h"
  53 #include "macro.h"
  54 #include "utf8.h"
  55
  56 bool unichar_is_valid(char32_t ch) {
  57
  58         if (ch >= 0x110000) /* End of unicode space */
  59                 return false;
  60         if ((ch & 0xFFFFF800) == 0xD800) /* Reserved area for UTF-16 */
  61                 return false;
  62         if ((ch >= 0xFDD0) && (ch <= 0xFDEF)) /* Reserved */
  63                 return false;
  64         if ((ch & 0xFFFE) == 0xFFFE) /* BOM (Byte Order Mark) */
  65                 return false;
  66
  67         return true;
  68 }
  69
  70 static bool unichar_is_control(char32_t ch) {
  71
  72         /*
  73           0 to ' '-1 is the C0 range.
  74           DEL=0x7F, and DEL+1 to 0x9F is C1 range.
  75           '\t' is in C0 range, but more or less harmless and commonly used.
  76         */
  77
  78         return (ch < ' ' && ch != '\t' && ch != '\n') ||
  79                 (0x7F <= ch && ch <= 0x9F);
  80 }
  81
  82 /* count of characters used to encode one unicode char */
  83 static int utf8_encoded_expected_len(const char *str) {
  84         unsigned char c;
  85
  86         assert(str);
  87
  88         c = (unsigned char) str[0];
  89         if (c < 0x80)
  90                 return 1;
  91         if ((c & 0xe0) == 0xc0)
  92                 return 2;
  93         if ((c & 0xf0) == 0xe0)
  94                 return 3;
  95         if ((c & 0xf8) == 0xf0)
  96                 return 4;
  97         if ((c & 0xfc) == 0xf8)
  98                 return 5;
  99         if ((c & 0xfe) == 0xfc)
 100                 return 6;
 101
 102         return 0;
 103 }
 104
 105 /* decode one unicode char */
 106 int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar) {
 107         char32_t unichar;
 108         int len, i;
 109
 110         assert(str);
 111
 112         len = utf8_encoded_expected_len(str);
 113
 114         switch (len) {
 115         case 1:
 116                 *ret_unichar = (char32_t)str[0];
 117                 return 0;
 118         case 2:
 119                 unichar = str[0] & 0x1f;
 120                 break;
 121         case 3:
 122                 unichar = (char32_t)str[0] & 0x0f;
 123                 break;
 124         case 4:
 125                 unichar = (char32_t)str[0] & 0x07;
 126                 break;
 127         case 5:
 128                 unichar = (char32_t)str[0] & 0x03;
 129                 break;
 130         case 6:
 131                 unichar = (char32_t)str[0] & 0x01;
 132                 break;
 133         default:
 134                 return -EINVAL;
 135         }
 136
 137         for (i = 1; i < len; i++) {
 138                 if (((char32_t)str[i] & 0xc0) != 0x80)
 139                         return -EINVAL;
 140                 unichar <<= 6;
 141                 unichar |= (char32_t)str[i] & 0x3f;
 142         }
 143
 144         *ret_unichar = unichar;
 145
 146         return 0;
 147 }
 148
 149 bool utf8_is_printable_newline(const char* str, size_t length, bool newline) {
 150         const char *p;
 151
 152         assert(str);
 153
 154         for (p = str; length;) {
 155                 int encoded_len, r;
 156                 char32_t val;
 157
 158                 encoded_len = utf8_encoded_valid_unichar(p);
 159                 if (encoded_len < 0 ||
 160                     (size_t) encoded_len > length)
 161                         return false;
 162
 163                 r = utf8_encoded_to_unichar(p, &val);
 164                 if (r < 0 ||
 165                     unichar_is_control(val) ||
 166                     (!newline && val == '\n'))
 167                         return false;
 168
 169                 length -= encoded_len;
 170                 p += encoded_len;
 171         }
 172
 173         return true;
 174 }
 175
 176 const char *utf8_is_valid(const char *str) {
 177         const uint8_t *p;
 178
 179         assert(str);
 180
 181         for (p = (const uint8_t*) str; *p; ) {
 182                 int len;
 183
 184                 len = utf8_encoded_valid_unichar((const char *)p);
 185                 if (len < 0)
 186                         return NULL;
 187
 188                 p += len;
 189         }
 190
 191         return str;
 192 }
 193
 194 char *utf8_escape_invalid(const char *str) {
 195         char *p, *s;
 196
 197         assert(str);
 198
 199         p = s = malloc(strlen(str) * 4 + 1);
 200         if (!p)
 201                 return NULL;
 202
 203         while (*str) {
 204                 int len;
 205
 206                 len = utf8_encoded_valid_unichar(str);
 207                 if (len > 0) {
 208                         s = mempcpy(s, str, len);
 209                         str += len;
 210                 } else {
 211                         s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);
 212                         str += 1;
 213                 }
 214         }
 215
 216         *s = '\0';
 217
 218         return p;
 219 }
 220
 221 char *utf8_escape_non_printable(const char *str) {
 222         char *p, *s;
 223
 224         assert(str);
 225
 226         p = s = malloc(strlen(str) * 4 + 1);
 227         if (!p)
 228                 return NULL;
 229
 230         while (*str) {
 231                 int len;
 232
 233                 len = utf8_encoded_valid_unichar(str);
 234                 if (len > 0) {
 235                         if (utf8_is_printable(str, len)) {
 236                                 s = mempcpy(s, str, len);
 237                                 str += len;
 238                         } else {
 239                                 while (len > 0) {
 240                                         *(s++) = '\\';
 241                                         *(s++) = 'x';
 242                                         *(s++) = hexchar((int) *str >> 4);
 243                                         *(s++) = hexchar((int) *str);
 244
 245                                         str += 1;
 246                                         len --;
 247                                 }
 248                         }
 249                 } else {
 250                         s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);
 251                         str += 1;
 252                 }
 253         }
 254
 255         *s = '\0';
 256
 257         return p;
 258 }
 259
 260 char *ascii_is_valid(const char *str) {
 261         const char *p;
 262
 263         assert(str);
 264
 265         for (p = str; *p; p++)
 266                 if ((unsigned char) *p >= 128)
 267                         return NULL;
 268
 269         return (char*) str;
 270 }
 271
 272 /**
 273  * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
 274  * @out_utf8: output buffer of at least 4 bytes or NULL
 275  * @g: UCS-4 character to encode
 276  *
 277  * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
 278  * The length of the character is returned. It is not zero-terminated! If the
 279  * output buffer is NULL, only the length is returned.
 280  *
 281  * Returns: The length in bytes that the UTF-8 representation does or would
 282  *          occupy.
 283  */
 284 size_t utf8_encode_unichar(char *out_utf8, char32_t g) {
 285
 286         if (g < (1 << 7)) {
 287                 if (out_utf8)
 288                         out_utf8[0] = g & 0x7f;
 289                 return 1;
 290         } else if (g < (1 << 11)) {
 291                 if (out_utf8) {
 292                         out_utf8[0] = 0xc0 | ((g >> 6) & 0x1f);
 293                         out_utf8[1] = 0x80 | (g & 0x3f);
 294                 }
 295                 return 2;
 296         } else if (g < (1 << 16)) {
 297                 if (out_utf8) {
 298                         out_utf8[0] = 0xe0 | ((g >> 12) & 0x0f);
 299                         out_utf8[1] = 0x80 | ((g >> 6) & 0x3f);
 300                         out_utf8[2] = 0x80 | (g & 0x3f);
 301                 }
 302                 return 3;
 303         } else if (g < (1 << 21)) {
 304                 if (out_utf8) {
 305                         out_utf8[0] = 0xf0 | ((g >> 18) & 0x07);
 306                         out_utf8[1] = 0x80 | ((g >> 12) & 0x3f);
 307                         out_utf8[2] = 0x80 | ((g >> 6) & 0x3f);
 308                         out_utf8[3] = 0x80 | (g & 0x3f);
 309                 }
 310                 return 4;
 311         }
 312
 313         return 0;
 314 }
 315
 316 char *utf16_to_utf8(const void *s, size_t length) {
 317         const uint8_t *f;
 318         char *r, *t;
 319
 320         r = new(char, (length * 4 + 1) / 2 + 1);
 321         if (!r)
 322                 return NULL;
 323
 324         f = s;
 325         t = r;
 326
 327         while (f < (const uint8_t*) s + length) {
 328                 char16_t w1, w2;
 329
 330                 /* see RFC 2781 section 2.2 */
 331
 332                 w1 = f[1] << 8 | f[0];
 333                 f += 2;
 334
 335                 if (!utf16_is_surrogate(w1)) {
 336                         t += utf8_encode_unichar(t, w1);
 337
 338                         continue;
 339                 }
 340
 341                 if (utf16_is_trailing_surrogate(w1))
 342                         continue;
 343                 else if (f >= (const uint8_t*) s + length)
 344                         break;
 345
 346                 w2 = f[1] << 8 | f[0];
 347                 f += 2;
 348
 349                 if (!utf16_is_trailing_surrogate(w2)) {
 350                         f -= 2;
 351                         continue;
 352                 }
 353
 354                 t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));
 355         }
 356
 357         *t = 0;
 358         return r;
 359 }
 360
 361 /* expected size used to encode one unicode char */
 362 static int utf8_unichar_to_encoded_len(char32_t unichar) {
 363
 364         if (unichar < 0x80)
 365                 return 1;
 366         if (unichar < 0x800)
 367                 return 2;
 368         if (unichar < 0x10000)
 369                 return 3;
 370         if (unichar < 0x200000)
 371                 return 4;
 372         if (unichar < 0x4000000)
 373                 return 5;
 374
 375         return 6;
 376 }
 377
 378 /* validate one encoded unicode char and return its length */
 379 int utf8_encoded_valid_unichar(const char *str) {
 380         int len, i, r;
 381         char32_t unichar;
 382
 383         assert(str);
 384
 385         len = utf8_encoded_expected_len(str);
 386         if (len == 0)
 387                 return -EINVAL;
 388
 389         /* ascii is valid */
 390         if (len == 1)
 391                 return 1;
 392
 393         /* check if expected encoded chars are available */
 394         for (i = 0; i < len; i++)
 395                 if ((str[i] & 0x80) != 0x80)
 396                         return -EINVAL;
 397
 398         r = utf8_encoded_to_unichar(str, &unichar);
 399         if (r < 0)
 400                 return r;
 401
 402         /* check if encoded length matches encoded value */
 403         if (utf8_unichar_to_encoded_len(unichar) != len)
 404                 return -EINVAL;
 405
 406         /* check if value has valid range */
 407         if (!unichar_is_valid(unichar))
 408                 return -EINVAL;
 409
 410         return len;
 411 }