]>
Commit | Line | Data |
---|---|---|
b4c522fa IB |
1 | |
2 | /* Compiler implementation of the D programming language | |
3 | * Copyright (C) 2003-2018 by The D Language Foundation, All Rights Reserved | |
4 | * written by Walter Bright | |
5 | * http://www.digitalmars.com | |
6 | * Distributed under the Boost Software License, Version 1.0. | |
7 | * http://www.boost.org/LICENSE_1_0.txt | |
8 | * https://github.com/D-Programming-Language/dmd/blob/master/src/utf.h | |
9 | */ | |
10 | ||
11 | #pragma once | |
12 | ||
f9ab59ff | 13 | #include "root/dsystem.h" |
b4c522fa IB |
14 | |
15 | /// A UTF-8 code unit | |
16 | typedef unsigned char utf8_t; | |
17 | /// A UTF-16 code unit | |
18 | typedef unsigned short utf16_t; | |
19 | /// A UTF-32 code unit | |
20 | typedef unsigned int utf32_t; | |
21 | typedef utf32_t dchar_t; | |
22 | ||
23 | #define ALPHA_TABLE_LENGTH 245 | |
24 | static utf16_t const ALPHA_TABLE[ALPHA_TABLE_LENGTH][2] = | |
25 | { | |
26 | { 0x00AA, 0x00AA }, { 0x00B5, 0x00B5 }, { 0x00B7, 0x00B7 }, { 0x00BA, 0x00BA }, | |
27 | { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 }, { 0x00F8, 0x01F5 }, { 0x01FA, 0x0217 }, | |
28 | { 0x0250, 0x02A8 }, { 0x02B0, 0x02B8 }, { 0x02BB, 0x02BB }, { 0x02BD, 0x02C1 }, | |
29 | { 0x02D0, 0x02D1 }, { 0x02E0, 0x02E4 }, { 0x037A, 0x037A }, { 0x0386, 0x0386 }, | |
30 | { 0x0388, 0x038A }, { 0x038C, 0x038C }, { 0x038E, 0x03A1 }, { 0x03A3, 0x03CE }, | |
31 | { 0x03D0, 0x03D6 }, { 0x03DA, 0x03DA }, { 0x03DC, 0x03DC }, { 0x03DE, 0x03DE }, | |
32 | { 0x03E0, 0x03E0 }, { 0x03E2, 0x03F3 }, { 0x0401, 0x040C }, { 0x040E, 0x044F }, | |
33 | { 0x0451, 0x045C }, { 0x045E, 0x0481 }, { 0x0490, 0x04C4 }, { 0x04C7, 0x04C8 }, | |
34 | { 0x04CB, 0x04CC }, { 0x04D0, 0x04EB }, { 0x04EE, 0x04F5 }, { 0x04F8, 0x04F9 }, | |
35 | { 0x0531, 0x0556 }, { 0x0559, 0x0559 }, { 0x0561, 0x0587 }, { 0x05B0, 0x05B9 }, | |
36 | { 0x05BB, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, { 0x05D0, 0x05EA }, | |
37 | { 0x05F0, 0x05F2 }, { 0x0621, 0x063A }, { 0x0640, 0x0652 }, { 0x0660, 0x0669 }, | |
38 | { 0x0670, 0x06B7 }, { 0x06BA, 0x06BE }, { 0x06C0, 0x06CE }, { 0x06D0, 0x06DC }, | |
39 | { 0x06E5, 0x06E8 }, { 0x06EA, 0x06ED }, { 0x06F0, 0x06F9 }, { 0x0901, 0x0903 }, | |
40 | { 0x0905, 0x0939 }, { 0x093D, 0x094D }, { 0x0950, 0x0952 }, { 0x0958, 0x0963 }, | |
41 | { 0x0966, 0x096F }, { 0x0981, 0x0983 }, { 0x0985, 0x098C }, { 0x098F, 0x0990 }, | |
42 | { 0x0993, 0x09A8 }, { 0x09AA, 0x09B0 }, { 0x09B2, 0x09B2 }, { 0x09B6, 0x09B9 }, | |
43 | { 0x09BE, 0x09C4 }, { 0x09C7, 0x09C8 }, { 0x09CB, 0x09CD }, { 0x09DC, 0x09DD }, | |
44 | { 0x09DF, 0x09E3 }, { 0x09E6, 0x09F1 }, { 0x0A02, 0x0A02 }, { 0x0A05, 0x0A0A }, | |
45 | { 0x0A0F, 0x0A10 }, { 0x0A13, 0x0A28 }, { 0x0A2A, 0x0A30 }, { 0x0A32, 0x0A33 }, | |
46 | { 0x0A35, 0x0A36 }, { 0x0A38, 0x0A39 }, { 0x0A3E, 0x0A42 }, { 0x0A47, 0x0A48 }, | |
47 | { 0x0A4B, 0x0A4D }, { 0x0A59, 0x0A5C }, { 0x0A5E, 0x0A5E }, { 0x0A66, 0x0A6F }, | |
48 | { 0x0A74, 0x0A74 }, { 0x0A81, 0x0A83 }, { 0x0A85, 0x0A8B }, { 0x0A8D, 0x0A8D }, | |
49 | { 0x0A8F, 0x0A91 }, { 0x0A93, 0x0AA8 }, { 0x0AAA, 0x0AB0 }, { 0x0AB2, 0x0AB3 }, | |
50 | { 0x0AB5, 0x0AB9 }, { 0x0ABD, 0x0AC5 }, { 0x0AC7, 0x0AC9 }, { 0x0ACB, 0x0ACD }, | |
51 | { 0x0AD0, 0x0AD0 }, { 0x0AE0, 0x0AE0 }, { 0x0AE6, 0x0AEF }, { 0x0B01, 0x0B03 }, | |
52 | { 0x0B05, 0x0B0C }, { 0x0B0F, 0x0B10 }, { 0x0B13, 0x0B28 }, { 0x0B2A, 0x0B30 }, | |
53 | { 0x0B32, 0x0B33 }, { 0x0B36, 0x0B39 }, { 0x0B3D, 0x0B43 }, { 0x0B47, 0x0B48 }, | |
54 | { 0x0B4B, 0x0B4D }, { 0x0B5C, 0x0B5D }, { 0x0B5F, 0x0B61 }, { 0x0B66, 0x0B6F }, | |
55 | { 0x0B82, 0x0B83 }, { 0x0B85, 0x0B8A }, { 0x0B8E, 0x0B90 }, { 0x0B92, 0x0B95 }, | |
56 | { 0x0B99, 0x0B9A }, { 0x0B9C, 0x0B9C }, { 0x0B9E, 0x0B9F }, { 0x0BA3, 0x0BA4 }, | |
57 | { 0x0BA8, 0x0BAA }, { 0x0BAE, 0x0BB5 }, { 0x0BB7, 0x0BB9 }, { 0x0BBE, 0x0BC2 }, | |
58 | { 0x0BC6, 0x0BC8 }, { 0x0BCA, 0x0BCD }, { 0x0BE7, 0x0BEF }, { 0x0C01, 0x0C03 }, | |
59 | { 0x0C05, 0x0C0C }, { 0x0C0E, 0x0C10 }, { 0x0C12, 0x0C28 }, { 0x0C2A, 0x0C33 }, | |
60 | { 0x0C35, 0x0C39 }, { 0x0C3E, 0x0C44 }, { 0x0C46, 0x0C48 }, { 0x0C4A, 0x0C4D }, | |
61 | { 0x0C60, 0x0C61 }, { 0x0C66, 0x0C6F }, { 0x0C82, 0x0C83 }, { 0x0C85, 0x0C8C }, | |
62 | { 0x0C8E, 0x0C90 }, { 0x0C92, 0x0CA8 }, { 0x0CAA, 0x0CB3 }, { 0x0CB5, 0x0CB9 }, | |
63 | { 0x0CBE, 0x0CC4 }, { 0x0CC6, 0x0CC8 }, { 0x0CCA, 0x0CCD }, { 0x0CDE, 0x0CDE }, | |
64 | { 0x0CE0, 0x0CE1 }, { 0x0CE6, 0x0CEF }, { 0x0D02, 0x0D03 }, { 0x0D05, 0x0D0C }, | |
65 | { 0x0D0E, 0x0D10 }, { 0x0D12, 0x0D28 }, { 0x0D2A, 0x0D39 }, { 0x0D3E, 0x0D43 }, | |
66 | { 0x0D46, 0x0D48 }, { 0x0D4A, 0x0D4D }, { 0x0D60, 0x0D61 }, { 0x0D66, 0x0D6F }, | |
67 | { 0x0E01, 0x0E3A }, { 0x0E40, 0x0E5B }, { 0x0E81, 0x0E82 }, | |
68 | { 0x0E84, 0x0E84 }, { 0x0E87, 0x0E88 }, { 0x0E8A, 0x0E8A }, { 0x0E8D, 0x0E8D }, | |
69 | { 0x0E94, 0x0E97 }, { 0x0E99, 0x0E9F }, { 0x0EA1, 0x0EA3 }, { 0x0EA5, 0x0EA5 }, | |
70 | { 0x0EA7, 0x0EA7 }, { 0x0EAA, 0x0EAB }, { 0x0EAD, 0x0EAE }, { 0x0EB0, 0x0EB9 }, | |
71 | { 0x0EBB, 0x0EBD }, { 0x0EC0, 0x0EC4 }, { 0x0EC6, 0x0EC6 }, { 0x0EC8, 0x0ECD }, | |
72 | { 0x0ED0, 0x0ED9 }, { 0x0EDC, 0x0EDD }, { 0x0F00, 0x0F00 }, { 0x0F18, 0x0F19 }, | |
73 | { 0x0F20, 0x0F33 }, { 0x0F35, 0x0F35 }, { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, | |
74 | { 0x0F3E, 0x0F47 }, { 0x0F49, 0x0F69 }, { 0x0F71, 0x0F84 }, { 0x0F86, 0x0F8B }, | |
75 | { 0x0F90, 0x0F95 }, { 0x0F97, 0x0F97 }, { 0x0F99, 0x0FAD }, { 0x0FB1, 0x0FB7 }, | |
76 | { 0x0FB9, 0x0FB9 }, { 0x10A0, 0x10C5 }, { 0x10D0, 0x10F6 }, { 0x1E00, 0x1E9B }, | |
77 | { 0x1EA0, 0x1EF9 }, { 0x1F00, 0x1F15 }, { 0x1F18, 0x1F1D }, { 0x1F20, 0x1F45 }, | |
78 | { 0x1F48, 0x1F4D }, { 0x1F50, 0x1F57 }, { 0x1F59, 0x1F59 }, { 0x1F5B, 0x1F5B }, | |
79 | { 0x1F5D, 0x1F5D }, { 0x1F5F, 0x1F7D }, { 0x1F80, 0x1FB4 }, { 0x1FB6, 0x1FBC }, | |
80 | { 0x1FBE, 0x1FBE }, { 0x1FC2, 0x1FC4 }, { 0x1FC6, 0x1FCC }, { 0x1FD0, 0x1FD3 }, | |
81 | { 0x1FD6, 0x1FDB }, { 0x1FE0, 0x1FEC }, { 0x1FF2, 0x1FF4 }, { 0x1FF6, 0x1FFC }, | |
82 | { 0x203F, 0x2040 }, { 0x207F, 0x207F }, { 0x2102, 0x2102 }, { 0x2107, 0x2107 }, | |
83 | { 0x210A, 0x2113 }, { 0x2115, 0x2115 }, { 0x2118, 0x211D }, { 0x2124, 0x2124 }, | |
84 | { 0x2126, 0x2126 }, { 0x2128, 0x2128 }, { 0x212A, 0x2131 }, { 0x2133, 0x2138 }, | |
85 | { 0x2160, 0x2182 }, { 0x3005, 0x3007 }, { 0x3021, 0x3029 }, { 0x3041, 0x3093 }, | |
86 | { 0x309B, 0x309C }, { 0x30A1, 0x30F6 }, { 0x30FB, 0x30FC }, { 0x3105, 0x312C }, | |
87 | { 0x4E00, 0x9FA5 }, { 0xAC00, 0xD7A3 }, | |
88 | }; | |
89 | ||
90 | char const *const UTF8_DECODE_OK = NULL; | |
91 | extern char const UTF8_DECODE_OUTSIDE_CODE_SPACE[]; | |
92 | extern char const UTF8_DECODE_TRUNCATED_SEQUENCE[]; | |
93 | extern char const UTF8_DECODE_OVERLONG[]; | |
94 | extern char const UTF8_DECODE_INVALID_TRAILER[]; | |
95 | extern char const UTF8_DECODE_INVALID_CODE_POINT[]; | |
96 | ||
97 | char const *const UTF16_DECODE_OK = NULL; | |
98 | extern char const UTF16_DECODE_TRUNCATED_SEQUENCE[]; | |
99 | extern char const UTF16_DECODE_INVALID_SURROGATE[]; | |
100 | extern char const UTF16_DECODE_UNPAIRED_SURROGATE[]; | |
101 | extern char const UTF16_DECODE_INVALID_CODE_POINT[]; | |
102 | ||
103 | /// \return true if \a c is a valid, non-private UTF-32 code point | |
104 | bool utf_isValidDchar(dchar_t c); | |
105 | ||
106 | bool isUniAlpha(dchar_t c); | |
107 | ||
108 | int utf_codeLengthChar(dchar_t c); | |
109 | int utf_codeLengthWchar(dchar_t c); | |
110 | int utf_codeLength(int sz, dchar_t c); | |
111 | ||
112 | void utf_encodeChar(utf8_t *s, dchar_t c); | |
113 | void utf_encodeWchar(utf16_t *s, dchar_t c); | |
114 | void utf_encode(int sz, void *s, dchar_t c); | |
115 | ||
116 | const char *utf_decodeChar(utf8_t const *s, size_t len, size_t *pidx, dchar_t *presult); | |
117 | const char *utf_decodeWchar(utf16_t const *s, size_t len, size_t *pidx, dchar_t *presult); |