return N;
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf32_in_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf32_in_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char in[] = "bш\uAAAA\U0010AAAA";
- const char32_t exp_literal[] = U"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- std::copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 11, "");
- static_assert (array_size (exp_literal) == 5, "");
- static_assert (array_size (exp) == 5, "");
- VERIFY (char_traits<char>::length (in) == 10);
- VERIFY (char_traits<char32_t>::length (exp_literal) == 4);
- VERIFY (char_traits<CharT>::length (exp) == 4);
+ const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 5, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 4);
test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}, {10, 4}};
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
for (auto t : offsets)
{
- CharT out[array_size (exp)] = {};
+ InternT out[array_size (exp)] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf32_in_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf32_in_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char in[] = "bш\uAAAA\U0010AAAA";
- const char32_t exp_literal[] = U"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- std::copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 11, "");
- static_assert (array_size (exp_literal) == 5, "");
- static_assert (array_size (exp) == 5, "");
- VERIFY (char_traits<char>::length (in) == 10);
- VERIFY (char_traits<char32_t>::length (exp_literal) == 4);
- VERIFY (char_traits<CharT>::length (exp) == 4);
+ const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 5, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 4);
test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf32_in_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf32_in_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char valid_in[] = "bш\uAAAA\U0010AAAA";
- const char32_t exp_literal[] = U"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- std::copy (begin (exp_literal), end (exp_literal), begin (exp));
+ // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP, 4-byte CP
+ const unsigned char input[] = "b\u0448\uD700\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uD700\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 5, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 4);
+
+ // There are 5 classes of errors in UTF-8 decoding
+ // 1. Missing leading byte
+ // 2. Missing trailing byte
+ // 3. Surrogate CP
+ // 4. Overlong sequence
+ // 5. CP out of Unicode range
+ test_offsets_error<unsigned char> offsets[] = {
+
+ // 1. Missing leading byte. We will replace the leading byte with
+ // non-leading byte, such as a byte that is always invalid or a trailing
+ // byte.
- static_assert (array_size (valid_in) == 11, "");
- static_assert (array_size (exp_literal) == 5, "");
- static_assert (array_size (exp) == 5, "");
- VERIFY (char_traits<char>::length (valid_in) == 10);
- VERIFY (char_traits<char32_t>::length (exp_literal) == 4);
- VERIFY (char_traits<CharT>::length (exp) == 4);
+ // replace leading byte with invalid byte
+ {1, 4, 0, 0, 0xFF, 0},
+ {3, 4, 1, 1, 0xFF, 1},
+ {6, 4, 3, 2, 0xFF, 3},
+ {10, 4, 6, 3, 0xFF, 6},
- test_offsets_error<char> offsets[] = {
+ // replace leading byte with trailing byte
+ {1, 4, 0, 0, 0b10101010, 0},
+ {3, 4, 1, 1, 0b10101010, 1},
+ {6, 4, 3, 2, 0b10101010, 3},
+ {10, 4, 6, 3, 0b10101010, 6},
- // replace leading byte with invalid byte
- {1, 4, 0, 0, '\xFF', 0},
- {3, 4, 1, 1, '\xFF', 1},
- {6, 4, 3, 2, '\xFF', 3},
- {10, 4, 6, 3, '\xFF', 6},
+ // 2. Missing trailing byte. We will replace the trailing byte with
+ // non-trailing byte, such as a byte that is always invalid or a leading
+ // byte (simple ASCII byte in our case).
// replace first trailing byte with ASCII byte
{3, 4, 1, 1, 'z', 2},
{10, 4, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte
- {3, 4, 1, 1, '\xFF', 2},
- {6, 4, 3, 2, '\xFF', 4},
- {10, 4, 6, 3, '\xFF', 7},
+ {3, 4, 1, 1, 0xFF, 2},
+ {6, 4, 3, 2, 0xFF, 4},
+ {10, 4, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte
{6, 4, 3, 2, 'z', 5},
{10, 4, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte
- {6, 4, 3, 2, '\xFF', 5},
- {10, 4, 6, 3, '\xFF', 8},
+ {6, 4, 3, 2, 0xFF, 5},
+ {10, 4, 6, 3, 0xFF, 8},
// replace third trailing byte
{10, 4, 6, 3, 'z', 9},
- {10, 4, 6, 3, '\xFF', 9},
+ {10, 4, 6, 3, 0xFF, 9},
+
+ // 2.1 The following test-cases raise doubt whether error or partial should
+ // be returned. For example, we have 4-byte sequence with valid leading
+ // byte. If we hide the last byte we need to return partial. But, if the
+ // second or third byte, which are visible to the call to codecvt, are
+ // malformed then error should be returned.
// replace first trailing byte with ASCII byte, also incomplete at end
{5, 4, 3, 2, 'z', 4},
{9, 4, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte, also incomplete at end
- {5, 4, 3, 2, '\xFF', 4},
- {8, 4, 6, 3, '\xFF', 7},
- {9, 4, 6, 3, '\xFF', 7},
+ {5, 4, 3, 2, 0xFF, 4},
+ {8, 4, 6, 3, 0xFF, 7},
+ {9, 4, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte, also incomplete at end
{9, 4, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte, also incomplete at end
- {9, 4, 6, 3, '\xFF', 8},
+ {9, 4, 6, 3, 0xFF, 8},
+
+ // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte
+ // CP U+D700
+ {6, 4, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D800
+ {6, 4, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB00
+ {6, 4, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC00
+ {6, 4, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF00
+
+ // 4. Overlong sequence. The CPs in the input are chosen such as modifying
+ // just the leading byte is enough to make them overlong, i.e. for the
+ // 3-byte and 4-byte CP the second byte (first trailing) has enough leading
+ // zeroes.
+ {3, 4, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong
+ {3, 4, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong
+ {6, 4, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong
+ {10, 4, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong
+
+ // 5. CP above range
+ // turn U+10AAAA into U+14AAAA by changing its leading byte
+ {10, 4, 6, 3, 0b11110101, 6},
+ // turn U+10AAAA into U+11AAAA by changing its 2nd byte
+ {10, 4, 6, 3, 0b10011010, 7},
};
for (auto t : offsets)
{
- char in[array_size (valid_in)] = {};
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- char_traits<char>::copy (in, valid_in, array_size (valid_in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf32_in (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf32_in (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_utf32_in_ok (cvt);
utf8_to_utf32_in_partial (cvt);
utf8_to_utf32_in_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf32_to_utf8_out_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf32_to_utf8_out_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char32_t in_literal[] = U"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
- CharT in[array_size (in_literal)] = {};
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 5, "");
- static_assert (array_size (in) == 5, "");
- static_assert (array_size (exp) == 11, "");
- VERIFY (char_traits<char32_t>::length (in_literal) == 4);
- VERIFY (char_traits<CharT>::length (in) == 4);
- VERIFY (char_traits<char>::length (exp) == 10);
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 4);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
const test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}, {4, 10}};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<char>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf32_to_utf8_out_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf32_to_utf8_out_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char32_t in_literal[] = U"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
- CharT in[array_size (in_literal)] = {};
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 5, "");
- static_assert (array_size (in) == 5, "");
- static_assert (array_size (exp) == 11, "");
- VERIFY (char_traits<char32_t>::length (in_literal) == 4);
- VERIFY (char_traits<CharT>::length (in) == 4);
- VERIFY (char_traits<char>::length (exp) == 10);
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 4);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
const test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf32_to_utf8_out_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf32_to_utf8_out_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- const char32_t valid_in[] = U"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
-
- static_assert (array_size (valid_in) == 5, "");
- static_assert (array_size (exp) == 11, "");
- VERIFY (char_traits<char32_t>::length (valid_in) == 4);
- VERIFY (char_traits<char>::length (exp) == 10);
-
- test_offsets_error<CharT> offsets[] = {{4, 10, 0, 0, 0x00110000, 0},
- {4, 10, 1, 1, 0x00110000, 1},
- {4, 10, 2, 3, 0x00110000, 2},
- {4, 10, 3, 6, 0x00110000, 3}};
+ // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 4);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
+
+ test_offsets_error<InternT> offsets[] = {
+
+ // Surrogate CP
+ {4, 10, 0, 0, 0xD800, 0},
+ {4, 10, 1, 1, 0xDBFF, 1},
+ {4, 10, 2, 3, 0xDC00, 2},
+ {4, 10, 3, 6, 0xDFFF, 3},
+
+ // CP out of range
+ {4, 10, 0, 0, 0x00110000, 0},
+ {4, 10, 1, 1, 0x00110000, 1},
+ {4, 10, 2, 3, 0x00110000, 2},
+ {4, 10, 3, 6, 0x00110000, 3}};
for (auto t : offsets)
{
- CharT in[array_size (valid_in)] = {};
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- copy (begin (valid_in), end (valid_in), begin (in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf32_to_utf8_out (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf32_to_utf8_out (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf32_to_utf8_out_ok (cvt);
utf32_to_utf8_out_partial (cvt);
utf32_to_utf8_out_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-test_utf8_utf32_codecvts (const std::codecvt<CharT, char, mbstate_t> &cvt)
+test_utf8_utf32_cvt (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_utf32_in (cvt);
utf32_to_utf8_out (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf16_in_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf16_in_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char in[] = "bш\uAAAA\U0010AAAA";
- const char16_t exp_literal[] = u"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 11, "");
- static_assert (array_size (exp_literal) == 6, "");
- static_assert (array_size (exp) == 6, "");
- VERIFY (char_traits<char>::length (in) == 10);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 5);
- VERIFY (char_traits<CharT>::length (exp) == 5);
+ const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 6, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 5);
test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}, {10, 5}};
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
for (auto t : offsets)
{
- CharT out[array_size (exp)] = {};
+ InternT out[array_size (exp)] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf16_in_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf16_in_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char in[] = "bш\uAAAA\U0010AAAA";
- const char16_t exp_literal[] = u"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 11, "");
- static_assert (array_size (exp_literal) == 6, "");
- static_assert (array_size (exp) == 6, "");
- VERIFY (char_traits<char>::length (in) == 10);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 5);
- VERIFY (char_traits<CharT>::length (exp) == 5);
+ const unsigned char input[] = "b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 6, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 5);
test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf16_in_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf16_in_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- const char valid_in[] = "bш\uAAAA\U0010AAAA";
- const char16_t exp_literal[] = u"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
+ // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP, 4-byte CP
+ const unsigned char input[] = "b\u0448\uD700\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uD700\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 6, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 5);
+
+ // There are 5 classes of errors in UTF-8 decoding
+ // 1. Missing leading byte
+ // 2. Missing trailing byte
+ // 3. Surrogate CP
+ // 4. Overlong sequence
+ // 5. CP out of Unicode range
+ test_offsets_error<unsigned char> offsets[] = {
+
+ // 1. Missing leading byte. We will replace the leading byte with
+ // non-leading byte, such as a byte that is always invalid or a trailing
+ // byte.
- static_assert (array_size (valid_in) == 11, "");
- static_assert (array_size (exp_literal) == 6, "");
- static_assert (array_size (exp) == 6, "");
- VERIFY (char_traits<char>::length (valid_in) == 10);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 5);
- VERIFY (char_traits<CharT>::length (exp) == 5);
+ // replace leading byte with invalid byte
+ {1, 5, 0, 0, 0xFF, 0},
+ {3, 5, 1, 1, 0xFF, 1},
+ {6, 5, 3, 2, 0xFF, 3},
+ {10, 5, 6, 3, 0xFF, 6},
- test_offsets_error<char> offsets[] = {
+ // replace leading byte with trailing byte
+ {1, 5, 0, 0, 0b10101010, 0},
+ {3, 5, 1, 1, 0b10101010, 1},
+ {6, 5, 3, 2, 0b10101010, 3},
+ {10, 5, 6, 3, 0b10101010, 6},
- // replace leading byte with invalid byte
- {1, 5, 0, 0, '\xFF', 0},
- {3, 5, 1, 1, '\xFF', 1},
- {6, 5, 3, 2, '\xFF', 3},
- {10, 5, 6, 3, '\xFF', 6},
+ // 2. Missing trailing byte. We will replace the trailing byte with
+ // non-trailing byte, such as a byte that is always invalid or a leading
+ // byte (simple ASCII byte in our case).
// replace first trailing byte with ASCII byte
{3, 5, 1, 1, 'z', 2},
{10, 5, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte
- {3, 5, 1, 1, '\xFF', 2},
- {6, 5, 3, 2, '\xFF', 4},
- {10, 5, 6, 3, '\xFF', 7},
+ {3, 5, 1, 1, 0xFF, 2},
+ {6, 5, 3, 2, 0xFF, 4},
+ {10, 5, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte
{6, 5, 3, 2, 'z', 5},
{10, 5, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte
- {6, 5, 3, 2, '\xFF', 5},
- {10, 5, 6, 3, '\xFF', 8},
+ {6, 5, 3, 2, 0xFF, 5},
+ {10, 5, 6, 3, 0xFF, 8},
// replace third trailing byte
{10, 5, 6, 3, 'z', 9},
- {10, 5, 6, 3, '\xFF', 9},
+ {10, 5, 6, 3, 0xFF, 9},
+
+ // 2.1 The following test-cases raise doubt whether error or partial should
+ // be returned. For example, we have 4-byte sequence with valid leading
+ // byte. If we hide the last byte we need to return partial. But, if the
+ // second or third byte, which are visible to the call to codecvt, are
+ // malformed then error should be returned.
// replace first trailing byte with ASCII byte, also incomplete at end
{5, 5, 3, 2, 'z', 4},
{9, 5, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte, also incomplete at end
- {5, 5, 3, 2, '\xFF', 4},
- {8, 5, 6, 3, '\xFF', 7},
- {9, 5, 6, 3, '\xFF', 7},
+ {5, 5, 3, 2, 0xFF, 4},
+ {8, 5, 6, 3, 0xFF, 7},
+ {9, 5, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte, also incomplete at end
{9, 5, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte, also incomplete at end
- {9, 5, 6, 3, '\xFF', 8},
+ {9, 5, 6, 3, 0xFF, 8},
+
+ // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte
+ // CP U+D700
+ {6, 5, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D800
+ {6, 5, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB00
+ {6, 5, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC00
+ {6, 5, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF00
+
+ // 4. Overlong sequence. The CPs in the input are chosen such as modifying
+ // just the leading byte is enough to make them overlong, i.e. for the
+ // 3-byte and 4-byte CP the second byte (first trailing) has enough leading
+ // zeroes.
+ {3, 5, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong
+ {3, 5, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong
+ {6, 5, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong
+ {10, 5, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong
+
+ // 5. CP above range
+ // turn U+10AAAA into U+14AAAA by changing its leading byte
+ {10, 5, 6, 3, 0b11110101, 6},
+ // turn U+10AAAA into U+11AAAA by changing its 2nd byte
+ {10, 5, 6, 3, 0b10011010, 7},
};
for (auto t : offsets)
{
- char in[array_size (valid_in)] = {};
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- char_traits<char>::copy (in, valid_in, array_size (valid_in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_utf16_in (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_utf16_in (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_utf16_in_ok (cvt);
utf8_to_utf16_in_partial (cvt);
utf8_to_utf16_in_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf16_to_utf8_out_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf16_to_utf8_out_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char16_t in_literal[] = u"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
- CharT in[array_size (in_literal)];
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 6, "");
- static_assert (array_size (exp) == 11, "");
- static_assert (array_size (in) == 6, "");
- VERIFY (char_traits<char16_t>::length (in_literal) == 5);
- VERIFY (char_traits<char>::length (exp) == 10);
- VERIFY (char_traits<CharT>::length (in) == 5);
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 5);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
const test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}, {5, 10}};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<char>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf16_to_utf8_out_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf16_to_utf8_out_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
- const char16_t in_literal[] = u"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
- CharT in[array_size (in_literal)];
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 6, "");
- static_assert (array_size (exp) == 11, "");
- static_assert (array_size (in) == 6, "");
- VERIFY (char_traits<char16_t>::length (in_literal) == 5);
- VERIFY (char_traits<char>::length (exp) == 10);
- VERIFY (char_traits<CharT>::length (in) == 5);
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 5);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
const test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf16_to_utf8_out_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf16_to_utf8_out_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- const char16_t valid_in[] = u"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
-
- static_assert (array_size (valid_in) == 6, "");
- static_assert (array_size (exp) == 11, "");
- VERIFY (char_traits<char16_t>::length (valid_in) == 5);
- VERIFY (char_traits<char>::length (exp) == 10);
-
- test_offsets_error<CharT> offsets[] = {
+ // UTF-8 string of 1-byte CP, 2-byte CP, 3-byte CP and 4-byte CP
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 5);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
+
+ // The only possible error in UTF-16 is unpaired surrogate code units.
+ // So we replace valid code points (scalar values) with lone surrogate CU.
+ test_offsets_error<InternT> offsets[] = {
{5, 10, 0, 0, 0xD800, 0},
{5, 10, 0, 0, 0xDBFF, 0},
{5, 10, 0, 0, 0xDC00, 0},
for (auto t : offsets)
{
- CharT in[array_size (valid_in)] = {};
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- copy (begin (valid_in), end (valid_in), begin (in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf16_to_utf8_out (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf16_to_utf8_out (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf16_to_utf8_out_ok (cvt);
utf16_to_utf8_out_partial (cvt);
utf16_to_utf8_out_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-test_utf8_utf16_cvts (const std::codecvt<CharT, char, mbstate_t> &cvt)
+test_utf8_utf16_cvt (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_utf16_in (cvt);
utf16_to_utf8_out (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_ucs2_in_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_ucs2_in_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP
- const char in[] = "bш\uAAAA";
- const char16_t exp_literal[] = u"bш\uAAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 7, "");
- static_assert (array_size (exp_literal) == 4, "");
- static_assert (array_size (exp) == 4, "");
- VERIFY (char_traits<char>::length (in) == 6);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 3);
- VERIFY (char_traits<CharT>::length (exp) == 3);
+ const unsigned char input[] = "b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 7, "");
+ static_assert (array_size (expected) == 4, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 6);
+ VERIFY (char_traits<InternT>::length (exp) == 3);
test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {3, 2}, {6, 3}};
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
for (auto t : offsets)
{
- CharT out[array_size (exp)] = {};
+ InternT out[array_size (exp)] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<CharT>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_ucs2_in_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_ucs2_in_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP
- const char in[] = "bш\uAAAA";
- const char16_t exp_literal[] = u"bш\uAAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
-
- static_assert (array_size (in) == 7, "");
- static_assert (array_size (exp_literal) == 4, "");
- static_assert (array_size (exp) == 4, "");
- VERIFY (char_traits<char>::length (in) == 6);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 3);
- VERIFY (char_traits<CharT>::length (exp) == 3);
+ const unsigned char input[] = "b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 7, "");
+ static_assert (array_size (expected) == 4, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 6);
+ VERIFY (char_traits<InternT>::length (exp) == 3);
test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
for (auto t : offsets)
{
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_ucs2_in_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_ucs2_in_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- const char valid_in[] = "bш\uAAAA\U0010AAAA";
- const char16_t exp_literal[] = u"bш\uAAAA\U0010AAAA";
- CharT exp[array_size (exp_literal)] = {};
- copy (begin (exp_literal), end (exp_literal), begin (exp));
+ const unsigned char input[] = "b\u0448\uD700\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uD700\U0010AAAA";
+ static_assert (array_size (input) == 11, "");
+ static_assert (array_size (expected) == 6, "");
+
+ ExternT in[array_size (input)];
+ InternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<ExternT>::length (in) == 10);
+ VERIFY (char_traits<InternT>::length (exp) == 5);
+
+ // There are 5 classes of errors in UTF-8 decoding
+ // 1. Missing leading byte
+ // 2. Missing trailing byte
+ // 3. Surrogate CP
+ // 4. Overlong sequence
+ // 5. CP out of Unicode range
+ test_offsets_error<unsigned char> offsets[] = {
+
+ // 1. Missing leading byte. We will replace the leading byte with
+ // non-leading byte, such as a byte that is always invalid or a trailing
+ // byte.
- static_assert (array_size (valid_in) == 11, "");
- static_assert (array_size (exp_literal) == 6, "");
- static_assert (array_size (exp) == 6, "");
- VERIFY (char_traits<char>::length (valid_in) == 10);
- VERIFY (char_traits<char16_t>::length (exp_literal) == 5);
- VERIFY (char_traits<CharT>::length (exp) == 5);
+ // replace leading byte with invalid byte
+ {1, 5, 0, 0, 0xFF, 0},
+ {3, 5, 1, 1, 0xFF, 1},
+ {6, 5, 3, 2, 0xFF, 3},
+ {10, 5, 6, 3, 0xFF, 6},
- test_offsets_error<char> offsets[] = {
+ // replace leading byte with trailing byte
+ {1, 5, 0, 0, 0b10101010, 0},
+ {3, 5, 1, 1, 0b10101010, 1},
+ {6, 5, 3, 2, 0b10101010, 3},
+ {10, 5, 6, 3, 0b10101010, 6},
- // replace leading byte with invalid byte
- {1, 5, 0, 0, '\xFF', 0},
- {3, 5, 1, 1, '\xFF', 1},
- {6, 5, 3, 2, '\xFF', 3},
- {10, 5, 6, 3, '\xFF', 6},
+ // 2. Missing trailing byte. We will replace the trailing byte with
+ // non-trailing byte, such as a byte that is always invalid or a leading
+ // byte (simple ASCII byte in our case).
// replace first trailing byte with ASCII byte
{3, 5, 1, 1, 'z', 2},
{10, 5, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte
- {3, 5, 1, 1, '\xFF', 2},
- {6, 5, 3, 2, '\xFF', 4},
- {10, 5, 6, 3, '\xFF', 7},
+ {3, 5, 1, 1, 0xFF, 2},
+ {6, 5, 3, 2, 0xFF, 4},
+ {10, 5, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte
{6, 5, 3, 2, 'z', 5},
{10, 5, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte
- {6, 5, 3, 2, '\xFF', 5},
- {10, 5, 6, 3, '\xFF', 8},
+ {6, 5, 3, 2, 0xFF, 5},
+ {10, 5, 6, 3, 0xFF, 8},
// replace third trailing byte
{10, 5, 6, 3, 'z', 9},
- {10, 5, 6, 3, '\xFF', 9},
-
- // When we see a leading byte of 4-byte CP, we should return error, no
- // matter if it is incomplete at the end or has errors in the trailing
- // bytes.
-
- // Don't replace anything, show full 4-byte CP
- {10, 4, 6, 3, 'b', 0},
- {10, 5, 6, 3, 'b', 0},
+ {10, 5, 6, 3, 0xFF, 9},
- // Don't replace anything, show incomplete 4-byte CP at the end
- {7, 4, 6, 3, 'b', 0}, // incomplete fourth CP
- {8, 4, 6, 3, 'b', 0}, // incomplete fourth CP
- {9, 4, 6, 3, 'b', 0}, // incomplete fourth CP
- {7, 5, 6, 3, 'b', 0}, // incomplete fourth CP
- {8, 5, 6, 3, 'b', 0}, // incomplete fourth CP
- {9, 5, 6, 3, 'b', 0}, // incomplete fourth CP
+ // 2.1 The following test-cases raise doubt whether error or partial should
+ // be returned. For example, we have 4-byte sequence with valid leading
+ // byte. If we hide the last byte we need to return partial. But, if the
+ // second or third byte, which are visible to the call to codecvt, are
+ // malformed then error should be returned.
// replace first trailing byte with ASCII byte, also incomplete at end
{5, 5, 3, 2, 'z', 4},
-
- // replace first trailing byte with invalid byte, also incomplete at end
- {5, 5, 3, 2, '\xFF', 4},
-
- // replace first trailing byte with ASCII byte, also incomplete at end
{8, 5, 6, 3, 'z', 7},
{9, 5, 6, 3, 'z', 7},
// replace first trailing byte with invalid byte, also incomplete at end
- {8, 5, 6, 3, '\xFF', 7},
- {9, 5, 6, 3, '\xFF', 7},
+ {5, 5, 3, 2, 0xFF, 4},
+ {8, 5, 6, 3, 0xFF, 7},
+ {9, 5, 6, 3, 0xFF, 7},
// replace second trailing byte with ASCII byte, also incomplete at end
{9, 5, 6, 3, 'z', 8},
// replace second trailing byte with invalid byte, also incomplete at end
- {9, 5, 6, 3, '\xFF', 8},
+ {9, 5, 6, 3, 0xFF, 8},
+
+ // 3. Surrogate CP. We modify the second byte (first trailing) of the 3-byte
+ // CP U+D700
+ {6, 5, 3, 2, 0b10100000, 4}, // turn U+D700 into U+D800
+ {6, 5, 3, 2, 0b10101100, 4}, // turn U+D700 into U+DB00
+ {6, 5, 3, 2, 0b10110000, 4}, // turn U+D700 into U+DC00
+ {6, 5, 3, 2, 0b10111100, 4}, // turn U+D700 into U+DF00
+
+ // 4. Overlong sequence. The CPs in the input are chosen such as modifying
+ // just the leading byte is enough to make them overlong, i.e. for the
+ // 3-byte and 4-byte CP the second byte (first trailing) has enough leading
+ // zeroes.
+ {3, 5, 1, 1, 0b11000000, 1}, // make the 2-byte CP overlong
+ {3, 5, 1, 1, 0b11000001, 1}, // make the 2-byte CP overlong
+ {6, 5, 3, 2, 0b11100000, 3}, // make the 3-byte CP overlong
+ {10, 5, 6, 3, 0b11110000, 6}, // make the 4-byte CP overlong
+
+ // 5. CP above range
+ // turn U+10AAAA into U+14AAAA by changing its leading byte
+ {10, 5, 6, 3, 0b11110101, 6},
+ // turn U+10AAAA into U+11AAAA by changing its 2nd byte
+ {10, 5, 6, 3, 0b10011010, 7},
+ // Don't replace anything, show full 4-byte CP U+10AAAA
+ {10, 4, 6, 3, 'b', 0},
+ {10, 5, 6, 3, 'b', 0},
+ // Don't replace anything, show incomplete 4-byte CP at the end. It's still
+ // out of UCS2 range just by seeing the first byte.
+ {7, 4, 6, 3, 'b', 0}, // incomplete fourth CP
+ {8, 4, 6, 3, 'b', 0}, // incomplete fourth CP
+ {9, 4, 6, 3, 'b', 0}, // incomplete fourth CP
+ {7, 5, 6, 3, 'b', 0}, // incomplete fourth CP
+ {8, 5, 6, 3, 'b', 0}, // incomplete fourth CP
+ {9, 5, 6, 3, 'b', 0}, // incomplete fourth CP
};
for (auto t : offsets)
{
- char in[array_size (valid_in)] = {};
- CharT out[array_size (exp) - 1] = {};
+ InternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- char_traits<char>::copy (in, valid_in, array_size (valid_in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const char *) nullptr;
- auto out_next = (CharT *) nullptr;
+ auto in_next = (const ExternT *) nullptr;
+ auto out_next = (InternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<CharT>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-utf8_to_ucs2_in (const std::codecvt<CharT, char, mbstate_t> &cvt)
+utf8_to_ucs2_in (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_ucs2_in_ok (cvt);
utf8_to_ucs2_in_partial (cvt);
utf8_to_ucs2_in_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-ucs2_to_utf8_out_ok (const std::codecvt<CharT, char, mbstate_t> &cvt)
+ucs2_to_utf8_out_ok (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP
- const char16_t in_literal[] = u"bш\uAAAA";
- const char exp[] = "bш\uAAAA";
- CharT in[array_size (in_literal)] = {};
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 4, "");
- static_assert (array_size (exp) == 7, "");
- static_assert (array_size (in) == 4, "");
- VERIFY (char_traits<char16_t>::length (in_literal) == 3);
- VERIFY (char_traits<char>::length (exp) == 6);
- VERIFY (char_traits<CharT>::length (in) == 3);
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 7, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 3);
+ VERIFY (char_traits<ExternT>::length (exp) == 6);
const test_offsets_ok offsets[] = {{0, 0}, {1, 1}, {2, 3}, {3, 6}};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.ok);
VERIFY (in_next == in + t.in_size);
VERIFY (out_next == out + t.out_size);
- VERIFY (char_traits<char>::compare (out, exp, t.out_size) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.out_size) == 0);
if (t.out_size < array_size (out))
VERIFY (out[t.out_size] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-ucs2_to_utf8_out_partial (const std::codecvt<CharT, char, mbstate_t> &cvt)
+ucs2_to_utf8_out_partial (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
// UTF-8 string of 1-byte CP, 2-byte CP and 3-byte CP
- const char16_t in_literal[] = u"bш\uAAAA";
- const char exp[] = "bш\uAAAA";
- CharT in[array_size (in_literal)] = {};
- copy (begin (in_literal), end (in_literal), begin (in));
-
- static_assert (array_size (in_literal) == 4, "");
- static_assert (array_size (exp) == 7, "");
- static_assert (array_size (in) == 4, "");
- VERIFY (char_traits<char16_t>::length (in_literal) == 3);
- VERIFY (char_traits<char>::length (exp) == 6);
- VERIFY (char_traits<CharT>::length (in) == 3);
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 7, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 3);
+ VERIFY (char_traits<ExternT>::length (exp) == 6);
const test_offsets_partial offsets[] = {
{1, 0, 0, 0}, // no space for first CP
};
for (auto t : offsets)
{
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.partial);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-ucs2_to_utf8_out_error (const std::codecvt<CharT, char, mbstate_t> &cvt)
+ucs2_to_utf8_out_error (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
using namespace std;
- const char16_t valid_in[] = u"bш\uAAAA\U0010AAAA";
- const char exp[] = "bш\uAAAA\U0010AAAA";
-
- static_assert (array_size (valid_in) == 6, "");
- static_assert (array_size (exp) == 11, "");
- VERIFY (char_traits<char16_t>::length (valid_in) == 5);
- VERIFY (char_traits<char>::length (exp) == 10);
-
- test_offsets_error<CharT> offsets[] = {
- {5, 10, 0, 0, 0xD800, 0},
- {5, 10, 0, 0, 0xDBFF, 0},
- {5, 10, 0, 0, 0xDC00, 0},
- {5, 10, 0, 0, 0xDFFF, 0},
-
- {5, 10, 1, 1, 0xD800, 1},
- {5, 10, 1, 1, 0xDBFF, 1},
- {5, 10, 1, 1, 0xDC00, 1},
- {5, 10, 1, 1, 0xDFFF, 1},
-
- {5, 10, 2, 3, 0xD800, 2},
- {5, 10, 2, 3, 0xDBFF, 2},
- {5, 10, 2, 3, 0xDC00, 2},
- {5, 10, 2, 3, 0xDFFF, 2},
-
- // dont replace anything, just show the surrogate pair
- {5, 10, 3, 6, u'b', 0},
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const unsigned char expected[] = "b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 11, "");
+
+ InternT in[array_size (input)];
+ ExternT exp[array_size (expected)];
+ copy (begin (input), end (input), begin (in));
+ copy (begin (expected), end (expected), begin (exp));
+ VERIFY (char_traits<InternT>::length (in) == 5);
+ VERIFY (char_traits<ExternT>::length (exp) == 10);
+
+ test_offsets_error<InternT> offsets[] = {
+ {3, 6, 0, 0, 0xD800, 0},
+ {3, 6, 0, 0, 0xDBFF, 0},
+ {3, 6, 0, 0, 0xDC00, 0},
+ {3, 6, 0, 0, 0xDFFF, 0},
+
+ {3, 6, 1, 1, 0xD800, 1},
+ {3, 6, 1, 1, 0xDBFF, 1},
+ {3, 6, 1, 1, 0xDC00, 1},
+ {3, 6, 1, 1, 0xDFFF, 1},
+
+ {3, 6, 2, 3, 0xD800, 2},
+ {3, 6, 2, 3, 0xDBFF, 2},
+ {3, 6, 2, 3, 0xDC00, 2},
+ {3, 6, 2, 3, 0xDFFF, 2},
// make the leading surrogate a trailing one
{5, 10, 3, 6, 0xDC00, 3},
// make the trailing surrogate a BMP char
{5, 10, 3, 6, u'z', 4},
+ // don't replace anything in the test cases bellow, just show the surrogate
+ // pair (fourth CP) fully or partially
+ {5, 10, 3, 6, u'b', 0},
{5, 7, 3, 6, u'b', 0}, // no space for fourth CP
{5, 8, 3, 6, u'b', 0}, // no space for fourth CP
{5, 9, 3, 6, u'b', 0}, // no space for fourth CP
{4, 7, 3, 6, u'b', 0}, // incomplete fourth CP, and no space for it
{4, 8, 3, 6, u'b', 0}, // incomplete fourth CP, and no space for it
{4, 9, 3, 6, u'b', 0}, // incomplete fourth CP, and no space for it
-
};
for (auto t : offsets)
{
- CharT in[array_size (valid_in)] = {};
- char out[array_size (exp) - 1] = {};
+ ExternT out[array_size (exp) - 1] = {};
VERIFY (t.in_size <= array_size (in));
VERIFY (t.out_size <= array_size (out));
VERIFY (t.expected_in_next <= t.in_size);
VERIFY (t.expected_out_next <= t.out_size);
- copy (begin (valid_in), end (valid_in), begin (in));
+ auto old_char = in[t.replace_pos];
in[t.replace_pos] = t.replace_char;
auto state = mbstate_t{};
- auto in_next = (const CharT *) nullptr;
- auto out_next = (char *) nullptr;
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (ExternT *) nullptr;
auto res = codecvt_base::result ();
res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
VERIFY (res == cvt.error);
VERIFY (in_next == in + t.expected_in_next);
VERIFY (out_next == out + t.expected_out_next);
- VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ VERIFY (char_traits<ExternT>::compare (out, exp, t.expected_out_next)
+ == 0);
if (t.expected_out_next < array_size (out))
VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
}
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-ucs2_to_utf8_out (const std::codecvt<CharT, char, mbstate_t> &cvt)
+ucs2_to_utf8_out (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
ucs2_to_utf8_out_ok (cvt);
ucs2_to_utf8_out_partial (cvt);
ucs2_to_utf8_out_error (cvt);
}
-template <class CharT>
+template <class InternT, class ExternT>
void
-test_utf8_ucs2_cvts (const std::codecvt<CharT, char, mbstate_t> &cvt)
+test_utf8_ucs2_cvt (const std::codecvt<InternT, ExternT, mbstate_t> &cvt)
{
utf8_to_ucs2_in (cvt);
ucs2_to_utf8_out (cvt);
}
+
+enum utf16_endianess
+{
+ utf16_big_endian,
+ utf16_little_endian
+};
+
+template <class Iter1, class Iter2>
+Iter2
+utf16_to_bytes (Iter1 f, Iter1 l, Iter2 o, utf16_endianess e)
+{
+ if (e == utf16_big_endian)
+ for (; f != l; ++f)
+ {
+ *o++ = (*f >> 8) & 0xFF;
+ *o++ = *f & 0xFF;
+ }
+ else
+ for (; f != l; ++f)
+ {
+ *o++ = *f & 0xFF;
+ *o++ = (*f >> 8) & 0xFF;
+ }
+ return o;
+}
+
+template <class InternT>
+void
+utf16_to_utf32_in_ok (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 5, "");
+
+ char in[array_size (input) * 2];
+ InternT exp[array_size (expected)];
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+ copy (begin (expected), end (expected), begin (exp));
+
+ test_offsets_ok offsets[] = {{0, 0}, {2, 1}, {4, 2}, {6, 3}, {10, 4}};
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp)] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res
+ = cvt.in (state, in, in + t.in_size, in_next, out, end (out), out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf16_to_utf32_in_partial (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 5, "");
+
+ char in[array_size (input) * 2];
+ InternT exp[array_size (expected)];
+ auto in_iter = begin (in);
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+ copy (begin (expected), end (expected), begin (exp));
+
+ test_offsets_partial offsets[] = {
+ {2, 0, 0, 0}, // no space for first CP
+ {1, 1, 0, 0}, // incomplete first CP
+ {1, 0, 0, 0}, // incomplete first CP, and no space for it
+
+ {4, 1, 2, 1}, // no space for second CP
+ {3, 2, 2, 1}, // incomplete second CP
+ {3, 1, 2, 1}, // incomplete second CP, and no space for it
+
+ {6, 2, 4, 2}, // no space for third CP
+ {5, 3, 4, 2}, // incomplete third CP
+ {5, 2, 4, 2}, // incomplete third CP, and no space for it
+
+ {10, 3, 6, 3}, // no space for fourth CP
+ {7, 4, 6, 3}, // incomplete fourth CP
+ {8, 4, 6, 3}, // incomplete fourth CP
+ {9, 4, 6, 3}, // incomplete fourth CP
+ {7, 3, 6, 3}, // incomplete fourth CP, and no space for it
+ {8, 3, 6, 3}, // incomplete fourth CP, and no space for it
+ {9, 3, 6, 3}, // incomplete fourth CP, and no space for it
+ };
+
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.partial);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf16_to_utf32_in_error (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const char32_t expected[] = U"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 5, "");
+
+ InternT exp[array_size (expected)];
+ copy (begin (expected), end (expected), begin (exp));
+
+ // The only possible error in UTF-16 is unpaired surrogate code units.
+ // So we replace valid code points (scalar values) with lone surrogate CU.
+ test_offsets_error<char16_t> offsets[] = {
+ {10, 4, 0, 0, 0xD800, 0},
+ {10, 4, 0, 0, 0xDBFF, 0},
+ {10, 4, 0, 0, 0xDC00, 0},
+ {10, 4, 0, 0, 0xDFFF, 0},
+
+ {10, 4, 2, 1, 0xD800, 1},
+ {10, 4, 2, 1, 0xDBFF, 1},
+ {10, 4, 2, 1, 0xDC00, 1},
+ {10, 4, 2, 1, 0xDFFF, 1},
+
+ {10, 4, 4, 2, 0xD800, 2},
+ {10, 4, 4, 2, 0xDBFF, 2},
+ {10, 4, 4, 2, 0xDC00, 2},
+ {10, 4, 4, 2, 0xDFFF, 2},
+
+ // make the leading surrogate a trailing one
+ {10, 4, 6, 3, 0xDC00, 3},
+ {10, 4, 6, 3, 0xDFFF, 3},
+
+ // make the trailing surrogate a leading one
+ {10, 4, 6, 3, 0xD800, 4},
+ {10, 4, 6, 3, 0xDBFF, 4},
+
+ // make the trailing surrogate a BMP char
+ {10, 4, 6, 3, u'z', 4},
+ };
+
+ for (auto t : offsets)
+ {
+ char in[array_size (input) * 2];
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto old_char = input[t.replace_pos];
+ input[t.replace_pos] = t.replace_char; // replace in input, not in in
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.error);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+
+ input[t.replace_pos] = old_char;
+ }
+}
+
+template <class InternT>
+void
+utf32_to_utf16_out_ok (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 6, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ const test_offsets_ok offsets[] = {{0, 0}, {1, 2}, {2, 4}, {3, 6}, {4, 10}};
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<char>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf32_to_utf16_out_partial (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 6, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ const test_offsets_partial offsets[] = {
+ {1, 0, 0, 0}, // no space for first CP
+ {1, 1, 0, 0}, // no space for first CP
+
+ {2, 2, 1, 2}, // no space for second CP
+ {2, 3, 1, 2}, // no space for second CP
+
+ {3, 4, 2, 4}, // no space for third CP
+ {3, 5, 2, 4}, // no space for third CP
+
+ {4, 6, 3, 6}, // no space for fourth CP
+ {4, 7, 3, 6}, // no space for fourth CP
+ {4, 8, 3, 6}, // no space for fourth CP
+ {4, 9, 3, 6}, // no space for fourth CP
+ };
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.partial);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf32_to_utf16_out_error (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char32_t input[] = U"b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 5, "");
+ static_assert (array_size (expected) == 6, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ test_offsets_error<InternT> offsets[] = {
+
+ // Surrogate CP
+ {4, 10, 0, 0, 0xD800, 0},
+ {4, 10, 1, 2, 0xDBFF, 1},
+ {4, 10, 2, 4, 0xDC00, 2},
+ {4, 10, 3, 6, 0xDFFF, 3},
+
+ // CP out of range
+ {4, 10, 0, 0, 0x00110000, 0},
+ {4, 10, 1, 2, 0x00110000, 1},
+ {4, 10, 2, 4, 0x00110000, 2},
+ {4, 10, 3, 6, 0x00110000, 3}};
+
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto old_char = in[t.replace_pos];
+ in[t.replace_pos] = t.replace_char;
+
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.error);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
+ }
+}
+
+template <class InternT>
+void
+test_utf16_utf32_cvt (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ utf16_to_utf32_in_ok (cvt, endianess);
+ utf16_to_utf32_in_partial (cvt, endianess);
+ utf16_to_utf32_in_error (cvt, endianess);
+ utf32_to_utf16_out_ok (cvt, endianess);
+ utf32_to_utf16_out_partial (cvt, endianess);
+ utf32_to_utf16_out_error (cvt, endianess);
+}
+
+template <class InternT>
+void
+utf16_to_ucs2_in_ok (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 4, "");
+
+ char in[array_size (input) * 2];
+ InternT exp[array_size (expected)];
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+ copy (begin (expected), end (expected), begin (exp));
+
+ test_offsets_ok offsets[] = {{0, 0}, {2, 1}, {4, 2}, {6, 3}};
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp)] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res
+ = cvt.in (state, in, in + t.in_size, in_next, out, end (out), out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf16_to_ucs2_in_partial (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 4, "");
+
+ char in[array_size (input) * 2];
+ InternT exp[array_size (expected)];
+ auto in_iter = begin (in);
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+ copy (begin (expected), end (expected), begin (exp));
+
+ test_offsets_partial offsets[] = {
+ {2, 0, 0, 0}, // no space for first CP
+ {1, 1, 0, 0}, // incomplete first CP
+ {1, 0, 0, 0}, // incomplete first CP, and no space for it
+
+ {4, 1, 2, 1}, // no space for second CP
+ {3, 2, 2, 1}, // incomplete second CP
+ {3, 1, 2, 1}, // incomplete second CP, and no space for it
+
+ {6, 2, 4, 2}, // no space for third CP
+ {5, 3, 4, 2}, // incomplete third CP
+ {5, 2, 4, 2}, // incomplete third CP, and no space for it
+ };
+
+ for (auto t : offsets)
+ {
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.partial);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+ }
+}
+
+template <class InternT>
+void
+utf16_to_ucs2_in_error (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 6, "");
+
+ InternT exp[array_size (expected)];
+ copy (begin (expected), end (expected), begin (exp));
+
+ // The only possible error in UTF-16 is unpaired surrogate code units.
+ // Additionally, because the target encoding is UCS-2, a proper pair of
+ // surrogates is also error. Simply, any surrogate CU is error.
+ test_offsets_error<char16_t> offsets[] = {
+ {6, 3, 0, 0, 0xD800, 0},
+ {6, 3, 0, 0, 0xDBFF, 0},
+ {6, 3, 0, 0, 0xDC00, 0},
+ {6, 3, 0, 0, 0xDFFF, 0},
+
+ {6, 3, 2, 1, 0xD800, 1},
+ {6, 3, 2, 1, 0xDBFF, 1},
+ {6, 3, 2, 1, 0xDC00, 1},
+ {6, 3, 2, 1, 0xDFFF, 1},
+
+ {6, 3, 4, 2, 0xD800, 2},
+ {6, 3, 4, 2, 0xDBFF, 2},
+ {6, 3, 4, 2, 0xDC00, 2},
+ {6, 3, 4, 2, 0xDFFF, 2},
+
+ // make the leading surrogate a trailing one
+ {10, 5, 6, 3, 0xDC00, 3},
+ {10, 5, 6, 3, 0xDFFF, 3},
+
+ // make the trailing surrogate a leading one
+ {10, 5, 6, 3, 0xD800, 4},
+ {10, 5, 6, 3, 0xDBFF, 4},
+
+ // make the trailing surrogate a BMP char
+ {10, 5, 6, 3, u'z', 4},
+
+ // don't replace anything in the test cases bellow, just show the surrogate
+ // pair (fourth CP) fully or partially (just the first surrogate)
+ {10, 5, 6, 3, u'b', 0},
+ {8, 5, 6, 3, u'b', 0},
+ {9, 5, 6, 3, u'b', 0},
+
+ {10, 4, 6, 3, u'b', 0},
+ {8, 4, 6, 3, u'b', 0},
+ {9, 4, 6, 3, u'b', 0},
+ };
+
+ for (auto t : offsets)
+ {
+ char in[array_size (input) * 2];
+ InternT out[array_size (exp) - 1] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto old_char = input[t.replace_pos];
+ input[t.replace_pos] = t.replace_char; // replace in input, not in in
+ utf16_to_bytes (begin (input), end (input), begin (in), endianess);
+
+ auto state = mbstate_t{};
+ auto in_next = (const char *) nullptr;
+ auto out_next = (InternT *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.in (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.error);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<InternT>::compare (out, exp, t.expected_out_next)
+ == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+
+ input[t.replace_pos] = old_char;
+ }
+}
+
+template <class InternT>
+void
+ucs2_to_utf16_out_ok (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 4, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ const test_offsets_ok offsets[] = {{0, 0}, {1, 2}, {2, 4}, {3, 6}};
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.ok);
+ VERIFY (in_next == in + t.in_size);
+ VERIFY (out_next == out + t.out_size);
+ VERIFY (char_traits<char>::compare (out, exp, t.out_size) == 0);
+ if (t.out_size < array_size (out))
+ VERIFY (out[t.out_size] == 0);
+ }
+}
+
+template <class InternT>
+void
+ucs2_to_utf16_out_partial (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA";
+ static_assert (array_size (input) == 4, "");
+ static_assert (array_size (expected) == 4, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ const test_offsets_partial offsets[] = {
+ {1, 0, 0, 0}, // no space for first CP
+ {1, 1, 0, 0}, // no space for first CP
+
+ {2, 2, 1, 2}, // no space for second CP
+ {2, 3, 1, 2}, // no space for second CP
+
+ {3, 4, 2, 4}, // no space for third CP
+ {3, 5, 2, 4}, // no space for third CP
+ };
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.partial);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+ }
+}
+
+template <class InternT>
+void
+ucs2_to_utf16_out_error (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ using namespace std;
+ const char16_t input[] = u"b\u0448\uAAAA\U0010AAAA";
+ const char16_t expected[] = u"b\u0448\uAAAA\U0010AAAA";
+ static_assert (array_size (input) == 6, "");
+ static_assert (array_size (expected) == 6, "");
+
+ InternT in[array_size (input)];
+ char exp[array_size (expected) * 2];
+ copy (begin (input), end (input), begin (in));
+ utf16_to_bytes (begin (expected), end (expected), begin (exp), endianess);
+
+ test_offsets_error<InternT> offsets[] = {
+ {3, 6, 0, 0, 0xD800, 0},
+ {3, 6, 0, 0, 0xDBFF, 0},
+ {3, 6, 0, 0, 0xDC00, 0},
+ {3, 6, 0, 0, 0xDFFF, 0},
+
+ {3, 6, 1, 2, 0xD800, 1},
+ {3, 6, 1, 2, 0xDBFF, 1},
+ {3, 6, 1, 2, 0xDC00, 1},
+ {3, 6, 1, 2, 0xDFFF, 1},
+
+ {3, 6, 2, 4, 0xD800, 2},
+ {3, 6, 2, 4, 0xDBFF, 2},
+ {3, 6, 2, 4, 0xDC00, 2},
+ {3, 6, 2, 4, 0xDFFF, 2},
+
+ // make the leading surrogate a trailing one
+ {5, 10, 3, 6, 0xDC00, 3},
+ {5, 10, 3, 6, 0xDFFF, 3},
+
+ // make the trailing surrogate a leading one
+ {5, 10, 3, 6, 0xD800, 4},
+ {5, 10, 3, 6, 0xDBFF, 4},
+
+ // make the trailing surrogate a BMP char
+ {5, 10, 3, 6, u'z', 4},
+
+ // don't replace anything in the test cases bellow, just show the surrogate
+ // pair (fourth CP) fully or partially (just the first surrogate)
+ {5, 10, 3, 6, u'b', 0},
+ {5, 8, 3, 6, u'b', 0},
+ {5, 9, 3, 6, u'b', 0},
+
+ {4, 10, 3, 6, u'b', 0},
+ {4, 8, 3, 6, u'b', 0},
+ {4, 9, 3, 6, u'b', 0},
+ };
+
+ for (auto t : offsets)
+ {
+ char out[array_size (exp) - 2] = {};
+ VERIFY (t.in_size <= array_size (in));
+ VERIFY (t.out_size <= array_size (out));
+ VERIFY (t.expected_in_next <= t.in_size);
+ VERIFY (t.expected_out_next <= t.out_size);
+ auto old_char = in[t.replace_pos];
+ in[t.replace_pos] = t.replace_char;
+
+ auto state = mbstate_t{};
+ auto in_next = (const InternT *) nullptr;
+ auto out_next = (char *) nullptr;
+ auto res = codecvt_base::result ();
+
+ res = cvt.out (state, in, in + t.in_size, in_next, out, out + t.out_size,
+ out_next);
+ VERIFY (res == cvt.error);
+ VERIFY (in_next == in + t.expected_in_next);
+ VERIFY (out_next == out + t.expected_out_next);
+ VERIFY (char_traits<char>::compare (out, exp, t.expected_out_next) == 0);
+ if (t.expected_out_next < array_size (out))
+ VERIFY (out[t.expected_out_next] == 0);
+
+ in[t.replace_pos] = old_char;
+ }
+}
+
+template <class InternT>
+void
+test_utf16_ucs2_cvt (const std::codecvt<InternT, char, mbstate_t> &cvt,
+ utf16_endianess endianess)
+{
+ utf16_to_ucs2_in_ok (cvt, endianess);
+ utf16_to_ucs2_in_partial (cvt, endianess);
+ utf16_to_ucs2_in_error (cvt, endianess);
+ ucs2_to_utf16_out_ok (cvt, endianess);
+ ucs2_to_utf16_out_partial (cvt, endianess);
+ ucs2_to_utf16_out_error (cvt, endianess);
+}
projects / thirdparty / gcc.git / commitdiff
