blob: 9e42f58ddba338df457f2c88e8edec8fd511358d [file] [log] [blame]
Dan Albert7a7f9952014-06-02 11:33:04 -07001/*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17
Elliott Hughes18181e62019-01-30 13:52:36 -080018#include <uchar.h>
Dan Albert7a7f9952014-06-02 11:33:04 -070019
20#include <gtest/gtest.h>
21
22#include <errno.h>
23#include <limits.h>
24#include <locale.h>
25#include <stdint.h>
26
Dan Albert1252ab02023-07-14 21:23:36 +000027// Modern versions of UTF-8 (https://datatracker.ietf.org/doc/html/rfc3629 and
28// newer) explicitly disallow code points beyond U+10FFFF, which exclude all 5-
29// and 6-byte sequences. Earlier versions of UTF-8 allowed the wider range:
30// https://datatracker.ietf.org/doc/html/rfc2279.
31//
32// Bionic's unicode implementation was written after the high values were
33// excluded, so it has never supported them. Other implementations (at least
34// as of glibc 2.36), do support those sequences.
35#if defined(__ANDROID__) || defined(ANDROID_HOST_MUSL)
Dan Albert78da2922023-07-19 18:23:46 +000036constexpr bool kLibcRejectsOverLongUtf8Sequences = true;
Dan Albert1252ab02023-07-14 21:23:36 +000037#elif defined(__GLIBC__)
Dan Albert78da2922023-07-19 18:23:46 +000038constexpr bool kLibcRejectsOverLongUtf8Sequences = false;
Dan Albert1252ab02023-07-14 21:23:36 +000039#else
Dan Albert78da2922023-07-19 18:23:46 +000040#error kLibcRejectsOverLongUtf8Sequences must be configured for this platform
Dan Albert1252ab02023-07-14 21:23:36 +000041#endif
42
Dan Albert9a9bbe52023-07-18 23:12:47 +000043// C23 7.30.1 (for each `mbrtoc*` function) says:
44//
45// Returns:
46//
47// 0 if the next n or fewer bytes complete the multibyte character that
48// corresponds to the null wide character (which is the value stored).
49//
50// (size_t)(-2) if the next n bytes contribute to an incomplete (but
51// potentially valid) multibyte character, and all n bytes have been
52// processed (no value is stored).
53//
54// Bionic historically interpreted the behavior when n is 0 to be the next 0
55// bytes decoding to the null. That's a pretty bad interpretation, and both
56// glibc and musl return -2 for that case.
57//
58// The tests currently checks the incorrect behavior for bionic because gtest
59// doesn't support explicit xfail annotations. The behavior difference here
60// should be fixed, but danalbert wants to add more tests before tackling the
61// bugs.
62#ifdef __ANDROID__
63constexpr size_t kExpectedResultForZeroLength = 0U;
64#else
65constexpr size_t kExpectedResultForZeroLength = static_cast<size_t>(-2);
66#endif
67
Dan Albert7a7f9952014-06-02 11:33:04 -070068TEST(uchar, sizeof_uchar_t) {
Dan Albert7a7f9952014-06-02 11:33:04 -070069 EXPECT_EQ(2U, sizeof(char16_t));
70 EXPECT_EQ(4U, sizeof(char32_t));
Dan Albert7a7f9952014-06-02 11:33:04 -070071}
72
73TEST(uchar, start_state) {
Dan Albert09d3b502023-07-20 22:09:30 +000074 // C23 does not appear to specify the behavior of the conversion functions if
75 // a state is reused before the character is completed. In the wchar.h section
76 // (7.31.6.3) it says:
77 //
78 // If an mbstate_t object has been altered by any of the functions
79 // described in this subclause, and is then used with a different
80 // multibyte character sequence, or in the other conversion direction, or
81 // with a different LC_CTYPE category setting than on earlier function
82 // calls, the behavior is undefined.
83 //
84 // But "described in this subclause" refers to the wchar.h functions, not the
85 // uchar.h ones.
86 //
87 // Since C has no opinion, we need to make a choice. While no caller should
88 // ever do this (what does it mean to begin decoding a UTF-32 character while
89 // still in the middle of a UTF-8 sequence?), considering that a decoding
90 // error seems the least surprising. Bionic and glibc both have that behavior.
91 // musl ignores the state (it also doesn't make much sense to read the state
92 // when the entire conversion completes in a single call) and decodes the
93 // UTF-32 character.
94#if !defined(ANDROID_HOST_MUSL)
Dan Alberta0d0e352023-07-19 22:12:59 +000095 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
96 uselocale(LC_GLOBAL_LOCALE);
97
Dan Albert7a7f9952014-06-02 11:33:04 -070098 char out[MB_LEN_MAX];
99 mbstate_t ps;
100
Dan Albert7a7f9952014-06-02 11:33:04 -0700101 memset(&ps, 0, sizeof(ps));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700102 EXPECT_EQ(static_cast<size_t>(-2), mbrtoc32(nullptr, "\xc2", 1, &ps));
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700103 errno = 0;
Dan Albert7a7f9952014-06-02 11:33:04 -0700104 EXPECT_EQ(static_cast<size_t>(-1), c32rtomb(out, 0x00a2, &ps));
105 EXPECT_EQ(EILSEQ, errno);
106
Dan Albert09d3b502023-07-20 22:09:30 +0000107 // Similarly (but not in compliance with the standard afaict), musl seems to
108 // ignore the state entirely for the UTF-32 functions rather than reset it.
109
Yi Kong32bc0fc2018-08-02 17:31:13 -0700110 // If the first argument to c32rtomb is nullptr or the second is L'\0' the shift
Dan Albert7a7f9952014-06-02 11:33:04 -0700111 // state should be reset.
112 memset(&ps, 0, sizeof(ps));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700113 EXPECT_EQ(static_cast<size_t>(-2), mbrtoc32(nullptr, "\xc2", 1, &ps));
114 EXPECT_EQ(1U, c32rtomb(nullptr, 0x00a2, &ps));
Dan Albert7a7f9952014-06-02 11:33:04 -0700115 EXPECT_TRUE(mbsinit(&ps));
116
117 memset(&ps, 0, sizeof(ps));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700118 EXPECT_EQ(static_cast<size_t>(-2), mbrtoc32(nullptr, "\xf0\xa4", 1, &ps));
Dan Albert7a7f9952014-06-02 11:33:04 -0700119 EXPECT_EQ(1U, c32rtomb(out, L'\0', &ps));
120 EXPECT_TRUE(mbsinit(&ps));
Dan Albert09d3b502023-07-20 22:09:30 +0000121#endif
Dan Albert7a7f9952014-06-02 11:33:04 -0700122}
123
124TEST(uchar, c16rtomb_null_out) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000125 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
126 uselocale(LC_GLOBAL_LOCALE);
127
Yi Kong32bc0fc2018-08-02 17:31:13 -0700128 EXPECT_EQ(1U, c16rtomb(nullptr, L'\0', nullptr));
129 EXPECT_EQ(1U, c16rtomb(nullptr, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700130}
131
132TEST(uchar, c16rtomb_null_char) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000133 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
134 uselocale(LC_GLOBAL_LOCALE);
135
Dan Albert7a7f9952014-06-02 11:33:04 -0700136 char bytes[MB_LEN_MAX];
Yi Kong32bc0fc2018-08-02 17:31:13 -0700137 EXPECT_EQ(1U, c16rtomb(bytes, L'\0', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700138}
139
140TEST(uchar, c16rtomb) {
Dan Albert7a7f9952014-06-02 11:33:04 -0700141 char bytes[MB_LEN_MAX];
142
143 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700144 EXPECT_EQ(1U, c16rtomb(bytes, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700145 EXPECT_EQ('h', bytes[0]);
146
147 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
148 uselocale(LC_GLOBAL_LOCALE);
149
150 // 1-byte UTF-8.
151 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700152 EXPECT_EQ(1U, c16rtomb(bytes, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700153 EXPECT_EQ('h', bytes[0]);
154 // 2-byte UTF-8.
155 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700156 EXPECT_EQ(2U, c16rtomb(bytes, 0x00a2, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700157 EXPECT_EQ('\xc2', bytes[0]);
158 EXPECT_EQ('\xa2', bytes[1]);
159 // 3-byte UTF-8.
160 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700161 EXPECT_EQ(3U, c16rtomb(bytes, 0x20ac, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700162 EXPECT_EQ('\xe2', bytes[0]);
163 EXPECT_EQ('\x82', bytes[1]);
164 EXPECT_EQ('\xac', bytes[2]);
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700165 // 4-byte UTF-8 from a surrogate pair...
Dan Albert7a7f9952014-06-02 11:33:04 -0700166 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700167 EXPECT_EQ(0U, c16rtomb(bytes, 0xdbea, nullptr));
168 EXPECT_EQ(4U, c16rtomb(bytes, 0xdfcd, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700169 EXPECT_EQ('\xf4', bytes[0]);
170 EXPECT_EQ('\x8a', bytes[1]);
171 EXPECT_EQ('\xaf', bytes[2]);
172 EXPECT_EQ('\x8d', bytes[3]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700173}
174
175TEST(uchar, c16rtomb_invalid) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000176 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
177 uselocale(LC_GLOBAL_LOCALE);
178
Dan Albert7a7f9952014-06-02 11:33:04 -0700179 char bytes[MB_LEN_MAX];
180
181 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700182 EXPECT_EQ(static_cast<size_t>(-1), c16rtomb(bytes, 0xdfcd, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700183
Yi Kong32bc0fc2018-08-02 17:31:13 -0700184 EXPECT_EQ(0U, c16rtomb(bytes, 0xdbea, nullptr));
185 EXPECT_EQ(static_cast<size_t>(-1), c16rtomb(bytes, 0xdbea, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700186}
187
188TEST(uchar, mbrtoc16_null) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000189 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
190 uselocale(LC_GLOBAL_LOCALE);
191
Yi Kong32bc0fc2018-08-02 17:31:13 -0700192 ASSERT_EQ(0U, mbrtoc16(nullptr, nullptr, 0, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700193}
194
195TEST(uchar, mbrtoc16_zero_len) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000196 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
197 uselocale(LC_GLOBAL_LOCALE);
198
Dan Albert7a7f9952014-06-02 11:33:04 -0700199 char16_t out;
200
Alexander Ivchenko68b01662014-06-11 16:20:54 +0400201 out = L'x';
Dan Albert9a9bbe52023-07-18 23:12:47 +0000202 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc16(&out, "hello", 0, nullptr));
203 EXPECT_EQ(L'x', out);
Dan Albert7a7f9952014-06-02 11:33:04 -0700204
Dan Albert9a9bbe52023-07-18 23:12:47 +0000205 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc16(&out, "hello", 0, nullptr));
206 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc16(&out, "", 0, nullptr));
207 EXPECT_EQ(1U, mbrtoc16(&out, "hello", 1, nullptr));
208 EXPECT_EQ(L'h', out);
Dan Albert7a7f9952014-06-02 11:33:04 -0700209}
210
211TEST(uchar, mbrtoc16) {
Dan Albert7a7f9952014-06-02 11:33:04 -0700212 char16_t out;
213
214 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
215 uselocale(LC_GLOBAL_LOCALE);
216
217 // 1-byte UTF-8.
Yi Kong32bc0fc2018-08-02 17:31:13 -0700218 ASSERT_EQ(1U, mbrtoc16(&out, "abcdef", 6, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700219 ASSERT_EQ(L'a', out);
220 // 2-byte UTF-8.
Yi Kong32bc0fc2018-08-02 17:31:13 -0700221 ASSERT_EQ(2U, mbrtoc16(&out, "\xc2\xa2" "cdef", 6, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700222 ASSERT_EQ(static_cast<char16_t>(0x00a2), out);
223 // 3-byte UTF-8.
Yi Kong32bc0fc2018-08-02 17:31:13 -0700224 ASSERT_EQ(3U, mbrtoc16(&out, "\xe2\x82\xac" "def", 6, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700225 ASSERT_EQ(static_cast<char16_t>(0x20ac), out);
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700226 // 4-byte UTF-8 will be returned as a surrogate pair...
Dan Albert1e8e0c12023-07-19 19:14:45 +0000227#ifdef __BIONIC__
228 // https://issuetracker.google.com/289419882
229 //
230 // We misread the spec when implementing this. The first call should return
231 // the length of the decoded character, and the second call should return -3
232 // to indicate that the output is a continuation of the character decoded by
233 // the first call.
234 //
235 // C23 7.30.1.3.4:
236 //
237 // between 1 and n inclusive if the next n or fewer bytes complete a valid
238 // multibyte character (which is the value stored); the value returned is
239 // the number of bytes that complete the multibyte character.
240 //
241 // (size_t)(-3) if the next character resulting from a previous call has
242 // been stored (no bytes from the input have been consumed by this call).
243 //
244 // Leaving the test for the wrong outputs here while we clean up and improve
245 // the rest of the tests to get a better handle on the behavior differences
246 // before fixing the bug.
Dan Albert7a7f9952014-06-02 11:33:04 -0700247 ASSERT_EQ(static_cast<size_t>(-3),
Yi Kong32bc0fc2018-08-02 17:31:13 -0700248 mbrtoc16(&out, "\xf4\x8a\xaf\x8d", 6, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700249 ASSERT_EQ(static_cast<char16_t>(0xdbea), out);
Yi Kong32bc0fc2018-08-02 17:31:13 -0700250 ASSERT_EQ(4U, mbrtoc16(&out, "\xf4\x8a\xaf\x8d" "ef", 6, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700251 ASSERT_EQ(static_cast<char16_t>(0xdfcd), out);
Dan Albert1e8e0c12023-07-19 19:14:45 +0000252#else
253 ASSERT_EQ(4U, mbrtoc16(&out, "\xf4\x8a\xaf\x8d", 6, nullptr));
254 ASSERT_EQ(static_cast<char16_t>(0xdbea), out);
255 ASSERT_EQ(static_cast<size_t>(-3), mbrtoc16(&out,
256 "\xf4\x8a\xaf\x8d"
257 "ef",
258 6, nullptr));
259 ASSERT_EQ(static_cast<char16_t>(0xdfcd), out);
260#endif
Dan Albert1252ab02023-07-14 21:23:36 +0000261}
262
263TEST(uchar, mbrtoc16_long_sequences) {
264 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
265 uselocale(LC_GLOBAL_LOCALE);
266
267 char16_t out = u'\0';
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700268 errno = 0;
Dan Albert1252ab02023-07-14 21:23:36 +0000269 auto result = mbrtoc16(&out, "\xf8\xa1\xa2\xa3\xa4", 5, nullptr);
Dan Albert78da2922023-07-19 18:23:46 +0000270 if (kLibcRejectsOverLongUtf8Sequences) {
Dan Albert1252ab02023-07-14 21:23:36 +0000271 EXPECT_EQ(static_cast<size_t>(-1), result);
272 EXPECT_EQ(EILSEQ, errno);
273 EXPECT_EQ(u'\0', out);
Dan Albert78da2922023-07-19 18:23:46 +0000274 } else {
275 EXPECT_EQ(5U, result);
276 EXPECT_EQ(0, errno);
277 EXPECT_EQ(u'\uf94a', out);
Dan Albert1252ab02023-07-14 21:23:36 +0000278 }
Dan Albert7a7f9952014-06-02 11:33:04 -0700279}
280
281TEST(uchar, mbrtoc16_reserved_range) {
Dan Albertef8e1582023-07-19 19:34:12 +0000282 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
Dan Alberta0d0e352023-07-19 22:12:59 +0000283 uselocale(LC_GLOBAL_LOCALE);
Dan Albertef8e1582023-07-19 19:34:12 +0000284
285 errno = 0;
286 char16_t out = u'\0';
287 EXPECT_EQ(static_cast<size_t>(-1), mbrtoc16(&out, "\xf0\x80\xbf\xbf", 6, nullptr));
288 EXPECT_EQ(u'\0', out);
289 EXPECT_EQ(EILSEQ, errno);
Dan Albert7a7f9952014-06-02 11:33:04 -0700290}
291
292TEST(uchar, mbrtoc16_beyond_range) {
Dan Albertf5b8c7d2023-07-19 19:38:11 +0000293 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
Dan Alberta0d0e352023-07-19 22:12:59 +0000294 uselocale(LC_GLOBAL_LOCALE);
Dan Albertf5b8c7d2023-07-19 19:38:11 +0000295
296 errno = 0;
297 char16_t out = u'\0';
298 auto result = mbrtoc16(&out, "\xf5\x80\x80\x80", 6, nullptr);
299 if (kLibcRejectsOverLongUtf8Sequences) {
300 EXPECT_EQ(static_cast<size_t>(-1), result);
301 EXPECT_EQ(u'\0', out);
302 EXPECT_EQ(EILSEQ, errno);
303 } else {
304 EXPECT_EQ(4U, result);
305 EXPECT_EQ(u'\xdcc0', out);
306 EXPECT_EQ(0, errno);
307 }
Dan Albert7a7f9952014-06-02 11:33:04 -0700308}
309
Dan Albert7a7f9952014-06-02 11:33:04 -0700310void test_mbrtoc16_incomplete(mbstate_t* ps) {
311 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
312 uselocale(LC_GLOBAL_LOCALE);
313
314 char16_t out;
315 // 2-byte UTF-8.
316 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\xc2", 1, ps));
317 ASSERT_EQ(1U, mbrtoc16(&out, "\xa2" "cdef", 5, ps));
318 ASSERT_EQ(static_cast<char16_t>(0x00a2), out);
319 ASSERT_TRUE(mbsinit(ps));
320 // 3-byte UTF-8.
321 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\xe2", 1, ps));
322 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\x82", 1, ps));
323 ASSERT_EQ(1U, mbrtoc16(&out, "\xac" "def", 4, ps));
324 ASSERT_EQ(static_cast<char16_t>(0x20ac), out);
325 ASSERT_TRUE(mbsinit(ps));
326 // 4-byte UTF-8.
327 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\xf4", 1, ps));
328 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\x8a\xaf", 2, ps));
Dan Albert1e8e0c12023-07-19 19:14:45 +0000329#ifdef __BIONIC__
330 // https://issuetracker.google.com/289419882
331 // See explanation in mbrtoc16 test for the same bug.
Dan Albert7a7f9952014-06-02 11:33:04 -0700332 ASSERT_EQ(static_cast<size_t>(-3), mbrtoc16(&out, "\x8d" "ef", 3, ps));
333 ASSERT_EQ(static_cast<char16_t>(0xdbea), out);
334 ASSERT_EQ(1U, mbrtoc16(&out, "\x80" "ef", 3, ps));
335 ASSERT_EQ(static_cast<char16_t>(0xdfcd), out);
Dan Albert1e8e0c12023-07-19 19:14:45 +0000336#else
337 ASSERT_EQ(1U, mbrtoc16(&out,
338 "\x8d"
339 "ef",
340 3, ps));
341 ASSERT_EQ(static_cast<char16_t>(0xdbea), out);
342 ASSERT_EQ(static_cast<size_t>(-3), mbrtoc16(&out,
343 "\x80"
344 "ef",
345 3, ps));
346 ASSERT_EQ(static_cast<char16_t>(0xdfcd), out);
347#endif
Dan Albert7a7f9952014-06-02 11:33:04 -0700348 ASSERT_TRUE(mbsinit(ps));
349
350 // Invalid 2-byte
351 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc16(&out, "\xc2", 1, ps));
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700352 errno = 0;
Dan Albert7a7f9952014-06-02 11:33:04 -0700353 ASSERT_EQ(static_cast<size_t>(-1), mbrtoc16(&out, "\x20" "cdef", 5, ps));
354 ASSERT_EQ(EILSEQ, errno);
355}
Dan Albert7a7f9952014-06-02 11:33:04 -0700356
357TEST(uchar, mbrtoc16_incomplete) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000358 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
359 uselocale(LC_GLOBAL_LOCALE);
360
Dan Albert7a7f9952014-06-02 11:33:04 -0700361 mbstate_t ps;
362 memset(&ps, 0, sizeof(ps));
363
364 test_mbrtoc16_incomplete(&ps);
Yi Kong32bc0fc2018-08-02 17:31:13 -0700365 test_mbrtoc16_incomplete(nullptr);
Dan Albert7a7f9952014-06-02 11:33:04 -0700366}
367
368TEST(uchar, c32rtomb) {
Yi Kong32bc0fc2018-08-02 17:31:13 -0700369 EXPECT_EQ(1U, c32rtomb(nullptr, L'\0', nullptr));
370 EXPECT_EQ(1U, c32rtomb(nullptr, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700371
372 char bytes[MB_LEN_MAX];
373
Elliott Hughes697f42a2017-07-14 17:00:05 -0700374 memset(bytes, 1, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700375 EXPECT_EQ(1U, c32rtomb(bytes, L'\0', nullptr));
Elliott Hughes697f42a2017-07-14 17:00:05 -0700376 EXPECT_EQ('\0', bytes[0]);
377 EXPECT_EQ('\x01', bytes[1]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700378
379 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700380 EXPECT_EQ(1U, c32rtomb(bytes, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700381 EXPECT_EQ('h', bytes[0]);
382
383 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
384 uselocale(LC_GLOBAL_LOCALE);
385
386 // 1-byte UTF-8.
387 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700388 EXPECT_EQ(1U, c32rtomb(bytes, L'h', nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700389 EXPECT_EQ('h', bytes[0]);
390 // 2-byte UTF-8.
391 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700392 EXPECT_EQ(2U, c32rtomb(bytes, 0x00a2, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700393 EXPECT_EQ('\xc2', bytes[0]);
394 EXPECT_EQ('\xa2', bytes[1]);
395 // 3-byte UTF-8.
396 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700397 EXPECT_EQ(3U, c32rtomb(bytes, 0x20ac, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700398 EXPECT_EQ('\xe2', bytes[0]);
399 EXPECT_EQ('\x82', bytes[1]);
400 EXPECT_EQ('\xac', bytes[2]);
401 // 4-byte UTF-8.
402 memset(bytes, 0, sizeof(bytes));
Yi Kong32bc0fc2018-08-02 17:31:13 -0700403 EXPECT_EQ(4U, c32rtomb(bytes, 0x24b62, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700404 EXPECT_EQ('\xf0', bytes[0]);
405 EXPECT_EQ('\xa4', bytes[1]);
406 EXPECT_EQ('\xad', bytes[2]);
407 EXPECT_EQ('\xa2', bytes[3]);
408 // Invalid code point.
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700409 errno = 0;
Yi Kong32bc0fc2018-08-02 17:31:13 -0700410 EXPECT_EQ(static_cast<size_t>(-1), c32rtomb(bytes, 0xffffffff, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700411 EXPECT_EQ(EILSEQ, errno);
Dan Albert7a7f9952014-06-02 11:33:04 -0700412}
413
Elliott Hughes402c7622018-07-06 17:18:05 -0700414TEST(uchar, mbrtoc32_valid_non_characters) {
Elliott Hughes402c7622018-07-06 17:18:05 -0700415 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
416 uselocale(LC_GLOBAL_LOCALE);
417
418 char32_t out[8] = {};
419 ASSERT_EQ(3U, mbrtoc32(out, "\xef\xbf\xbe", 3, nullptr));
420 ASSERT_EQ(0xfffeU, out[0]);
421 ASSERT_EQ(3U, mbrtoc32(out, "\xef\xbf\xbf", 3, nullptr));
422 ASSERT_EQ(0xffffU, out[0]);
Elliott Hughes402c7622018-07-06 17:18:05 -0700423}
424
425TEST(uchar, mbrtoc32_out_of_range) {
Elliott Hughes402c7622018-07-06 17:18:05 -0700426 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
427 uselocale(LC_GLOBAL_LOCALE);
428
Dan Albert9fa76f12023-07-18 22:58:53 +0000429 char32_t out = U'\0';
Elliott Hughes402c7622018-07-06 17:18:05 -0700430 errno = 0;
Dan Albert9fa76f12023-07-18 22:58:53 +0000431 auto result = mbrtoc32(&out, "\xf5\x80\x80\x80", 4, nullptr);
Dan Albert78da2922023-07-19 18:23:46 +0000432 if (kLibcRejectsOverLongUtf8Sequences) {
Dan Albert9fa76f12023-07-18 22:58:53 +0000433 EXPECT_EQ(static_cast<size_t>(-1), result);
434 EXPECT_EQ(EILSEQ, errno);
435 EXPECT_EQ(U'\0', out);
Dan Albert78da2922023-07-19 18:23:46 +0000436 } else {
437 EXPECT_EQ(4U, result);
438 EXPECT_EQ(0, errno);
439 EXPECT_EQ(U'\x140000', out);
Dan Albert9fa76f12023-07-18 22:58:53 +0000440 }
Elliott Hughes402c7622018-07-06 17:18:05 -0700441}
442
Dan Albert7a7f9952014-06-02 11:33:04 -0700443TEST(uchar, mbrtoc32) {
Dan Albert7a7f9952014-06-02 11:33:04 -0700444 char32_t out[8];
445
Alexander Ivchenko68b01662014-06-11 16:20:54 +0400446 out[0] = L'x';
Dan Albert9a9bbe52023-07-18 23:12:47 +0000447 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc32(out, "hello", 0, nullptr));
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000448 EXPECT_EQ(static_cast<char32_t>(L'x'), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700449
Dan Albert9a9bbe52023-07-18 23:12:47 +0000450 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc32(out, "hello", 0, nullptr));
451 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc32(out, "", 0, nullptr));
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000452 EXPECT_EQ(1U, mbrtoc32(out, "hello", 1, nullptr));
453 EXPECT_EQ(static_cast<char32_t>(L'h'), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700454
Dan Albert9a9bbe52023-07-18 23:12:47 +0000455 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc32(nullptr, "hello", 0, nullptr));
456 EXPECT_EQ(kExpectedResultForZeroLength, mbrtoc32(nullptr, "", 0, nullptr));
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000457 EXPECT_EQ(1U, mbrtoc32(nullptr, "hello", 1, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700458
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000459 EXPECT_EQ(0U, mbrtoc32(nullptr, nullptr, 0, nullptr));
Dan Albert7a7f9952014-06-02 11:33:04 -0700460
461 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
462 uselocale(LC_GLOBAL_LOCALE);
463
464 // 1-byte UTF-8.
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000465 EXPECT_EQ(1U, mbrtoc32(out, "abcdef", 6, nullptr));
466 EXPECT_EQ(static_cast<char32_t>(L'a'), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700467 // 2-byte UTF-8.
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000468 EXPECT_EQ(2U, mbrtoc32(out,
469 "\xc2\xa2"
470 "cdef",
471 6, nullptr));
472 EXPECT_EQ(static_cast<char32_t>(0x00a2), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700473 // 3-byte UTF-8.
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000474 EXPECT_EQ(3U, mbrtoc32(out,
475 "\xe2\x82\xac"
476 "def",
477 6, nullptr));
478 EXPECT_EQ(static_cast<char32_t>(0x20ac), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700479 // 4-byte UTF-8.
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000480 EXPECT_EQ(4U, mbrtoc32(out,
481 "\xf0\xa4\xad\xa2"
482 "ef",
483 6, nullptr));
484 EXPECT_EQ(static_cast<char32_t>(0x24b62), out[0]);
Dan Albert7a7f9952014-06-02 11:33:04 -0700485#if defined(__BIONIC__) // glibc allows this.
486 // Illegal 5-byte UTF-8.
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700487 errno = 0;
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000488 EXPECT_EQ(static_cast<size_t>(-1), mbrtoc32(out,
489 "\xf8\xa1\xa2\xa3\xa4"
490 "f",
491 6, nullptr));
492 EXPECT_EQ(EILSEQ, errno);
Dan Albert7a7f9952014-06-02 11:33:04 -0700493#endif
494 // Illegal over-long sequence.
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700495 errno = 0;
Dan Albert0b8fbdf2023-07-18 22:21:36 +0000496 EXPECT_EQ(static_cast<size_t>(-1), mbrtoc32(out,
497 "\xf0\x82\x82\xac"
498 "ef",
499 6, nullptr));
500 EXPECT_EQ(EILSEQ, errno);
Dan Albert7a7f9952014-06-02 11:33:04 -0700501}
502
Dan Albert7a7f9952014-06-02 11:33:04 -0700503void test_mbrtoc32_incomplete(mbstate_t* ps) {
504 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
505 uselocale(LC_GLOBAL_LOCALE);
506
507 char32_t out;
508 // 2-byte UTF-8.
509 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\xc2", 1, ps));
510 ASSERT_EQ(1U, mbrtoc32(&out, "\xa2" "cdef", 5, ps));
511 ASSERT_EQ(static_cast<char32_t>(0x00a2), out);
512 ASSERT_TRUE(mbsinit(ps));
513 // 3-byte UTF-8.
514 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\xe2", 1, ps));
515 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\x82", 1, ps));
516 ASSERT_EQ(1U, mbrtoc32(&out, "\xac" "def", 4, ps));
517 ASSERT_EQ(static_cast<char32_t>(0x20ac), out);
518 ASSERT_TRUE(mbsinit(ps));
519 // 4-byte UTF-8.
520 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\xf0", 1, ps));
521 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\xa4\xad", 2, ps));
522 ASSERT_EQ(1U, mbrtoc32(&out, "\xa2" "ef", 3, ps));
523 ASSERT_EQ(static_cast<char32_t>(0x24b62), out);
524 ASSERT_TRUE(mbsinit(ps));
525
526 // Invalid 2-byte
527 ASSERT_EQ(static_cast<size_t>(-2), mbrtoc32(&out, "\xc2", 1, ps));
Elliott Hughes3d8156d2021-11-05 17:47:07 -0700528 errno = 0;
Dan Albert7a7f9952014-06-02 11:33:04 -0700529 ASSERT_EQ(static_cast<size_t>(-1), mbrtoc32(&out, "\x20" "cdef", 5, ps));
530 ASSERT_EQ(EILSEQ, errno);
531}
Dan Albert7a7f9952014-06-02 11:33:04 -0700532
533TEST(uchar, mbrtoc32_incomplete) {
Dan Alberta0d0e352023-07-19 22:12:59 +0000534 ASSERT_STREQ("C.UTF-8", setlocale(LC_CTYPE, "C.UTF-8"));
535 uselocale(LC_GLOBAL_LOCALE);
536
Dan Albert7a7f9952014-06-02 11:33:04 -0700537 mbstate_t ps;
538 memset(&ps, 0, sizeof(ps));
539
540 test_mbrtoc32_incomplete(&ps);
Yi Kong32bc0fc2018-08-02 17:31:13 -0700541 test_mbrtoc32_incomplete(nullptr);
Dan Albert7a7f9952014-06-02 11:33:04 -0700542}