libutils/Unicode_test.cpp

/*
 * Copyright (C) 2010 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#define LOG_TAG "Unicode_test"

#include <sys/mman.h>
#include <unistd.h>

#include <log/log.h>
#include <utils/Unicode.h>

#include <gtest/gtest.h>

namespace android {

class UnicodeTest : public testing::Test {
protected:
    virtual void SetUp() {
    }

    virtual void TearDown() {
    }

    char16_t const * const kSearchString = u"I am a leaf on the wind.";

    constexpr static size_t BUFSIZE = 64;       // large enough for all tests

    void TestUTF8toUTF16(std::initializer_list<uint8_t> input,
                         std::initializer_list<char16_t> expect,
                         const char* err_msg_length = "",
                         ssize_t expected_length = 0) {
        uint8_t empty_str[] = {};
        char16_t output[BUFSIZE];

        const size_t inlen = input.size(), outlen = expect.size();
        ASSERT_LT(outlen, BUFSIZE);

        const uint8_t *input_data = inlen ? std::data(input) : empty_str;
        ssize_t measured = utf8_to_utf16_length(input_data, inlen);
        EXPECT_EQ(expected_length ? : (ssize_t)outlen, measured) << err_msg_length;

        utf8_to_utf16(input_data, inlen, output, outlen + 1);
        for (size_t i = 0; i < outlen; i++) {
            EXPECT_EQ(std::data(expect)[i], output[i]);
        }
        EXPECT_EQ(0, output[outlen]) << "should be null terminated";
    }

    void TestUTF16toUTF8(std::initializer_list<char16_t> input,
                         std::initializer_list<char> expect,
                         const char* err_msg_length = "",
                         ssize_t expected_length = 0) {
        char16_t empty_str[] = {};
        char output[BUFSIZE];

        const size_t inlen = input.size(), outlen = expect.size();
        ASSERT_LT(outlen, BUFSIZE);

        const char16_t *input_data = inlen ? std::data(input) : empty_str;
        ssize_t measured = utf16_to_utf8_length(input_data, inlen);
        EXPECT_EQ(expected_length ? : (ssize_t)outlen, measured) << err_msg_length;

        utf16_to_utf8(input_data, inlen, output, outlen + 1);
        for (size_t i = 0; i < outlen; i++) {
            EXPECT_EQ(std::data(expect)[i], output[i]);
        }
        EXPECT_EQ(0, output[outlen]) << "should be null terminated";
    }
};

TEST_F(UnicodeTest, UTF8toUTF16ZeroLength) {
    TestUTF8toUTF16({}, {},
        "Zero length input should return zero length output.");
}

TEST_F(UnicodeTest, UTF8toUTF16ASCII) {
    TestUTF8toUTF16(
        { 0x30 },               // U+0030 or ASCII '0'
        { 0x0030 },
        "ASCII codepoints should have a length of 1 char16_t");
}

TEST_F(UnicodeTest, UTF8toUTF16Plane1) {
    TestUTF8toUTF16(
        { 0xE2, 0x8C, 0xA3 },   // U+2323 SMILE
        { 0x2323 },
        "Plane 1 codepoints should have a length of 1 char16_t");
}

TEST_F(UnicodeTest, UTF8toUTF16Surrogate) {
    TestUTF8toUTF16(
        { 0xF0, 0x90, 0x80, 0x80 },   // U+10000
        { 0xD800, 0xDC00 },
        "Surrogate pairs should have a length of 2 char16_t");
}

TEST_F(UnicodeTest, UTF8toUTF16TruncatedUTF8) {
    TestUTF8toUTF16(
        { 0xE2, 0x8C },       // Truncated U+2323 SMILE
        { },                  // Conversion should still work but produce nothing
        "Truncated UTF-8 should return -1 to indicate invalid",
        -1);
}

TEST_F(UnicodeTest, UTF8toUTF16Normal) {
    TestUTF8toUTF16({
        0x30,                   // U+0030, 1 UTF-16 character
        0xC4, 0x80,             // U+0100, 1 UTF-16 character
        0xE2, 0x8C, 0xA3,       // U+2323, 1 UTF-16 character
        0xF0, 0x90, 0x80, 0x80, // U+10000, 2 UTF-16 character
    }, {
        0x0030,
        0x0100,
        0x2323,
        0xD800, 0xDC00
    });
}

TEST_F(UnicodeTest, UTF8toUTF16Invalid) {
    // TODO: The current behavior of utf8_to_utf16 is to treat invalid
    // leading byte (>= 0xf8) as a 4-byte UTF8 sequence, and to treat
    // invalid trailing byte(s) (i.e. bytes not having MSB set) as if
    // they are valid and do the normal conversion. However, a better
    // handling would be to treat invalid sequences as errors, such
    // cases need to be reported and invalid characters (e.g. U+FFFD)
    // could be produced at the place of error.  Until a fix is ready
    // and compatibility is not an issue, we will keep testing the
    // current behavior
    TestUTF8toUTF16({
        0xf8,                   // invalid leading byte
        0xc4, 0x00,             // U+0100 with invalid trailing byte
        0xe2, 0x0c, 0xa3,       // U+2323 with invalid trailing bytes
        0xf0, 0x10, 0x00, 0x00, // U+10000 with invalid trailing bytes
    }, {
        0x4022,                 // invalid leading byte (>=0xfc) is treated
                                // as valid for 4-byte UTF8 sequence
	0x000C,
	0x00A3,                 // invalid leadnig byte (b'10xxxxxx) is
                                // treated as valid single UTF-8 byte
        0xD800,                 // invalid trailing bytes are treated
        0xDC00,                 // as valid bytes and follow normal
    });
}

TEST_F(UnicodeTest, UTF16toUTF8ZeroLength) {
    // TODO: The current behavior of utf16_to_utf8_length() is that
    // it returns -1 if the input is a zero length UTF16 string.
    // This is inconsistent with utf8_to_utf16_length() where a zero
    // length string returns 0.  However, to fix the current behavior,
    // we could have compatibility issue.  Until then, we will keep
    // testing the current behavior
    TestUTF16toUTF8({}, {},
        "Zero length UTF16 input should return length of -1.", -1);
}

TEST_F(UnicodeTest, UTF16toUTF8ASCII) {
    TestUTF16toUTF8(
        { 0x0030 },  // U+0030 or ASCII '0'
        { '\x30' },
        "ASCII codepoints in UTF16 should give a length of 1 in UTF8");
}

TEST_F(UnicodeTest, UTF16toUTF8Plane1) {
    TestUTF16toUTF8(
        { 0x2323 },  // U+2323 SMILE
        { '\xE2', '\x8C', '\xA3' },
        "Plane 1 codepoints should have a length of 3 char in UTF-8");
}

TEST_F(UnicodeTest, UTF16toUTF8Surrogate) {
    TestUTF16toUTF8(
        { 0xD800, 0xDC00 },  // U+10000
        { '\xF0', '\x90', '\x80', '\x80' },
        "Surrogate pairs should have a length of 4 chars");
}

TEST_F(UnicodeTest, UTF16toUTF8UnpairedSurrogate) {
    TestUTF16toUTF8(
        { 0xD800 },     // U+10000 with high surrogate pair only
        { },            // Unpaired surrogate should be ignored
        "A single unpaired high surrogate should have a length of 0 chars");

    TestUTF16toUTF8(
        { 0xDC00 },     // U+10000 with low surrogate pair only
        { },            // Unpaired surrogate should be ignored
        "A single unpaired low surrogate should have a length of 0 chars");

    TestUTF16toUTF8(
        // U+0030, U+0100, U+10000 with high surrogate pair only, U+2323
        { 0x0030, 0x0100, 0xDC00, 0x2323 },
        { '\x30', '\xC4', '\x80', '\xE2', '\x8C', '\xA3' },
        "Unpaired high surrogate should be skipped in the middle");

    TestUTF16toUTF8(
        // U+0030, U+0100, U+10000 with high surrogate pair only, U+2323
        { 0x0030, 0x0100, 0xDC00, 0x2323 },
        { '\x30', '\xC4', '\x80', '\xE2', '\x8C', '\xA3' },
        "Unpaired low surrogate should be skipped in the middle");
}

TEST_F(UnicodeTest, UTF16toUTF8CorrectInvalidSurrogate) {
    // http://b/29250543
    // d841d8 is an invalid start for a surrogate pair. Make sure this is handled by ignoring the
    // first character in the pair and handling the rest correctly.
    TestUTF16toUTF8(
        { 0xD841, 0xD841, 0xDC41 },     // U+20441
        { '\xF0', '\xA0', '\x91', '\x81' },
        "Invalid start for a surrogate pair should be ignored");
}

TEST_F(UnicodeTest, UTF16toUTF8Normal) {
    TestUTF16toUTF8({
        0x0024, // U+0024 ($) --> 0x24,           1 UTF-8 byte
        0x00A3, // U+00A3 (£) --> 0xC2 0xA3,      2 UTF-8 bytes
        0x0939, // U+0939 (ह) --> 0xE0 0xA4 0xB9, 3 UTF-8 bytes
        0x20AC, // U+20AC (€) --> 0xE2 0x82 0xAC, 3 UTF-8 bytes
        0xD55C, // U+D55C (한)--> 0xED 0x95 0x9C, 3 UTF-8 bytes
        0xD801, 0xDC37, // U+10437 (𐐷) --> 0xF0 0x90 0x90 0xB7, 4 UTF-8 bytes
    }, {
        '\x24',
        '\xC2', '\xA3',
        '\xE0', '\xA4', '\xB9',
        '\xE2', '\x82', '\xAC',
        '\xED', '\x95', '\x9C',
        '\xF0', '\x90', '\x90', '\xB7',
    });
}

TEST_F(UnicodeTest, strstr16EmptyTarget) {
    EXPECT_EQ(strstr16(kSearchString, u""), kSearchString)
            << "should return the original pointer";
}

TEST_F(UnicodeTest, strstr16EmptyTarget_bug) {
    // In the original code when target is an empty string strlen16() would
    // start reading the memory until a "terminating null" (that is, zero)
    // character is found.   This happens because "*target++" in the original
    // code would increment the pointer beyond the actual string.
    void* memptr;
    const size_t alignment = sysconf(_SC_PAGESIZE);
    const size_t size = 2 * alignment;
    ASSERT_EQ(posix_memalign(&memptr, alignment, size), 0);
    // Fill allocated memory.
    memset(memptr, 'A', size);
    // Create a pointer to an "empty" string on the first page.
    char16_t* const emptyString = (char16_t* const)((char*)memptr + alignment - 4);
    *emptyString = (char16_t)0;
    // Protect the second page to show that strstr16() violates that.
    ASSERT_EQ(mprotect((char*)memptr + alignment, alignment, PROT_NONE), 0);
    // Test strstr16(): when bug is present a segmentation fault is raised.
    ASSERT_EQ(strstr16((char16_t*)memptr, emptyString), (char16_t*)memptr)
        << "should not read beyond the first char16_t.";
    // Reset protection of the second page
    ASSERT_EQ(mprotect((char*)memptr + alignment, alignment, PROT_READ | PROT_WRITE), 0);
    // Free allocated memory.
    free(memptr);
}

TEST_F(UnicodeTest, strstr16SameString) {
    const char16_t* result = strstr16(kSearchString, kSearchString);
    EXPECT_EQ(kSearchString, result)
            << "should return the original pointer";
}

TEST_F(UnicodeTest, strstr16TargetStartOfString) {
    const char16_t* result = strstr16(kSearchString, u"I am");
    EXPECT_EQ(kSearchString, result)
            << "should return the original pointer";
}


TEST_F(UnicodeTest, strstr16TargetEndOfString) {
    const char16_t* result = strstr16(kSearchString, u"wind.");
    EXPECT_EQ(kSearchString+19, result);
}

TEST_F(UnicodeTest, strstr16TargetWithinString) {
    const char16_t* result = strstr16(kSearchString, u"leaf");
    EXPECT_EQ(kSearchString+7, result);
}

TEST_F(UnicodeTest, strstr16TargetNotPresent) {
    const char16_t* result = strstr16(kSearchString, u"soar");
    EXPECT_EQ(nullptr, result);
}

// http://b/29267949
// Test that overreading in utf8_to_utf16_length is detected
TEST_F(UnicodeTest, InvalidUtf8OverreadDetected) {
    // An utf8 char starting with \xc4 is two bytes long.
    // Add extra zeros so no extra memory is read in case the code doesn't
    // work as expected.
    static char utf8[] = "\xc4\x00\x00\x00";
    ASSERT_DEATH(utf8_to_utf16_length((uint8_t *) utf8, strlen(utf8),
            true /* overreadIsFatal */), "" /* regex for ASSERT_DEATH */);
}

}