UTF8Util.hpp

/*
 * Open Chinese Convert
 *
 * Copyright 2013 Carbo Kuo <byvoid@byvoid.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
#pragma once
 
#ifdef _MSC_VER
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif // _MSC_VER
 
#include <cstdint>
#include <cstring>
 
#include "Common.hpp"
#include "Exception.hpp"
 
namespace opencc {
class OPENCC_EXPORT UTF8Util {
public:
  static void SkipUtf8Bom(FILE* fp);

  static size_t NextCharLengthNoException(const char* str) {
    const unsigned char ch = static_cast<unsigned char>(*str);
    if ((ch & 0xF0) == 0xE0) {
      return 3;
    } else if ((ch & 0x80) == 0x00) {
      return 1;
    } else if ((ch & 0xE0) == 0xC0) {
      return 2;
    } else if ((ch & 0xF8) == 0xF0) {
      return 4;
    } else if ((ch & 0xFC) == 0xF8) {
      return 5;
    } else if ((ch & 0xFE) == 0xFC) {
      return 6;
    }
    return 0;
  }

  static size_t NextCharLength(const char* str) {
    size_t length = NextCharLengthNoException(str);
    if (length == 0) {
      throw InvalidUTF8(str);
    }
    return length;
  }

  static size_t PrevCharLength(const char* str) {
    const char* candidate = str - 1;
    size_t distance = 1;
    while (distance < 6) {
      const unsigned char ch = static_cast<unsigned char>(*candidate);
      if ((ch & 0xC0) != 0x80) {
        break;
      }
      candidate--;
      distance++;
    }
 
    const size_t length = NextCharLengthNoException(candidate);
    if (length == distance) {
      return length;
    }
    throw InvalidUTF8(str);
  }

  static const char* NextChar(const char* str) {
    return str + NextCharLength(str);
  }

  static const char* PrevChar(const char* str) {
    return str - PrevCharLength(str);
  }

  static const uint32_t kFirstIdeographicDescriptionOperator = 0x2FF0;
  static const uint32_t kLastIdeographicDescriptionOperator = 0x2FFF;
 
  static size_t IdeographicDescriptionOperatorArity(uint32_t codePoint) {
    switch (codePoint) {
    case 0x2FF2:
    case 0x2FF3:
      return 3;
    case 0x2FFE:
    case 0x2FFF:
      return 1;
    case 0x2FF0:
    case 0x2FF1:
    case 0x2FF4:
    case 0x2FF5:
    case 0x2FF6:
    case 0x2FF7:
    case 0x2FF8:
    case 0x2FF9:
    case 0x2FFA:
    case 0x2FFB:
    case 0x2FFC:
    case 0x2FFD:
      return 2;
    default:
      return 0;
    }
  }
 
  static size_t NextIdeographicDescriptionSequenceLength(const char* str,
                                                         size_t len) {
    const size_t kMaxIDSDepth = 16;
    const size_t kMaxIDSCodePoints = 64;
    if (len == 0) {
      return 0;
    }
    const size_t charLen = NextCharLengthNoException(str);
    if (charLen == 0 || charLen > len) {
      return 0;
    }
    const uint32_t codePoint = CodePointNoException(str, charLen);
    if (IdeographicDescriptionOperatorArity(codePoint) == 0) {
      return 0;
    }
 
    size_t consumed = 0;
    size_t codePoints = 0;
    if (ConsumeIdeographicDescriptionSequence(
            str, len, kMaxIDSDepth, kMaxIDSCodePoints, &consumed,
            &codePoints) == IDSParseStatus::Complete) {
      return consumed;
    }
    return 0;
  }
 
  static bool IsIncompleteIdeographicDescriptionSequencePrefix(const char* str,
                                                               size_t len) {
    const size_t kMaxIDSDepth = 16;
    const size_t kMaxIDSCodePoints = 64;
    if (len == 0) {
      return false;
    }
    const size_t charLen = NextCharLengthNoException(str);
    if (charLen == 0 || charLen > len) {
      return false;
    }
    const uint32_t codePoint = CodePointNoException(str, charLen);
    if (IdeographicDescriptionOperatorArity(codePoint) == 0) {
      return false;
    }
 
    size_t consumed = 0;
    size_t codePoints = 0;
    return ConsumeIdeographicDescriptionSequence(
               str, len, kMaxIDSDepth, kMaxIDSCodePoints, &consumed,
               &codePoints) == IDSParseStatus::Incomplete;
  }
 
  static bool IsVariationSelector(uint32_t codePoint) {
    return (codePoint >= 0xFE00 && codePoint <= 0xFE0F) ||
           (codePoint >= 0xE0100 && codePoint <= 0xE01EF);
  }
 
  static bool ContainsVariationSelector(const char* str, size_t len) {
    const char* pStr = str;
    const char* strEnd = str + len;
    while (pStr < strEnd) {
      const size_t remainingLength = strEnd - pStr;
      const size_t charLen = NextCharLengthNoException(pStr);
      if (charLen == 0) {
        ++pStr;
        continue;
      }
      if (charLen > remainingLength) {
        return false;
      }
      if (IsVariationSelector(CodePointNoException(pStr, charLen))) {
        return true;
      }
      pStr += charLen;
    }
    return false;
  }

  static size_t Length(const char* str) {
    size_t length = 0;
    while (*str != '\0') {
      const size_t charLen = NextCharLengthNoException(str);
      if (charLen == 0) {
        throw InvalidUTF8(str);
      }
      // Verify all continuation bytes are present before the null terminator.
      // Use a while loop (not a for-with-return) to avoid complex control flow
      // that triggers MSVC LTCG code-generator bugs.
      size_t i = 1;
      while (i < charLen && str[i] != '\0') {
        ++i;
      }
      if (i < charLen) {
        throw InvalidUTF8(str); // Truncated sequence: throw, don't silently skip
      }
      str += charLen;
      ++length;
    }
    return length;
  }

  static const char* FindNextInline(const char* str, const char ch) {
    while (!IsLineEndingOrFileEnding(*str) && *str != ch) {
      str = NextChar(str);
    }
    return str;
  }

  static bool IsLineEndingOrFileEnding(const char ch) {
    return ch == '\0' || ch == '\n' || ch == '\r';
  }

  static std::string FromSubstr(const char* str, size_t length) {
    std::string newStr;
    newStr.resize(length);
    memcpy(newStr.data(), str, length);
    return newStr;
  }

  static bool NotShorterThan(const char* str, size_t byteLength) {
    while (byteLength > 0) {
      if (*str == '\0') {
        return false;
      }
      byteLength--;
      str++;
    }
    return true;
  }

  static std::string TruncateUTF8(const char* str, size_t maxByteLength) {
    std::string wordTrunc;
    if (NotShorterThan(str, maxByteLength)) {
      size_t len = 0;
      const char* pStr = str;
      for (;;) {
        const size_t charLength = NextCharLength(pStr);
        if (len + charLength > maxByteLength) {
          break;
        }
        pStr += charLength;
        len += charLength;
      }
      wordTrunc = FromSubstr(str, len);
    } else {
      wordTrunc = str;
    }
    return wordTrunc;
  }

  static void ReplaceAll(std::string& str, const char* from, const char* to) {
    std::string::size_type pos = 0;
    std::string::size_type fromLen = strlen(from);
    std::string::size_type toLen = strlen(to);
    while ((pos = str.find(from, pos)) != std::string::npos) {
      str.replace(pos, fromLen, to);
      pos += toLen;
    }
  }

  static std::string Join(const std::vector<std::string>& strings,
                          const std::string& separator) {
    std::ostringstream buffer;
    bool first = true;
    for (const auto& str : strings) {
      if (!first) {
        buffer << separator;
      }
      buffer << str;
      first = false;
    }
    return buffer.str();
  }

  static std::string Join(const std::vector<std::string>& strings) {
    std::ostringstream buffer;
    for (const auto& str : strings) {
      buffer << str;
    }
    return buffer.str();
  }
 
  static void GetByteMap(const char* str, const size_t utf8Length,
                         std::vector<size_t>* byteMap) {
    if (byteMap->size() < utf8Length) {
      byteMap->resize(utf8Length);
    }
    const char* pstr = str;
    for (size_t i = 0; i < utf8Length; i++) {
      (*byteMap)[i] = pstr - str;
      pstr = NextChar(pstr);
    }
  }
 
#ifdef _MSC_VER
  static std::wstring GetPlatformString(const std::string& str) {
    return U8ToU16(str);
  }
#else
  static std::string GetPlatformString(const std::string& str) { return str; }
#endif // _MSC_VER
 
#ifdef _MSC_VER
  static std::string U16ToU8(const std::wstring& wstr) {
    std::string ret;
    int length = static_cast<int>(wstr.length());
    int convcnt = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, NULL, 0,
                                      NULL, NULL);
    if (convcnt > 0) {
      ret.resize(convcnt);
      WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), length, &ret[0], convcnt,
                          NULL, NULL);
    }
    return ret;
  }
 
  static std::wstring U8ToU16(const std::string& str) {
    std::wstring ret;
    int length = static_cast<int>(str.length());
    int convcnt = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, NULL, 0);
    if (convcnt > 0) {
      ret.resize(convcnt);
      MultiByteToWideChar(CP_UTF8, 0, str.c_str(), length, &ret[0], convcnt);
    }
    return ret;
  }
#endif // _MSC_VER
 
private:
  enum class IDSParseStatus {
    Complete,
    Incomplete,
    Invalid,
  };
 
  static uint32_t CodePointNoException(const char* str, size_t charLen) {
    const unsigned char first = static_cast<unsigned char>(str[0]);
    if (charLen == 1) {
      return first;
    }
 
    uint32_t codePoint = first & ((1U << (7 - charLen)) - 1);
    for (size_t i = 1; i < charLen; i++) {
      codePoint = (codePoint << 6) |
                  (static_cast<unsigned char>(str[i]) & 0x3FU);
    }
    return codePoint;
  }
 
  static IDSParseStatus ConsumeIdeographicDescriptionSequence(
      const char* str, size_t len, size_t depthLeft, size_t maxCodePoints,
      size_t* consumed, size_t* codePoints) {
    if (len == 0) {
      return IDSParseStatus::Incomplete;
    }
    if (depthLeft == 0 || *codePoints >= maxCodePoints) {
      return IDSParseStatus::Invalid;
    }
    const size_t charLen = NextCharLengthNoException(str);
    if (charLen == 0) {
      return IDSParseStatus::Invalid;
    }
    if (charLen > len) {
      return IDSParseStatus::Incomplete;
    }
    ++(*codePoints);
 
    const uint32_t codePoint = CodePointNoException(str, charLen);
    const size_t arity = IdeographicDescriptionOperatorArity(codePoint);
    if (arity == 0) {
      *consumed = charLen;
      return IDSParseStatus::Complete;
    }
 
    size_t offset = charLen;
    for (size_t i = 0; i < arity; i++) {
      if (offset >= len) {
        return IDSParseStatus::Incomplete;
      }
      size_t operandLength = 0;
      const IDSParseStatus operandStatus = ConsumeIdeographicDescriptionSequence(
          str + offset, len - offset, depthLeft - 1, maxCodePoints,
          &operandLength, codePoints);
      if (operandStatus != IDSParseStatus::Complete) {
        return operandStatus;
      }
      offset += operandLength;
    }
    *consumed = offset;
    return IDSParseStatus::Complete;
  }
};
} // namespace opencc