You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
86 lines
2.7 KiB
86 lines
2.7 KiB
#pragma once |
|
|
|
#include <cstdint> |
|
#include <string> |
|
#include <utility> |
|
|
|
/* Branchless UTF-8 decoder |
|
* |
|
* This is free and unencumbered software released into the public domain. |
|
*/ |
|
|
|
/* Decode the next character, C, from BUF, reporting errors in E. |
|
* |
|
* Since this is a branchless decoder, four bytes will be read from the |
|
* buffer regardless of the actual length of the next character. This |
|
* means the buffer _must_ have at least three bytes of zero padding |
|
* following the end of the data stream. |
|
* |
|
* Errors are reported in E, which will be non-zero if the parsed |
|
* character was somehow invalid: invalid byte sequence, non-canonical |
|
* encoding, or a surrogate half. |
|
* |
|
* The function returns a pointer to the next character. When an error |
|
* occurs, this pointer will be a guess that depends on the particular |
|
* error, but it will always advance at least one byte. |
|
*/ |
|
inline const unsigned char * |
|
utf8_decode(const unsigned char *buf, uint32_t *c, int *e) |
|
{ |
|
static const char lengths[] = { |
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, |
|
0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 3, 3, 4, 0 |
|
}; |
|
static const int masks[] = { 0x00, 0x7f, 0x1f, 0x0f, 0x07 }; |
|
static const uint32_t mins[] = { 4194304, 0, 128, 2048, 65536 }; |
|
static const int shiftc[] = { 0, 18, 12, 6, 0 }; |
|
static const int shifte[] = { 0, 6, 4, 2, 0 }; |
|
|
|
const unsigned char *s = buf; |
|
int len = lengths[s[0] >> 3]; |
|
|
|
/* Compute the pointer to the next character early so that the next |
|
* iteration can start working on the next character. Neither Clang |
|
* nor GCC figure out this reordering on their own. |
|
*/ |
|
const unsigned char *next = s + len + !len; |
|
|
|
/* Assume a four-byte character and load four bytes. Unused bits are |
|
* shifted out. |
|
*/ |
|
*c = (uint32_t)(s[0] & masks[len]) << 18; |
|
*c |= (uint32_t)(s[1] & 0x3f) << 12; |
|
*c |= (uint32_t)(s[2] & 0x3f) << 6; |
|
*c |= (uint32_t)(s[3] & 0x3f) << 0; |
|
*c >>= shiftc[len]; |
|
|
|
/* Accumulate the various error conditions. */ |
|
*e = (*c < mins[len]) << 6; // non-canonical encoding |
|
*e |= ((*c >> 11) == 0x1b) << 7; // surrogate half? |
|
*e |= (*c > 0x10FFFF) << 8; // out of range? |
|
*e |= (s[1] & 0xc0) >> 2; |
|
*e |= (s[2] & 0xc0) >> 4; |
|
*e |= (s[3]) >> 6; |
|
*e ^= 0x2a; // top two bits of each tail byte correct? |
|
*e >>= shifte[len]; |
|
|
|
return next; |
|
} |
|
|
|
inline std::string utf8_to_latin1(const char *in) |
|
{ |
|
std::string instr(in); |
|
instr.resize(instr.size() + 4); |
|
const unsigned char *buf = reinterpret_cast<const unsigned char *>(instr.data()); |
|
std::string ret; |
|
uint32_t next; |
|
int error; |
|
while (*buf) { |
|
buf = utf8_decode(buf, &next, &error); |
|
if (!error && next <= 255) |
|
ret.push_back(static_cast<char>(next)); |
|
else |
|
ret.push_back('?'); |
|
} |
|
return ret; |
|
}
|
|
|