mirror of
https://github.com/Alcaro/Flips.git
synced 2026-09-07 10:06:09 -05:00
546 lines
13 KiB
C++
546 lines
13 KiB
C++
// MIT License
|
|
//
|
|
// Copyright (c) 2016 Alfred Agrell
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in all
|
|
// copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
// SOFTWARE.
|
|
|
|
//The above license applies only to the WuTF directory, not the entire Arlib.
|
|
|
|
//See wutf.h for documentation.
|
|
|
|
#include "wutf.h"
|
|
|
|
static int decode(uint8_t head, const uint8_t* * tailptr, const uint8_t* end)
|
|
{
|
|
const int minforlen[] = { 0x7FFFFFFF, 0x80, 0x800, 0x10000, 0x7FFFFFFF };
|
|
|
|
const uint8_t * tail = *tailptr;
|
|
int numtrail = ((head&0xC0)==0xC0) + ((head&0xE0)==0xE0) + ((head&0xF0)==0xF0) + ((head&0xF8)==0xF8);
|
|
|
|
int codepoint = (head & (0x3F>>numtrail));
|
|
int i;
|
|
|
|
if (tail + numtrail > end) return -1;
|
|
|
|
for (i=0;i<3;i++)
|
|
{
|
|
if (numtrail>i)
|
|
{
|
|
if ((tail[i] & 0xC0) != 0x80) return -1;
|
|
codepoint = (codepoint<<6) | (tail[i] & 0x3F);
|
|
}
|
|
}
|
|
|
|
if (codepoint < minforlen[numtrail]) return -1;
|
|
|
|
*tailptr += numtrail;
|
|
|
|
return codepoint;
|
|
}
|
|
|
|
static int utf8_to_utf16_len(int flags, const uint8_t* ptr, const uint8_t* end)
|
|
{
|
|
int ret = 0;
|
|
const uint8_t* at = ptr;
|
|
|
|
while (at < end)
|
|
{
|
|
uint8_t head = *at++;
|
|
if (head <= 0x7F)
|
|
{
|
|
ret++;
|
|
continue;
|
|
}
|
|
|
|
uint32_t codepoint = (uint32_t)decode(head, &at, end);
|
|
ret++;
|
|
if ((codepoint&0xF800) == 0xD800 && !(flags & WUTF_WTF8))
|
|
{
|
|
if ((flags & WUTF_CESU8) && (codepoint&0xFC00) == 0xD800 &&
|
|
at+3 <= end && at[0]==0xED && (at[1]&0xF0)==0xB0 && (at[2]&0xC0)==0x80)
|
|
{
|
|
at+=3;
|
|
ret++;
|
|
}
|
|
else
|
|
{
|
|
at-=2;
|
|
goto fail;
|
|
}
|
|
}
|
|
if (codepoint>0x10FFFF)
|
|
{
|
|
if (codepoint != (uint32_t)-1) at-=3; // restore the tail
|
|
fail:
|
|
if ((flags&WUTF_INVALID_MASK) == WUTF_INVALID_ABORT) return WUTF_E_INVALID;
|
|
if ((flags&WUTF_INVALID_MASK) == WUTF_INVALID_DROP) ret--;
|
|
}
|
|
else if (codepoint>=0x10000) ret++; // surrogate
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static const uint8_t* utf8_end(const uint8_t* utf8, int utf8_len)
|
|
{
|
|
if (utf8_len >= 0)
|
|
{
|
|
return utf8 + utf8_len;
|
|
}
|
|
else
|
|
{
|
|
while (*utf8) utf8++;
|
|
utf8++; // go behind the NUL
|
|
return utf8;
|
|
}
|
|
}
|
|
|
|
int WuTF_utf8_to_utf16(int flags, const char* utf8, int utf8_len, uint16_t* utf16, int utf16_len)
|
|
{
|
|
if ((flags&WUTF_WTF8) && (flags&WUTF_CESU8)) return WUTF_E_INVALID;
|
|
if ((flags&WUTF_WTF8) && (flags&WUTF_INVALID_MASK)==WUTF_INVALID_DCXX) return WUTF_E_INVALID;
|
|
|
|
//I could run a bunch of special cases depending on whether cbMultiByte<0, etc,
|
|
//but there's no need. I'll optimize it if it ends up a bottleneck.
|
|
|
|
const uint8_t* iat = (const uint8_t*)utf8;
|
|
const uint8_t* iend = utf8_end(iat, utf8_len);
|
|
|
|
if (utf16_len == 0)
|
|
{
|
|
return utf8_to_utf16_len(flags, iat, iend);
|
|
}
|
|
|
|
uint16_t* oat = utf16;
|
|
uint16_t* oend = oat + utf16_len;
|
|
|
|
while (iat < iend)
|
|
{
|
|
if (oat+1 > oend) break;
|
|
|
|
uint8_t head = *iat++;
|
|
if (head <= 0x7F)
|
|
{
|
|
*oat++ = head;
|
|
continue;
|
|
}
|
|
|
|
uint32_t codepoint = (uint32_t)decode(head, &iat, iend); // -1 -> 0xFFFFFFFF
|
|
if (codepoint <= 0xFFFF)
|
|
{
|
|
if ((codepoint&0xF800) == 0xD800 && !(flags & WUTF_WTF8))
|
|
{
|
|
if ((flags & WUTF_CESU8) && (codepoint&0xFC00) == 0xD800 &&
|
|
iat+3 <= iend && iat[0]==0xED && (iat[1]&0xF0)==0xB0 && (iat[2]&0xC0)==0x80)
|
|
{
|
|
*oat++ = codepoint;
|
|
codepoint = (uint32_t)decode(*iat++, &iat, iend);
|
|
}
|
|
else
|
|
{
|
|
iat-=2;
|
|
goto fail;
|
|
}
|
|
}
|
|
*oat++ = codepoint;
|
|
}
|
|
else
|
|
{
|
|
if (codepoint > 0x10FFFF)
|
|
{
|
|
if (codepoint != (uint32_t)-1) iat-=3; // restore the tail
|
|
fail:
|
|
switch (flags & WUTF_INVALID_MASK)
|
|
{
|
|
case WUTF_INVALID_ABORT: return WUTF_E_INVALID;
|
|
case WUTF_INVALID_DROP: break;
|
|
case WUTF_INVALID_FFFD: *oat++ = 0xFFFD; break;
|
|
case WUTF_INVALID_DCXX: *oat++ = 0xDC00 + head; break;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (oat+2 > oend) break;
|
|
codepoint -= 0x10000;
|
|
*oat++ = 0xD800 | (codepoint>>10);
|
|
*oat++ = 0xDC00 | (codepoint&0x3FF);
|
|
}
|
|
}
|
|
if (iat != iend)
|
|
{
|
|
if (!(flags & WUTF_TRUNCATE)) return WUTF_E_TRUNCATE;
|
|
while (oat < oend) *oat++ = 0xFFFD;
|
|
}
|
|
|
|
return oat - utf16;
|
|
}
|
|
|
|
|
|
|
|
static int utf8_to_utf32_len(int flags, const uint8_t* ptr, const uint8_t* end)
|
|
{
|
|
int ret = 0;
|
|
const uint8_t* at = ptr;
|
|
|
|
while (at < end)
|
|
{
|
|
uint8_t head = *at++;
|
|
if (head <= 0x7F)
|
|
{
|
|
ret++;
|
|
continue;
|
|
}
|
|
|
|
uint32_t codepoint = (uint32_t)decode(head, &at, end);
|
|
ret++;
|
|
if (codepoint>0x10FFFF)
|
|
{
|
|
switch (flags & WUTF_INVALID_MASK)
|
|
{
|
|
case WUTF_INVALID_ABORT: return WUTF_E_INVALID;
|
|
case WUTF_INVALID_DROP: ret--; break;
|
|
case WUTF_INVALID_FFFD: break;
|
|
case WUTF_INVALID_DCXX: break;
|
|
}
|
|
}
|
|
//identical to utf16_len, just discard the surrogate check
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int WuTF_utf8_to_utf32(int flags, const char* utf8, int utf8_len, uint32_t* utf32, int utf32_len)
|
|
{
|
|
if (flags&WUTF_CESU8) return WUTF_E_INVALID;
|
|
if ((flags&WUTF_WTF8) && (flags&WUTF_INVALID_MASK)==WUTF_INVALID_DCXX) return WUTF_E_INVALID;
|
|
|
|
const uint8_t* iat = (const uint8_t*)utf8;
|
|
const uint8_t* iend = utf8_end(iat, utf8_len);
|
|
|
|
if (utf32_len == 0)
|
|
{
|
|
return utf8_to_utf32_len(flags, iat, iend);
|
|
}
|
|
|
|
uint32_t* oat = utf32;
|
|
uint32_t* oend = oat + utf32_len;
|
|
|
|
while (iat < iend)
|
|
{
|
|
if (oat+1 > oend) break;
|
|
|
|
uint8_t head = *iat++;
|
|
if (head <= 0x7F)
|
|
{
|
|
*oat++ = head;
|
|
continue;
|
|
}
|
|
|
|
uint32_t codepoint = (uint32_t)decode(head, &iat, iend); // -1 -> 0xFFFFFFFF
|
|
if (!(flags&WUTF_WTF8) && (codepoint&0xFFFFF800) == 0xD800)
|
|
{
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_ABORT) return WUTF_E_INVALID;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DROP) continue;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_FFFD) {}
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DCXX)
|
|
{
|
|
if ((head&0xFF00)==0xDC00) {}
|
|
else return WUTF_E_INVALID;
|
|
}
|
|
}
|
|
|
|
if (codepoint > 0x10FFFF)
|
|
{
|
|
switch (flags & WUTF_INVALID_MASK)
|
|
{
|
|
case WUTF_INVALID_ABORT: return WUTF_E_INVALID;
|
|
case WUTF_INVALID_DROP: break;
|
|
case WUTF_INVALID_FFFD: *oat++ = 0xFFFD; break;
|
|
case WUTF_INVALID_DCXX: *oat++ = 0xDC00 + head; break;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
*oat++ = codepoint;
|
|
}
|
|
if (iat != iend)
|
|
{
|
|
if (!(flags & WUTF_TRUNCATE)) return WUTF_E_TRUNCATE;
|
|
while (oat < oend) *oat++ = 0xFFFD;
|
|
}
|
|
|
|
return oat - utf32;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static int utf16_to_utf8_len(int flags, const uint16_t* ptr, const uint16_t* end)
|
|
{
|
|
int ret = 0;
|
|
const uint16_t* at = ptr;
|
|
|
|
while (at < end)
|
|
{
|
|
uint16_t head = *at++;
|
|
ret++;
|
|
if (head >= 0x80) ret++;
|
|
if (head >= 0x0800)
|
|
{
|
|
ret++;
|
|
if ((head&0xF800)==0xD800)
|
|
{
|
|
if (head<=0xDBFF && at < end && *at >= 0xDC00 && *at <= 0xDFFF)
|
|
{
|
|
at++;
|
|
ret++;
|
|
continue;
|
|
}
|
|
|
|
if (!(flags & WUTF_WTF8))
|
|
{
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_ABORT) return WUTF_E_INVALID;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DROP) ret--; continue;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_FFFD) continue;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DCXX)
|
|
{
|
|
if ((head&0xFF00)==0xDC00) continue;
|
|
else return WUTF_E_INVALID;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static const uint16_t* utf16_end(const uint16_t* utf16, int utf16_len)
|
|
{
|
|
if (utf16_len >= 0)
|
|
{
|
|
return utf16 + utf16_len;
|
|
}
|
|
else
|
|
{
|
|
while (*utf16) utf16++;
|
|
utf16++; // go behind the NUL
|
|
return utf16;
|
|
}
|
|
}
|
|
|
|
int WuTF_utf16_to_utf8(int flags, const uint16_t* utf16, int utf16_len, char* utf8, int utf8_len)
|
|
{
|
|
if (flags&WUTF_CESU8) return WUTF_E_INVALID;
|
|
if ((flags&WUTF_WTF8) && (flags&WUTF_INVALID_MASK) == WUTF_INVALID_DCXX) return WUTF_E_INVALID;
|
|
|
|
const uint16_t* iat = utf16;
|
|
const uint16_t* iend = utf16_end(iat, utf16_len);
|
|
|
|
if (utf8_len == 0)
|
|
{
|
|
return utf16_to_utf8_len(flags, iat, iend);
|
|
}
|
|
|
|
uint8_t* oat = (uint8_t*)utf8;
|
|
uint8_t* oend = oat + utf8_len;
|
|
|
|
while (iat < iend)
|
|
{
|
|
uint16_t head = *iat++;
|
|
if (head <= 0x7F)
|
|
{
|
|
if (oat+1 > oend) break;
|
|
*oat++ = (uint8_t)head;
|
|
}
|
|
else if (head <= 0x07FF)
|
|
{
|
|
if (oat+2 > oend) break;
|
|
*oat++ = (((head>> 6) )|0xC0);
|
|
*oat++ = (((head )&0x3F)|0x80);
|
|
}
|
|
else
|
|
{
|
|
if ((head&0xF800)==0xD800)
|
|
{
|
|
if (head<=0xDBFF && iat < iend)
|
|
{
|
|
uint16_t tail = *iat;
|
|
if (tail >= 0xDC00 && tail <= 0xDFFF)
|
|
{
|
|
iat++;
|
|
if (oat+4 > oend) break;
|
|
uint32_t codepoint = 0x10000+((head&0x03FF)<<10)+(tail&0x03FF);
|
|
*oat++ = (((codepoint>>18)&0x07)|0xF0);
|
|
*oat++ = (((codepoint>>12)&0x3F)|0x80);
|
|
*oat++ = (((codepoint>>6 )&0x3F)|0x80);
|
|
*oat++ = (((codepoint )&0x3F)|0x80);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (!(flags & WUTF_WTF8))
|
|
{
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_ABORT) return WUTF_E_INVALID;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DROP) continue;
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_FFFD) { head = 0xFFFD; continue; }
|
|
if ((flags & WUTF_INVALID_MASK) == WUTF_INVALID_DCXX)
|
|
{
|
|
if ((head&0xFF00)==0xDC00)
|
|
{
|
|
if (oat+1 > oend) break;
|
|
*oat++ = (head&0x00FF); // don't bother ensuring that this ends up as invalid utf8, too much effort
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
return WUTF_E_INVALID;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (oat+3 > oend) break;
|
|
*oat++ = (((head>>12)&0x0F)|0xE0);
|
|
*oat++ = (((head>>6 )&0x3F)|0x80);
|
|
*oat++ = (((head )&0x3F)|0x80);
|
|
}
|
|
}
|
|
if (iat != iend)
|
|
{
|
|
if (!(flags & WUTF_TRUNCATE)) return WUTF_E_TRUNCATE;
|
|
while (oat < oend) *oat++ = '?'; // (probably) can't fit a U+FFFD, just shove in something
|
|
}
|
|
|
|
return oat - (uint8_t*)utf8;
|
|
}
|
|
|
|
|
|
|
|
static int utf32_to_utf8_len(int flags, const uint32_t* ptr, const uint32_t* end)
|
|
{
|
|
int ret = 0;
|
|
const uint32_t* at = ptr;
|
|
|
|
while (at < end)
|
|
{
|
|
uint32_t head = *at++;
|
|
ret++;
|
|
if (head >= 0x80) ret++;
|
|
if (head >= 0x0800) ret++;
|
|
if (head >= 0x10000) ret++;
|
|
if (head > 0x10FFFF)
|
|
{
|
|
switch (flags & WUTF_INVALID_MASK)
|
|
{
|
|
case WUTF_INVALID_ABORT: return WUTF_E_INVALID;
|
|
case WUTF_INVALID_DROP: ret-=4; break;
|
|
case WUTF_INVALID_FFFD: ret-=4; ret+=3; break;
|
|
case WUTF_INVALID_DCXX: ret-=4; ret+=3; break;
|
|
}
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static const uint32_t* utf32_end(const uint32_t* utf32, int utf32_len)
|
|
{
|
|
if (utf32_len >= 0)
|
|
{
|
|
return utf32 + utf32_len;
|
|
}
|
|
else
|
|
{
|
|
while (*utf32) utf32++;
|
|
utf32++; // go behind the NUL
|
|
return utf32;
|
|
}
|
|
}
|
|
|
|
int WuTF_utf32_to_utf8(int flags, const uint32_t* utf32, int utf32_len, char* utf8, int utf8_len)
|
|
{
|
|
if (flags&WUTF_CESU8) return WUTF_E_INVALID;
|
|
if ((flags&WUTF_INVALID_MASK) == WUTF_INVALID_DCXX) return WUTF_E_INVALID;
|
|
|
|
if (flags&WUTF_WTF8) return WUTF_E_INVALID; // TODO
|
|
if ((flags&WUTF_INVALID_MASK) != -1) return WUTF_E_INVALID; // TODO
|
|
|
|
const uint32_t* iat = utf32;
|
|
const uint32_t* iend = utf32_end(iat, utf32_len);
|
|
|
|
if (utf8_len == 0)
|
|
{
|
|
return utf32_to_utf8_len(flags, iat, iend);
|
|
}
|
|
|
|
uint8_t* oat = (uint8_t*)utf8;
|
|
uint8_t* oend = oat + utf8_len;
|
|
|
|
while (iat < iend)
|
|
{
|
|
uint32_t head = *iat++;
|
|
if (head <= 0x7F)
|
|
{
|
|
if (oat+1 > oend) break;
|
|
*oat++ = head;
|
|
}
|
|
else if (head <= 0x07FF)
|
|
{
|
|
if (oat+2 > oend) break;
|
|
*oat++ = (((head>> 6) )|0xC0);
|
|
*oat++ = (((head )&0x3F)|0x80);
|
|
}
|
|
else if (head <= 0xFFFF)
|
|
{
|
|
if (oat+3 > oend) break;
|
|
*oat++ = (((head>>12)&0x0F)|0xE0);
|
|
*oat++ = (((head>>6 )&0x3F)|0x80);
|
|
*oat++ = (((head )&0x3F)|0x80);
|
|
}
|
|
else if (head <= 0x10FFFF)
|
|
{
|
|
if (oat+4 > oend) break;
|
|
*oat++ = (((head>>18)&0x07)|0xF0);
|
|
*oat++ = (((head>>12)&0x3F)|0x80);
|
|
*oat++ = (((head>>6 )&0x3F)|0x80);
|
|
*oat++ = (((head )&0x3F)|0x80);
|
|
}
|
|
else
|
|
{
|
|
switch (flags & WUTF_INVALID_MASK)
|
|
{
|
|
case WUTF_INVALID_ABORT: return WUTF_E_INVALID;
|
|
case WUTF_INVALID_DROP: break;
|
|
case WUTF_INVALID_FFFD:
|
|
if (oat+3 <= oend)
|
|
{
|
|
*oat++ = 0xEF; // U+FFFD
|
|
*oat++ = 0xBF;
|
|
*oat++ = 0xBD;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if (iat != iend)
|
|
{
|
|
if (!(flags & WUTF_TRUNCATE)) return WUTF_E_TRUNCATE;
|
|
while (oat < oend) *oat++ = '?';
|
|
}
|
|
|
|
return oat - (uint8_t*)utf8;
|
|
}
|