diff --git a/Source/Core/Common/StringUtil.cpp b/Source/Core/Common/StringUtil.cpp index 19fc82e350..0b89807535 100644 --- a/Source/Core/Common/StringUtil.cpp +++ b/Source/Core/Common/StringUtil.cpp @@ -438,6 +438,31 @@ private: const CharType* const m_end_ptr; }; +template CharType> +class CP1252Decoder : public CodeUnitReader +{ +public: + using CodeUnitReader::CodeUnitReader; + + constexpr char32_t operator()() + { + const u8 code_unit = this->ReadCodeUnit(); + + // ISO/IEC 8859-1 "extended ASCII" equivalent values. + if (code_unit < 0x80u || code_unit > 0x9fu) + return code_unit; + + static constexpr auto unused = UNICODE_REPLACEMENT_CHARACTER; + + static constexpr std::array values_from_80_to_9f = { + 0x20ac, unused, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021, 0x02c6, 0x2030, 0x0160, + 0x2039, 0x0152, unused, 0x017d, unused, unused, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, + 0x2013, 0x2014, 0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, unused, 0x017e, 0x0178}; + + return values_from_80_to_9f[code_unit - 0x80u]; + } +}; + template CharType> class UTF8Decoder : public CodeUnitReader { @@ -702,11 +727,6 @@ std::string UTF8ToSHIFTJIS(std::string_view input) return UTF16ToCP(CODEPAGE_SHIFT_JIS, UTF8ToWString(input)); } -std::string CP1252ToUTF8(std::string_view input) -{ - return WStringToUTF8(CPToUTF16(CODEPAGE_WINDOWS_1252, input)); -} - std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) { const char16_t* str_end = std::find(str, str + max_size, '\0'); @@ -780,13 +800,6 @@ static std::string CodeToUTF8(const char* fromcode, std::basic_string_view in return CodeTo("UTF-8", fromcode, input); } -std::string CP1252ToUTF8(std::string_view input) -{ - // return CodeToUTF8("CP1252//TRANSLIT", input); - // return CodeToUTF8("CP1252//IGNORE", input); - return CodeToUTF8("CP1252", input); -} - std::string SHIFTJISToUTF8(std::string_view input) { // return CodeToUTF8("CP932", input); @@ -813,6 +826,11 @@ std::string UTF16BEToUTF8(const char16_t* str, size_t max_size) #endif +std::string CP1252ToUTF8(std::string_view input) +{ + return ReEncodeString, UTF8Encoder, char>(input); +} + std::string UTF16ToUTF8(std::u16string_view input) { return ReEncodeString, UTF8Encoder, char>(input); diff --git a/Source/UnitTests/Common/StringUtilTest.cpp b/Source/UnitTests/Common/StringUtilTest.cpp index 7b6f1828bc..6bf7d7ccdd 100644 --- a/Source/UnitTests/Common/StringUtilTest.cpp +++ b/Source/UnitTests/Common/StringUtilTest.cpp @@ -463,5 +463,5 @@ TEST(StringUtil, CharacterEncodingConversion) EXPECT_EQ(UTF8ToSHIFTJIS("イルカ"), "\x83\x43\x83\x8b\x83\x4a"); // CP1252 - EXPECT_EQ(CP1252ToUTF8("hello \xa5"), "hello ¥"); + EXPECT_EQ(CP1252ToUTF8("hello \x81\x80\xa2\x9f"), "hello " + utf8_replacement_char + "€¢Ÿ"); }