From 8a3a338c0b41af801e3ba2a7947aaade864417ef Mon Sep 17 00:00:00 2001 From: abcboy101 <16735361+abcboy101@users.noreply.github.com> Date: Sun, 17 Mar 2024 21:34:13 -0700 Subject: [PATCH] Fix encodings for Gen 3/4/5 transfers (#4220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjust Gen 3/4 encodings to be consistent with Gen 5 Unicode encodings Gen 3 quotation marks are displayed differently based on the game language Implement how Pal Park handles invalid characters and corrupts certain accented characters Implement how Poké Transfer handles invalid characters Use U+25BA BLACK RIGHT-POINTING POINTER, since this character is used as the pointer in menus/etc., rather than as a bullet or generic shape --- .../Templates/Gen3/EncounterTrade3.cs | 20 ++- .../Templates/Gen3/XD/EncounterTrade3XD.cs | 25 +++- PKHeX.Core/PKM/PK3.cs | 20 ++- PKHeX.Core/PKM/PK4.cs | 1 + PKHeX.Core/PKM/Strings/StringConverter.cs | 14 +- PKHeX.Core/PKM/Strings/StringConverter3.cs | 139 +++++++++++++----- PKHeX.Core/PKM/Strings/StringConverter345.cs | 103 +++++++++++-- .../PKM/Strings/StringConverter4Util.cs | 28 ++-- PKHeX.Core/Saves/Storage/BulkStorage.cs | 2 +- .../Saves/Substructures/Gen3/SecretBase3.cs | 4 +- 10 files changed, 276 insertions(+), 80 deletions(-) diff --git a/PKHeX.Core/Legality/Encounters/Templates/Gen3/EncounterTrade3.cs b/PKHeX.Core/Legality/Encounters/Templates/Gen3/EncounterTrade3.cs index 17a0e1a30..71f94838d 100644 --- a/PKHeX.Core/Legality/Encounters/Templates/Gen3/EncounterTrade3.cs +++ b/PKHeX.Core/Legality/Encounters/Templates/Gen3/EncounterTrade3.cs @@ -163,14 +163,30 @@ public bool IsTrainerMatch(PKM pk, ReadOnlySpan trainer, int language) { if (Species == (int)Core.Species.Jynx && pk.Version == GameVersion.LG && language == (int)LanguageID.Italian) language = 2; - return language != 0 && (uint)language < TrainerNames.Length && trainer.SequenceEqual(TrainerNames[language]); + if (language == 0 || (uint)language >= TrainerNames.Length) + return false; + var name = TrainerNames[language]; + if (pk.Context == EntityContext.Gen3) + return trainer.SequenceEqual(name); + + Span tmp = stackalloc char[name.Length]; + StringConverter345.TransferGlyphs34(name, language, tmp); + return trainer.SequenceEqual(tmp); } public bool IsNicknameMatch(PKM pk, ReadOnlySpan nickname, int language) { if (Species == (int)Core.Species.Jynx && pk.Version == GameVersion.LG && language == (int)LanguageID.Italian) language = 2; - return language != 0 && (uint)language < Nicknames.Length && nickname.SequenceEqual(Nicknames[language]); + if (language == 0 || (uint)language >= TrainerNames.Length) + return false; + var name = Nicknames[language]; + if (pk.Context == EntityContext.Gen3) + return nickname.SequenceEqual(name); + + Span tmp = stackalloc char[name.Length]; + StringConverter345.TransferGlyphs34(name, language, tmp); + return nickname.SequenceEqual(tmp); } public string GetNickname(int language) => (uint)language < Nicknames.Length ? Nicknames[language] : Nicknames[0]; diff --git a/PKHeX.Core/Legality/Encounters/Templates/Gen3/XD/EncounterTrade3XD.cs b/PKHeX.Core/Legality/Encounters/Templates/Gen3/XD/EncounterTrade3XD.cs index 597f21474..75a3b459c 100644 --- a/PKHeX.Core/Legality/Encounters/Templates/Gen3/XD/EncounterTrade3XD.cs +++ b/PKHeX.Core/Legality/Encounters/Templates/Gen3/XD/EncounterTrade3XD.cs @@ -173,13 +173,34 @@ private bool IsMatchPartial(PKM pk) public bool IsCompatible(PIDType val, PKM pk) => val is PIDType.CXD; public PIDType GetSuggestedCorrelation() => PIDType.CXD; - public bool IsTrainerMatch(PKM pk, ReadOnlySpan trainer, int language) => (uint)language < TrainerNames.Length && trainer.SequenceEqual(TrainerNames[language]); + public bool IsTrainerMatch(PKM pk, ReadOnlySpan trainer, int language) + { + if ((uint)language >= TrainerNames.Length) + return false; + if (language == 0 || (uint)language >= TrainerNames.Length) + return false; + var name = TrainerNames[language]; + if (pk.Context == EntityContext.Gen3) + return trainer.SequenceEqual(name); + + Span tmp = stackalloc char[name.Length]; + StringConverter345.TransferGlyphs34(name, language, tmp); + return trainer.SequenceEqual(tmp); + } public bool IsNicknameMatch(PKM pk, ReadOnlySpan nickname, int language) { if (!IsFixedNickname) return true; - return nickname.SequenceEqual(GetNickname(language)); + if (language == 0 || (uint)language >= Nicknames.Length) + return false; + var name = Nicknames[language]; + if (pk.Context == EntityContext.Gen3) + return nickname.SequenceEqual(name); + + Span tmp = stackalloc char[name.Length]; + StringConverter345.TransferGlyphs34(name, language, tmp); + return nickname.SequenceEqual(tmp); } public string GetNickname(int language) => (uint)language < Nicknames.Length ? Nicknames[language] : Nicknames[0]; diff --git a/PKHeX.Core/PKM/PK3.cs b/PKHeX.Core/PKM/PK3.cs index d0aed0e04..9d7d2e7e7 100644 --- a/PKHeX.Core/PKM/PK3.cs +++ b/PKHeX.Core/PKM/PK3.cs @@ -48,8 +48,8 @@ public override PK3 Clone() public override ushort SID16 { get => ReadUInt16LittleEndian(Data.AsSpan(0x06)); set => WriteUInt16LittleEndian(Data.AsSpan(0x06), value); } public override string Nickname { - get => StringConverter3.GetString(NicknameTrash, Japanese); - set => StringConverter3.SetString(NicknameTrash, IsEgg ? EggNameJapanese : value, 10, Japanese, StringConverterOption.None); + get => StringConverter3.GetString(NicknameTrash, Language); + set => StringConverter3.SetString(NicknameTrash, IsEgg ? EggNameJapanese : value, 10, Language, StringConverterOption.None); } public override int Language { get => Data[0x12]; set => Data[0x12] = (byte)value; } public bool FlagIsBadEgg { get => (Data[0x13] & 1) != 0; set => Data[0x13] = (byte)((Data[0x13] & ~1) | (value ? 1 : 0)); } @@ -57,8 +57,8 @@ public override string Nickname public bool FlagIsEgg { get => (Data[0x13] & 4) != 0; set => Data[0x13] = (byte)((Data[0x13] & ~4) | (value ? 4 : 0)); } public override string OriginalTrainerName { - get => StringConverter3.GetString(OriginalTrainerTrash, Japanese); - set => StringConverter3.SetString(OriginalTrainerTrash, value, 7, Japanese, StringConverterOption.None); + get => StringConverter3.GetString(OriginalTrainerTrash, Language); + set => StringConverter3.SetString(OriginalTrainerTrash, value, 7, Language, StringConverterOption.None); } public override byte MarkingValue { get => (byte)SwapBits(Data[0x1B], 1, 2); set => Data[0x1B] = (byte)SwapBits(value, 1, 2); } public ushort Checksum { get => ReadUInt16LittleEndian(Data.AsSpan(0x1C)); set => WriteUInt16LittleEndian(Data.AsSpan(0x1C), value); } @@ -299,15 +299,19 @@ public PK4 ConvertToPK4() // Yay for reusing string buffers! The game allocates a buffer and reuses it when creating strings. // Trash from the {unknown source} is currently in buffer. Set it to the Nickname region. var trash = StringConverter345.GetTrashBytes(pk4.Language); - var nickTrash = pk4.NicknameTrash[4..]; // min of 1 char and terminator, ignore first 2. - trash.CopyTo(nickTrash); - pk4.Nickname = IsEgg ? SpeciesName.GetSpeciesNameGeneration(pk4.Species, pk4.Language, 4) : Nickname; + var nickTrash = pk4.NicknameTrash; + trash.CopyTo(nickTrash[4..]); // min of 1 char and terminator, ignore first 2. + + if (IsEgg) + pk4.Nickname = SpeciesName.GetSpeciesNameGeneration(pk4.Species, pk4.Language, 4); + else + StringConverter345.TransferGlyphs34(NicknameTrash, Language, Japanese ? 5 : 10, nickTrash); pk4.IsNicknamed = !IsEgg && IsNicknamed; // Trash from the current string (Nickname) is in our string buffer. Slap the OT name over-top. var destOT = pk4.OriginalTrainerTrash; nickTrash[..destOT.Length].CopyTo(destOT); - pk4.OriginalTrainerName = OriginalTrainerName; + StringConverter345.TransferGlyphs34(OriginalTrainerTrash, Language, Japanese ? 5 : 7, destOT); var item = (ushort)HeldItem; if (item != 0) diff --git a/PKHeX.Core/PKM/PK4.cs b/PKHeX.Core/PKM/PK4.cs index 4f2e6748e..ec9e098e6 100644 --- a/PKHeX.Core/PKM/PK4.cs +++ b/PKHeX.Core/PKM/PK4.cs @@ -380,6 +380,7 @@ public static void TransferTrash(ReadOnlySpan src, Span dest) { Span temp = stackalloc char[13]; var len = StringConverter4.LoadString(src, temp); + StringConverter345.TransferGlyphs45(temp[..len]); StringConverter5.SetString(dest, temp[..len], len); } } diff --git a/PKHeX.Core/PKM/Strings/StringConverter.cs b/PKHeX.Core/PKM/Strings/StringConverter.cs index c1a1bd2c2..9d16ceab4 100644 --- a/PKHeX.Core/PKM/Strings/StringConverter.cs +++ b/PKHeX.Core/PKM/Strings/StringConverter.cs @@ -14,14 +14,15 @@ public static class StringConverter /// Generation string format /// Encoding is Japanese /// Encoding is Big Endian + /// Language specific conversion /// Decoded string. - public static string GetString(ReadOnlySpan data, byte generation, bool jp, bool isBigEndian = false) => generation switch + public static string GetString(ReadOnlySpan data, byte generation, bool jp, bool isBigEndian = false, int language = 0) => generation switch { 3 when isBigEndian => StringConverter3GC.GetString(data), 4 when isBigEndian => StringConverter4GC.GetString(data), 1 or 2 => StringConverter12.GetString(data, jp), - 3 => StringConverter3.GetString(data, jp), + 3 => StringConverter3.GetString(data, language), 4 => StringConverter4.GetString(data), 5 => StringConverter5.GetString(data), 6 => StringConverter6.GetString(data), @@ -39,14 +40,15 @@ public static class StringConverter /// Generation string format /// Encoding is Japanese /// Encoding is Big Endian + /// Language specific conversion /// Decoded string. - public static int LoadString(ReadOnlySpan data, Span result, byte generation, bool jp, bool isBigEndian = false) => generation switch + public static int LoadString(ReadOnlySpan data, Span result, byte generation, bool jp, bool isBigEndian = false, int language = 0) => generation switch { 3 when isBigEndian => StringConverter3GC.LoadString(data, result), 4 when isBigEndian => StringConverter4GC.LoadString(data, result), 1 or 2 => StringConverter12.LoadString(data, result, jp), - 3 => StringConverter3.LoadString(data, result, jp), + 3 => StringConverter3.LoadString(data, result, language), 4 => StringConverter4.LoadString(data, result), 5 => StringConverter5.LoadString(data, result), 6 => StringConverter6.LoadString(data, result), @@ -66,7 +68,7 @@ public static class StringConverter /// Generation string format /// Encoding is Japanese /// Encoding is Big Endian - /// Language specific conversion (Chinese) + /// Language specific conversion /// Count of bytes written to the . public static int SetString(Span destBuffer, ReadOnlySpan value, int maxLength, StringConverterOption option, byte generation, bool jp, bool isBigEndian, int language = 0) => generation switch @@ -75,7 +77,7 @@ public static class StringConverter 4 when isBigEndian => StringConverter4GC.SetString(destBuffer, value, maxLength, option), 1 or 2 => StringConverter12.SetString(destBuffer, value, maxLength, jp, option), - 3 => StringConverter3.SetString(destBuffer, value, maxLength, jp, option), + 3 => StringConverter3.SetString(destBuffer, value, maxLength, language, option), 4 => StringConverter4.SetString(destBuffer, value, maxLength, option), 5 => StringConverter5.SetString(destBuffer, value, maxLength, option), 6 => StringConverter6.SetString(destBuffer, value, maxLength, option), diff --git a/PKHeX.Core/PKM/Strings/StringConverter3.cs b/PKHeX.Core/PKM/Strings/StringConverter3.cs index f2717610b..6efeacf79 100644 --- a/PKHeX.Core/PKM/Strings/StringConverter3.cs +++ b/PKHeX.Core/PKM/Strings/StringConverter3.cs @@ -7,55 +7,74 @@ namespace PKHeX.Core; /// public static class StringConverter3 { - private const byte TerminatorByte = 0xFF; + public const byte TerminatorByte = 0xFF; private const char Terminator = (char)TerminatorByte; private const char Apostrophe = '\''; // ’ private const byte ApostropheByte = 0xB4; + private const byte QuoteLeftByte = 0xB1; + private const byte QuoteRightByte = 0xB2; /// /// Converts a Generation 3 encoded value array to string. /// /// Byte array containing string data. - /// Value source is Japanese font. + /// Language specific conversion /// Decoded string. - public static string GetString(ReadOnlySpan data, bool jp) + public static string GetString(ReadOnlySpan data, int language) { Span result = stackalloc char[data.Length]; - int i = LoadString(data, result, jp); + int i = LoadString(data, result, language); return new string(result[..i]); } - /// + /// + /// Byte array containing string data. + /// Value source is Japanese font. + public static string GetString(ReadOnlySpan data, bool jp) => GetString(data, jp ? (int)LanguageID.Japanese : (int)LanguageID.English); + + /// + /// Converts a Generation 3 encoded value array to string. + /// /// Encoded data /// Decoded character result buffer - /// Data source is Japanese. + /// Language specific conversion /// Character count loaded. - public static int LoadString(ReadOnlySpan data, Span result, bool jp) + public static int LoadString(ReadOnlySpan data, Span result, int language) { - var table = jp ? G3_JP : G3_EN; + var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN; int i = 0; for (; i < data.Length; i++) { var value = data[i]; - var c = table[value]; // Convert to Unicode + var c = value switch { + QuoteLeftByte => GetQuoteLeft(language), + QuoteRightByte => GetQuoteRight(language), + _ => table[value], + }; // Convert to Unicode if (c == Terminator) // Stop if Terminator/Invalid break; + c = StringConverter.SanitizeChar(c); result[i] = c; } return i; } + /// + /// Encoded data + /// Decoded character result buffer + /// Value source is Japanese font. + public static int LoadString(ReadOnlySpan data, Span result, bool jp) => LoadString(data, result, jp ? (int)LanguageID.Japanese : (int)LanguageID.English); + /// /// Converts a string to a Generation 3 encoded value array. /// /// /// Decoded string. /// Maximum length of the input - /// String destination is Japanese font. + /// Language specific conversion /// Buffer pre-formatting option /// Encoded data. - public static int SetString(Span buffer, ReadOnlySpan value, int maxLength, bool jp, - StringConverterOption option = StringConverterOption.ClearFF) + public static int SetString(Span buffer, ReadOnlySpan value, int maxLength, int language, StringConverterOption option = StringConverterOption.ClearFF) { if (value.Length > maxLength) value = value[..maxLength]; // Hard cap @@ -65,15 +84,12 @@ public static int LoadString(ReadOnlySpan data, Span result, bool jp else if (option is StringConverterOption.ClearZero) buffer.Clear(); - var table = jp ? G3_JP : G3_EN; + var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN; int i = 0; for (; i < value.Length; i++) { - var chr = value[i]; - if (chr == Apostrophe) // ’ - return ApostropheByte; - var b = (byte)table.IndexOf(chr); - if (b == TerminatorByte) + var c = StringConverter.UnSanitizeChar5(value[i]); + if (!TryGetIndex(table, c, language, out var b)) break; buffer[i] = b; } @@ -84,31 +100,84 @@ public static int LoadString(ReadOnlySpan data, Span result, bool jp return count; } + public static int SetString(Span buffer, ReadOnlySpan value, int maxLength, bool jp, StringConverterOption option = StringConverterOption.ClearFF) => + SetString(buffer, value, maxLength, jp ? (int)LanguageID.Japanese : (int)LanguageID.English, option); + + private static bool TryGetIndex(in ReadOnlySpan dict, char c, int language, out byte result) + { + var index = dict.IndexOf(c); + if (index == -1 || c == '“') + return TryGetUserFriendlyRemap(c, language, out result); + result = (byte)index; + return index != TerminatorByte; + } + /// /// Decodes a character from a Generation 3 encoded value. /// /// Generation 4 decoded character. - /// Character destination is Japanese font. + /// Language specific conversion /// Generation 3 encoded value. - public static char GetG3Char(byte chr, bool jp) + public static char GetG3Char(byte chr, int language) { - var table = jp ? G3_JP : G3_EN; - return table[chr]; + var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN; + return chr switch + { + QuoteLeftByte => GetQuoteLeft(language), + QuoteRightByte => GetQuoteRight(language), + _ => table[chr], + }; } /// /// Encodes a character to a Generation 3 encoded value. /// /// Generation 4 decoded character. - /// Character destination is Japanese font. + /// Language specific conversion /// Generation 3 encoded value. - public static byte SetG3Char(char chr, bool jp) + public static byte SetG3Char(char chr, int language) { - if (chr == Apostrophe) - return ApostropheByte; - var table = jp ? G3_JP : G3_EN; - var index = table.IndexOf(chr); - return (byte)index; + var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN; + TryGetIndex(table, chr, language, out var b); + return b; + } + + // Quotation marks are displayed differently based on the Gen3 game language. + // Pal Park converts these to the appropriate ones based on the PKM language. + private static char GetQuoteLeft(int language) => language switch + { + (int)LanguageID.English or (int)LanguageID.Italian or (int)LanguageID.Spanish => '“', + (int)LanguageID.French => '«', + (int)LanguageID.German => '„', + _ => '『', // Invalid languages use JP quote + }; + + private static char GetQuoteRight(int language) => language switch + { + (int)LanguageID.English or (int)LanguageID.Italian or (int)LanguageID.Spanish => '”', + (int)LanguageID.French => '»', + (int)LanguageID.German => '“', + _ => '』', // Invalid languages use JP quote + }; + + /// + /// Tries to remap the user input to a valid character. + /// + private static bool TryGetUserFriendlyRemap(char c, int language, out byte result) + { + result = c switch + { + Apostrophe => ApostropheByte, + '“' => language != (int)LanguageID.German ? QuoteLeftByte : QuoteRightByte, + '”' => QuoteRightByte, + '«' => QuoteLeftByte, + '»' => QuoteRightByte, + '„' => QuoteLeftByte, + '『' => QuoteLeftByte, + '』' => QuoteRightByte, + _ => TerminatorByte, + }; + return result != TerminatorByte; } private static ReadOnlySpan G3_EN => @@ -123,11 +192,11 @@ public static byte SetG3Char(char chr, bool jp) 'ミ', 'ム', 'メ', 'モ', 'ヤ', 'ユ', 'ヨ', 'ラ', 'リ', 'ル', 'レ', 'ロ', 'ワ', 'ヲ', 'ン', 'ァ', // 7 'ィ', 'ゥ', 'ェ', 'ォ', 'ャ', 'ュ', 'ョ', 'ガ', 'ギ', 'グ', 'ゲ', 'ゴ', 'ザ', 'ジ', 'ズ', 'ゼ', // 8 'ゾ', 'ダ', 'ヂ', 'ヅ', 'デ', 'ド', 'バ', 'ビ', 'ブ', 'ベ', 'ボ', 'パ', 'ピ', 'プ', 'ペ', 'ポ', // 9 - 'ッ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '?', '.', '-', '・',// A - '⑬', '“', '”', '‘', '’', '♂', '♀', '$', ',', '⑧', '/', 'A', 'B', 'C', 'D', 'E', // B + 'ッ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '?', '.', '-', '･',// A + '⑬', '“', '”', '‘', '’', '⑭', '⑮', '$', ',', '⑧', '/', 'A', 'B', 'C', 'D', 'E', // B 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', // C 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', // D - 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '▶', // E + 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '►', // E ':', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F // Make the total length 256 so that any byte access is always within the array @@ -147,11 +216,11 @@ public static byte SetG3Char(char chr, bool jp) 'ィ', 'ゥ', 'ェ', 'ォ', 'ャ', 'ュ', 'ョ', 'ガ', 'ギ', 'グ', 'ゲ', 'ゴ', 'ザ', 'ジ', 'ズ', 'ゼ', // 8 'ゾ', 'ダ', 'ヂ', 'ヅ', 'デ', 'ド', 'バ', 'ビ', 'ブ', 'ベ', 'ボ', 'パ', 'ピ', 'プ', 'ペ', 'ポ', // 9 'ッ', '０', '１', '２', '３', '４', '５', '６', '７', '８', '９', '！', '？', '。', 'ー', '・', // A - '⋯', '『', '』', '「', '」', '♂', '♀', '$', '.', '⑧', '/', 'Ａ', 'Ｂ', 'Ｃ', 'Ｄ', 'Ｅ', // B + '…', '『', '』', '「', '」', '♂', '♀', '円', '．', '×', '／', 'Ａ', 'Ｂ', 'Ｃ', 'Ｄ', 'Ｅ', // B 'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', 'Ｕ', // C 'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ', 'ｂ', 'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', 'ｋ', // D - 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', '▶', // E - ':', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F + 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', '►', // E + '：', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F // Make the total length 256 so that any byte access is always within the array Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, diff --git a/PKHeX.Core/PKM/Strings/StringConverter345.cs b/PKHeX.Core/PKM/Strings/StringConverter345.cs index 10d888244..4761cc804 100644 --- a/PKHeX.Core/PKM/Strings/StringConverter345.cs +++ b/PKHeX.Core/PKM/Strings/StringConverter345.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using static System.Buffers.Binary.BinaryPrimitives; namespace PKHeX.Core; @@ -23,16 +24,99 @@ public static class StringConverter345 _ => [], // No Trash Bytes }; + /// + /// Remaps Gen3 Glyphs to Gen4 Glyphs. + /// + public static void TransferGlyphs34(ReadOnlySpan data, int language, int maxLength, Span dest) + { + Span result = stackalloc char[data.Length]; + int count = TransferGlyphs34(data, result, language, maxLength); + StringConverter4.SetString(dest, result[..count], maxLength, StringConverterOption.None); + } + + private static int TransferGlyphs34(ReadOnlySpan data, Span result, int language, int maxLength) + { + var i = 0; + for (; i < data.Length; i++) + { + var c = data[i]; + if (c == StringConverter3.TerminatorByte) + break; + + // If the encoded character is in the affected range, treat it as Japanese. + var lang = IsJapanese3(c) ? (int)LanguageID.Japanese : language; + var b = StringConverter3.GetG3Char(c, lang); + + // If an invalid character (0xF7-0xFE) is present, the nickname/OT is replaced with all question marks. + // Based on the Gen4 game's language: "？？？？？" in Japanese, "??????????" for nicknames/"???????" for OTs in EFIGS, "?????" in Korean + // Since we can't tell the language of the Gen4 game from the PKM alone, use the PKM language instead. + if (b == StringConverter3.TerminatorByte) + { + var question = language == (int)LanguageID.Japanese ? '？' : '?'; + result[..maxLength].Fill(question); + return maxLength; + } + + result[i] = b; + } + return i; + } + + /// + /// Remaps Gen3 Glyphs to Gen4 Glyphs. + /// + public static void TransferGlyphs34(ReadOnlySpan input, int language, Span result) + { + if (language == (int)LanguageID.Japanese) + { + input.CopyTo(result); + return; + } + for (var i = 0; i < input.Length; i++) + { + var b = input[i]; + + // If the encoded character is in the affected range, reinterpret it as Japanese. + var c = StringConverter3.SetG3Char(b, language); + result[i] = IsJapanese3(c) ? StringConverter3.GetG3Char(c, (int)LanguageID.Japanese) : b; + } + } + + // Pal Park always uses the Japanese character set for kana and '円' regardless of language. + // Only legitimately affects Spanish in-game trades and default player names, where Á/Í/Ú become い/コ/つ in Gen4. + private const byte JapaneseStartGlyph = 0x01; // 'あ' + private const byte JapaneseEndGlyph = 0xA0; // 'ッ' + private const byte JapaneseYenGlyph = 0xB7; // '円' + + private static bool IsJapanese3(byte glyph) => glyph is (>= JapaneseStartGlyph and <= JapaneseEndGlyph) or JapaneseYenGlyph; + + /// + /// Remaps Gen4 Glyphs to Gen5 Glyphs. + /// + /// Input characters to transfer in place + public static void TransferGlyphs45(Span input) + { + for (int i = 0; i < input.Length; i++) + { + if (IsInvalid45(input[i])) + input[i] = '?'; + } + } + + // These characters are converted to halfwidth question marks upon transfer to Gen5, in addition to the empty/invalid characters in Gen4. + // None of these are user-enterable. Note that halfwidth '&' transfers properly, despite it not being user-enterable either. + private static bool IsInvalid45(char c) => c == StringConverter4.Terminator || InvalidSearchVal45.Contains(c); + private static readonly SearchValues InvalidSearchVal45 = SearchValues.Create("$_ª°ºÂÃÅÆÊËÎÏÐÔÕØÛÝÞãåæðõøýþÿŒœŞş←↑→↓⑧⑨⑩⑪⑫⒅⒆⒇⒈⒉⒊⒋⒌⒍⒎⒏►♈♉♊♋♌♍♎♏円＆＿"); + /// /// Remaps Gen5 Glyphs to unicode codepoint. /// - /// Input characters to transfer in place - /// Remapped string - public static void TransferGlyphs56(Span buffer) + /// Input characters to transfer in place + public static void TransferGlyphs56(Span input) { - for (int i = 0; i < buffer.Length; i += 2) + for (int i = 0; i < input.Length; i += 2) { - var span = buffer[i..]; + var span = input[i..]; var c = ReadUInt16LittleEndian(span); if (IsPrivateUseChar(c)) WriteUInt16LittleEndian(span, GetMigratedPrivateChar(c)); @@ -42,13 +126,12 @@ public static void TransferGlyphs56(Span buffer) /// /// Remaps private use unicode codepoint back to Gen5 private use codepoint. /// - /// Input characters to transfer in place - /// Remapped string - public static void TransferGlyphs65(Span buffer) + /// Input characters to transfer in place + public static void TransferGlyphs65(Span input) { - for (int i = 0; i < buffer.Length; i += 2) + for (int i = 0; i < input.Length; i += 2) { - var span = buffer[i..]; + var span = input[i..]; var c = ReadUInt16LittleEndian(span); if (IsPrivateUseCharUnicode(c)) WriteUInt16LittleEndian(span, GetUnmigratedPrivateChar(c)); diff --git a/PKHeX.Core/PKM/Strings/StringConverter4Util.cs b/PKHeX.Core/PKM/Strings/StringConverter4Util.cs index 63efdc725..f273b239f 100644 --- a/PKHeX.Core/PKM/Strings/StringConverter4Util.cs +++ b/PKHeX.Core/PKM/Strings/StringConverter4Util.cs @@ -63,7 +63,7 @@ public static void StripDiacriticsFR4(Span input) #region Conversion Data - private const int TableINTEnd = 0x01DE; // Ignore chars that can't be entered. + private const int TableINTEnd = 0x01EC; private const int TableKORStart = 0x400; private const int TableKOREnd = 0xD65; @@ -71,7 +71,6 @@ public static void StripDiacriticsFR4(Span input) private const ushort SaveInvalidAs = 0x1AC; // '?' private const char NUL = (char)StringConverter4.Terminator; private const char EMP = NUL; // Empty, not available on keyboard. - private const char YEN = NUL; // Yen, not available on keyboard. public static ReadOnlySpan TableINT => [ @@ -89,22 +88,23 @@ public static void StripDiacriticsFR4(Span input) 'Ｅ', 'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', // 0B0-0BF 'Ｕ', 'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ', 'ｂ', 'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', // 0C0-0CF 'ｋ', 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', // 0D0-0DF - EMP, '！', '？', '、', '。', '⋯', '・', '／', '「', '」', '『', '』', '（', '）', '⑭', '⑮', // 0E0-0EF - '＋', 'ー', '⑧', '⑨', '＝', '～', '：', '；', '．', '，', '♠', '♣', '♥', '♦', '★', '◎', // 0F0-0FF - '○', '□', '△', '◇', '＠', '♪', '％', '☀', '☁', '☂', '☃', '①', '②', '③', '☊', '⑤', // 100-10F - '⑥', '⑦', YEN, '♈', '♉', '♊', '♋', '♌', '♍', '♎', '♏', '←', '↑', '↓', '→', '‣', // 110-11F + EMP, '！', '？', '、', '。', '…', '・', '／', '「', '」', '『', '』', '（', '）', '♂', '♀', // 0E0-0EF + '＋', 'ー', '×', '÷', '＝', '～', '：', '；', '．', '，', '♠', '♣', '♥', '♦', '★', '◎', // 0F0-0FF + '○', '□', '△', '◇', '＠', '♪', '％', '☀', '☁', '☂', '☃', '①', '②', '③', '④', '⑤', // 100-10F + '⑥', '⑦', '円', '♈', '♉', '♊', '♋', '♌', '♍', '♎', '♏', '←', '↑', '↓', '→', '►', // 110-11F '＆', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', // 120-12F 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', // 130-13F 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', // 140-14F 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'À', // 150-15F 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 'Ð', // 160-16F - 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×', 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 'à', // 170-17F + 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '⑧', 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 'à', // 170-17F 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 'ð', // 180-18F - 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ', 'Œ', // 190-19F - 'œ', 'Ş', 'ş', 'ª', 'º', '¹', '²', '³', '$', '¡', '¿', '!', '?', ',', '.', '⑬', // 1A0-1AF - '･', '/', '‘', '’', '“', '”', '„', '《', '》', '(', ')', '♂', '♀', '+', '-', '*', // 1B0-1BF + 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '⑨', 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ', 'Œ', // 190-19F + 'œ', 'Ş', 'ş', 'ª', 'º', '⑩', '⑪', '⑫', '$', '¡', '¿', '!', '?', ',', '.', '⑬', // 1A0-1AF + '･', '/', '‘', '’', '“', '”', '„', '«', '»', '(', ')', '⑭', '⑮', '+', '-', '*', // 1B0-1BF '#', '=', '&', '~', ':', ';', '⑯', '⑰', '⑱', '⑲', '⑳', '⑴', '⑵', '⑶', '⑷', '⑸', // 1C0-1CF - '@', '⑹', '%', '⑺', '⑻', '⑼', '⑽', '⑾', '⑿', '⒀', '⒁', '⒂', '⒃', '⒄', ' ', // 1D0-1DE* + '@', '⑹', '%', '⑺', '⑻', '⑼', '⑽', '⑾', '⑿', '⒀', '⒁', '⒂', '⒃', '⒄', ' ', '⒅', // 1D0-1DF + '⒆', '⒇', '⒈', '⒉', '⒊', '⒋', '⒌', '⒍', '°', '_', '＿', '⒎', '⒏', // 1E0-1EC* ]; public static ReadOnlySpan TableKOR => @@ -258,9 +258,9 @@ public static void StripDiacriticsFR4(Span input) '흩', '희', '흰', '흴', '흼', '흽', '힁', '히', '힉', '힌', '힐', '힘', '힙', '힛', '힝', NUL, // D20-D2F // 33 Jamo characters - NUL, 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', // D30-D3F - 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅭ', 'ᅮ', 'ᅲ', // D40-D4F - 'ᅳ', 'ᅵ', NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, // D50-D5F + NUL, 'ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', // D30-D3F + 'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ', 'ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅛ', 'ㅜ', 'ㅠ', // D40-D4F + 'ㅡ', 'ㅣ', NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, // D50-D5F // 5 additional syllables NUL, '뢔', '쌰', '쎼', '쓔', '쬬', // D60-D65 diff --git a/PKHeX.Core/Saves/Storage/BulkStorage.cs b/PKHeX.Core/Saves/Storage/BulkStorage.cs index 22ed5d709..77b572d4e 100644 --- a/PKHeX.Core/Saves/Storage/BulkStorage.cs +++ b/PKHeX.Core/Saves/Storage/BulkStorage.cs @@ -53,7 +53,7 @@ protected BulkStorage(byte[] data, Type t, int start, int slotsPerBox = 30) : ba public sealed override int GetPartyOffset(int slot) => int.MinValue; public override string GetString(ReadOnlySpan data) - => StringConverter.GetString(data, Generation, blank.Japanese, IsBigEndian); + => StringConverter.GetString(data, Generation, blank.Japanese, IsBigEndian, Language); public override int SetString(Span destBuffer, ReadOnlySpan value, int maxLength, StringConverterOption option) => StringConverter.SetString(destBuffer, value, maxLength, option: option, generation: Generation, jp: blank.Japanese, isBigEndian: IsBigEndian, language: Language); diff --git a/PKHeX.Core/Saves/Substructures/Gen3/SecretBase3.cs b/PKHeX.Core/Saves/Substructures/Gen3/SecretBase3.cs index e0bea6a1c..fecd75501 100644 --- a/PKHeX.Core/Saves/Substructures/Gen3/SecretBase3.cs +++ b/PKHeX.Core/Saves/Substructures/Gen3/SecretBase3.cs @@ -31,8 +31,8 @@ public int RegistryStatus public string OriginalTrainerName { - get => StringConverter3.GetString(Data.Slice(2, 7), Japanese); - set => StringConverter3.SetString(Data.Slice(2, 7), value, 7, Japanese, StringConverterOption.ClearFF); + get => StringConverter3.GetString(Data.Slice(2, 7), Language); + set => StringConverter3.SetString(Data.Slice(2, 7), value, 7, Language, StringConverterOption.ClearFF); } public uint OT_ID