Leahnaya/PKHeX
1
0
Fork 0
mirror of https://github.com/kwsch/PKHeX.git synced 2026-09-13 11:25:19 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix encodings for Gen 3/4/5 transfers (#4220)

Browse Source 
Adjust Gen 3/4 encodings to be consistent with Gen 5 Unicode encodings
Gen 3 quotation marks are displayed differently based on the game language
Implement how Pal Park handles invalid characters and corrupts certain accented characters
Implement how Poké Transfer handles invalid characters
Use U+25BA BLACK RIGHT-POINTING POINTER, since this character is used as the pointer in menus/etc., rather than as a bullet or generic shape
This commit is contained in:
abcboy101
2024-03-17 21:34:13 -07:00
committed by GitHub
parent 2b6f153831
commit 8a3a338c0b
10 changed files with 276 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
PKHeX.Core/Legality/Encounters/Templates/Gen3/EncounterTrade3.cs
View File

@@ -163,14 +163,30 @@ public bool IsTrainerMatch(PKM pk, ReadOnlySpan<char> trainer, int language)
{
if (Species == (int)Core.Species.Jynx && pk.Version == GameVersion.LG && language == (int)LanguageID.Italian)
language = 2;
return language != 0 && (uint)language < TrainerNames.Length && trainer.SequenceEqual(TrainerNames[language]);
if (language == 0 || (uint)language >= TrainerNames.Length)
return false;
var name = TrainerNames[language];
if (pk.Context == EntityContext.Gen3)
return trainer.SequenceEqual(name);
Span<char> tmp = stackalloc char[name.Length];
StringConverter345.TransferGlyphs34(name, language, tmp);
return trainer.SequenceEqual(tmp);
}
public bool IsNicknameMatch(PKM pk, ReadOnlySpan<char> nickname, int language)
{
if (Species == (int)Core.Species.Jynx && pk.Version == GameVersion.LG && language == (int)LanguageID.Italian)
language = 2;
return language != 0 && (uint)language < Nicknames.Length && nickname.SequenceEqual(Nicknames[language]);
if (language == 0 || (uint)language >= TrainerNames.Length)
return false;
var name = Nicknames[language];
if (pk.Context == EntityContext.Gen3)
return nickname.SequenceEqual(name);
Span<char> tmp = stackalloc char[name.Length];
StringConverter345.TransferGlyphs34(name, language, tmp);
return nickname.SequenceEqual(tmp);
}
public string GetNickname(int language) => (uint)language < Nicknames.Length ? Nicknames[language] : Nicknames[0];

25
PKHeX.Core/Legality/Encounters/Templates/Gen3/XD/EncounterTrade3XD.cs
View File

@@ -173,13 +173,34 @@ private bool IsMatchPartial(PKM pk)
public bool IsCompatible(PIDType val, PKM pk) => val is PIDType.CXD;
public PIDType GetSuggestedCorrelation() => PIDType.CXD;
public bool IsTrainerMatch(PKM pk, ReadOnlySpan<char> trainer, int language) => (uint)language < TrainerNames.Length && trainer.SequenceEqual(TrainerNames[language]);
public bool IsTrainerMatch(PKM pk, ReadOnlySpan<char> trainer, int language)
{
if ((uint)language >= TrainerNames.Length)
return false;
if (language == 0 || (uint)language >= TrainerNames.Length)
return false;
var name = TrainerNames[language];
if (pk.Context == EntityContext.Gen3)
return trainer.SequenceEqual(name);
Span<char> tmp = stackalloc char[name.Length];
StringConverter345.TransferGlyphs34(name, language, tmp);
return trainer.SequenceEqual(tmp);
}
public bool IsNicknameMatch(PKM pk, ReadOnlySpan<char> nickname, int language)
{
if (!IsFixedNickname)
return true;
return nickname.SequenceEqual(GetNickname(language));
if (language == 0 || (uint)language >= Nicknames.Length)
return false;
var name = Nicknames[language];
if (pk.Context == EntityContext.Gen3)
return nickname.SequenceEqual(name);
Span<char> tmp = stackalloc char[name.Length];
StringConverter345.TransferGlyphs34(name, language, tmp);
return nickname.SequenceEqual(tmp);
}
public string GetNickname(int language) => (uint)language < Nicknames.Length ? Nicknames[language] : Nicknames[0];

20
PKHeX.Core/PKM/PK3.cs
View File

@@ -48,8 +48,8 @@ public override PK3 Clone()
public override ushort SID16 { get => ReadUInt16LittleEndian(Data.AsSpan(0x06)); set => WriteUInt16LittleEndian(Data.AsSpan(0x06), value); }
public override string Nickname
{
get => StringConverter3.GetString(NicknameTrash, Japanese);
set => StringConverter3.SetString(NicknameTrash, IsEgg ? EggNameJapanese : value, 10, Japanese, StringConverterOption.None);
get => StringConverter3.GetString(NicknameTrash, Language);
set => StringConverter3.SetString(NicknameTrash, IsEgg ? EggNameJapanese : value, 10, Language, StringConverterOption.None);
}
public override int Language { get => Data[0x12]; set => Data[0x12] = (byte)value; }
public bool FlagIsBadEgg { get => (Data[0x13] & 1) != 0; set => Data[0x13] = (byte)((Data[0x13] & ~1) | (value ? 1 : 0)); }
@@ -57,8 +57,8 @@ public override string Nickname
public bool FlagIsEgg { get => (Data[0x13] & 4) != 0; set => Data[0x13] = (byte)((Data[0x13] & ~4) | (value ? 4 : 0)); }
public override string OriginalTrainerName
{
get => StringConverter3.GetString(OriginalTrainerTrash, Japanese);
set => StringConverter3.SetString(OriginalTrainerTrash, value, 7, Japanese, StringConverterOption.None);
get => StringConverter3.GetString(OriginalTrainerTrash, Language);
set => StringConverter3.SetString(OriginalTrainerTrash, value, 7, Language, StringConverterOption.None);
}
public override byte MarkingValue { get => (byte)SwapBits(Data[0x1B], 1, 2); set => Data[0x1B] = (byte)SwapBits(value, 1, 2); }
public ushort Checksum { get => ReadUInt16LittleEndian(Data.AsSpan(0x1C)); set => WriteUInt16LittleEndian(Data.AsSpan(0x1C), value); }
@@ -299,15 +299,19 @@ public PK4 ConvertToPK4()
// Yay for reusing string buffers! The game allocates a buffer and reuses it when creating strings.
// Trash from the {unknown source} is currently in buffer. Set it to the Nickname region.
var trash = StringConverter345.GetTrashBytes(pk4.Language);
var nickTrash = pk4.NicknameTrash[4..]; // min of 1 char and terminator, ignore first 2.
trash.CopyTo(nickTrash);
pk4.Nickname = IsEgg ? SpeciesName.GetSpeciesNameGeneration(pk4.Species, pk4.Language, 4) : Nickname;
var nickTrash = pk4.NicknameTrash;
trash.CopyTo(nickTrash[4..]); // min of 1 char and terminator, ignore first 2.
if (IsEgg)
pk4.Nickname = SpeciesName.GetSpeciesNameGeneration(pk4.Species, pk4.Language, 4);
else
StringConverter345.TransferGlyphs34(NicknameTrash, Language, Japanese ? 5 : 10, nickTrash);
pk4.IsNicknamed = !IsEgg && IsNicknamed;
// Trash from the current string (Nickname) is in our string buffer. Slap the OT name over-top.
var destOT = pk4.OriginalTrainerTrash;
nickTrash[..destOT.Length].CopyTo(destOT);
pk4.OriginalTrainerName = OriginalTrainerName;
StringConverter345.TransferGlyphs34(OriginalTrainerTrash, Language, Japanese ? 5 : 7, destOT);
var item = (ushort)HeldItem;
if (item != 0)

1
PKHeX.Core/PKM/PK4.cs
View File

@@ -380,6 +380,7 @@ public static void TransferTrash(ReadOnlySpan<byte> src, Span<byte> dest)
{
Span<char> temp = stackalloc char[13];
var len = StringConverter4.LoadString(src, temp);
StringConverter345.TransferGlyphs45(temp[..len]);
StringConverter5.SetString(dest, temp[..len], len);
}
}

14
PKHeX.Core/PKM/Strings/StringConverter.cs
View File

@@ -14,14 +14,15 @@ public static class StringConverter
/// <param name="generation">Generation string format</param>
/// <param name="jp">Encoding is Japanese</param>
/// <param name="isBigEndian">Encoding is Big Endian</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Decoded string.</returns>
public static string GetString(ReadOnlySpan<byte> data, byte generation, bool jp, bool isBigEndian = false) => generation switch
public static string GetString(ReadOnlySpan<byte> data, byte generation, bool jp, bool isBigEndian = false, int language = 0) => generation switch
{
3 when isBigEndian => StringConverter3GC.GetString(data),
4 when isBigEndian => StringConverter4GC.GetString(data),
1 or 2 => StringConverter12.GetString(data, jp),
3 => StringConverter3.GetString(data, jp),
3 => StringConverter3.GetString(data, language),
4 => StringConverter4.GetString(data),
5 => StringConverter5.GetString(data),
6 => StringConverter6.GetString(data),
@@ -39,14 +40,15 @@ public static class StringConverter
/// <param name="generation">Generation string format</param>
/// <param name="jp">Encoding is Japanese</param>
/// <param name="isBigEndian">Encoding is Big Endian</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Decoded string.</returns>
public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, byte generation, bool jp, bool isBigEndian = false) => generation switch
public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, byte generation, bool jp, bool isBigEndian = false, int language = 0) => generation switch
{
3 when isBigEndian => StringConverter3GC.LoadString(data, result),
4 when isBigEndian => StringConverter4GC.LoadString(data, result),
1 or 2 => StringConverter12.LoadString(data, result, jp),
3 => StringConverter3.LoadString(data, result, jp),
3 => StringConverter3.LoadString(data, result, language),
4 => StringConverter4.LoadString(data, result),
5 => StringConverter5.LoadString(data, result),
6 => StringConverter6.LoadString(data, result),
@@ -66,7 +68,7 @@ public static class StringConverter
/// <param name="generation">Generation string format</param>
/// <param name="jp">Encoding is Japanese</param>
/// <param name="isBigEndian">Encoding is Big Endian</param>
/// <param name="language">Language specific conversion (Chinese)</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Count of bytes written to the <see cref="destBuffer"/>.</returns>
public static int SetString(Span<byte> destBuffer, ReadOnlySpan<char> value, int maxLength, StringConverterOption option,
byte generation, bool jp, bool isBigEndian, int language = 0) => generation switch
@@ -75,7 +77,7 @@ public static class StringConverter
4 when isBigEndian => StringConverter4GC.SetString(destBuffer, value, maxLength, option),
1 or 2 => StringConverter12.SetString(destBuffer, value, maxLength, jp, option),
3 => StringConverter3.SetString(destBuffer, value, maxLength, jp, option),
3 => StringConverter3.SetString(destBuffer, value, maxLength, language, option),
4 => StringConverter4.SetString(destBuffer, value, maxLength, option),
5 => StringConverter5.SetString(destBuffer, value, maxLength, option),
6 => StringConverter6.SetString(destBuffer, value, maxLength, option),

139
PKHeX.Core/PKM/Strings/StringConverter3.cs
View File

@@ -7,55 +7,74 @@ namespace PKHeX.Core;
/// </summary>
public static class StringConverter3
{
private const byte TerminatorByte = 0xFF;
public const byte TerminatorByte = 0xFF;
private const char Terminator = (char)TerminatorByte;
private const char Apostrophe = '\''; // ’
private const byte ApostropheByte = 0xB4;
private const byte QuoteLeftByte = 0xB1;
private const byte QuoteRightByte = 0xB2;
/// <summary>
/// Converts a Generation 3 encoded value array to string.
/// </summary>
/// <param name="data">Byte array containing string data.</param>
/// <param name="jp">Value source is Japanese font.</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Decoded string.</returns>
public static string GetString(ReadOnlySpan<byte> data, bool jp)
public static string GetString(ReadOnlySpan<byte> data, int language)
{
Span<char> result = stackalloc char[data.Length];
int i = LoadString(data, result, jp);
int i = LoadString(data, result, language);
return new string(result[..i]);
}
/// <inheritdoc cref="GetString(ReadOnlySpan{byte},bool)"/>
/// <inheritdoc cref="GetString(ReadOnlySpan{byte},int)"/>
/// <param name="data">Byte array containing string data.</param>
/// <param name="jp">Value source is Japanese font.</param>
public static string GetString(ReadOnlySpan<byte> data, bool jp) => GetString(data, jp ? (int)LanguageID.Japanese : (int)LanguageID.English);
/// <summary>
/// Converts a Generation 3 encoded value array to string.
/// </summary>
/// <param name="data">Encoded data</param>
/// <param name="result">Decoded character result buffer</param>
/// <param name="jp">Data source is Japanese.</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Character count loaded.</returns>
public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, bool jp)
public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, int language)
{
var table = jp ? G3_JP : G3_EN;
var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN;
int i = 0;
for (; i < data.Length; i++)
{
var value = data[i];
var c = table[value]; // Convert to Unicode
var c = value switch {
QuoteLeftByte => GetQuoteLeft(language),
QuoteRightByte => GetQuoteRight(language),
_ => table[value],
}; // Convert to Unicode
if (c == Terminator) // Stop if Terminator/Invalid
break;
c = StringConverter.SanitizeChar(c);
result[i] = c;
}
return i;
}
/// <inheritdoc cref="LoadString(ReadOnlySpan{byte},Span{char},int)"/>
/// <param name="data">Encoded data</param>
/// <param name="result">Decoded character result buffer</param>
/// <param name="jp">Value source is Japanese font.</param>
public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, bool jp) => LoadString(data, result, jp ? (int)LanguageID.Japanese : (int)LanguageID.English);
/// <summary>
/// Converts a string to a Generation 3 encoded value array.
/// </summary>
/// <param name="buffer"></param>
/// <param name="value">Decoded string.</param>
/// <param name="maxLength">Maximum length of the input <see cref="value"/></param>
/// <param name="jp">String destination is Japanese font.</param>
/// <param name="language">Language specific conversion</param>
/// <param name="option">Buffer pre-formatting option</param>
/// <returns>Encoded data.</returns>
public static int SetString(Span<byte> buffer, ReadOnlySpan<char> value, int maxLength, bool jp,
StringConverterOption option = StringConverterOption.ClearFF)
public static int SetString(Span<byte> buffer, ReadOnlySpan<char> value, int maxLength, int language, StringConverterOption option = StringConverterOption.ClearFF)
{
if (value.Length > maxLength)
value = value[..maxLength]; // Hard cap
@@ -65,15 +84,12 @@ public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, bool jp
else if (option is StringConverterOption.ClearZero)
buffer.Clear();
var table = jp ? G3_JP : G3_EN;
var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN;
int i = 0;
for (; i < value.Length; i++)
{
var chr = value[i];
if (chr == Apostrophe) // ’
return ApostropheByte;
var b = (byte)table.IndexOf(chr);
if (b == TerminatorByte)
var c = StringConverter.UnSanitizeChar5(value[i]);
if (!TryGetIndex(table, c, language, out var b))
break;
buffer[i] = b;
}
@@ -84,31 +100,84 @@ public static int LoadString(ReadOnlySpan<byte> data, Span<char> result, bool jp
return count;
}
public static int SetString(Span<byte> buffer, ReadOnlySpan<char> value, int maxLength, bool jp, StringConverterOption option = StringConverterOption.ClearFF) =>
SetString(buffer, value, maxLength, jp ? (int)LanguageID.Japanese : (int)LanguageID.English, option);
private static bool TryGetIndex(in ReadOnlySpan<char> dict, char c, int language, out byte result)
{
var index = dict.IndexOf(c);
if (index == -1 || c == '“')
return TryGetUserFriendlyRemap(c, language, out result);
result = (byte)index;
return index != TerminatorByte;
}
/// <summary>
/// Decodes a character from a Generation 3 encoded value.
/// </summary>
/// <param name="chr">Generation 4 decoded character.</param>
/// <param name="jp">Character destination is Japanese font.</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Generation 3 encoded value.</returns>
public static char GetG3Char(byte chr, bool jp)
public static char GetG3Char(byte chr, int language)
{
var table = jp ? G3_JP : G3_EN;
return table[chr];
var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN;
return chr switch
{
QuoteLeftByte => GetQuoteLeft(language),
QuoteRightByte => GetQuoteRight(language),
_ => table[chr],
};
}
/// <summary>
/// Encodes a character to a Generation 3 encoded value.
/// </summary>
/// <param name="chr">Generation 4 decoded character.</param>
/// <param name="jp">Character destination is Japanese font.</param>
/// <param name="language">Language specific conversion</param>
/// <returns>Generation 3 encoded value.</returns>
public static byte SetG3Char(char chr, bool jp)
public static byte SetG3Char(char chr, int language)
{
if (chr == Apostrophe)
return ApostropheByte;
var table = jp ? G3_JP : G3_EN;
var index = table.IndexOf(chr);
return (byte)index;
var table = (language == (int)LanguageID.Japanese) ? G3_JP : G3_EN;
TryGetIndex(table, chr, language, out var b);
return b;
}
// Quotation marks are displayed differently based on the Gen3 game language.
// Pal Park converts these to the appropriate ones based on the PKM language.
private static char GetQuoteLeft(int language) => language switch
{
(int)LanguageID.English or (int)LanguageID.Italian or (int)LanguageID.Spanish => '“',
(int)LanguageID.French => '«',
(int)LanguageID.German => '„',
_ => '『', // Invalid languages use JP quote
};
private static char GetQuoteRight(int language) => language switch
{
(int)LanguageID.English or (int)LanguageID.Italian or (int)LanguageID.Spanish => '”',
(int)LanguageID.French => '»',
(int)LanguageID.German => '“',
_ => '』', // Invalid languages use JP quote
};
/// <summary>
/// Tries to remap the user input to a valid character.
/// </summary>
private static bool TryGetUserFriendlyRemap(char c, int language, out byte result)
{
result = c switch
{
Apostrophe => ApostropheByte,
'“' => language != (int)LanguageID.German ? QuoteLeftByte : QuoteRightByte,
'”' => QuoteRightByte,
'«' => QuoteLeftByte,
'»' => QuoteRightByte,
'„' => QuoteLeftByte,
'『' => QuoteLeftByte,
'』' => QuoteRightByte,
_ => TerminatorByte,
};
return result != TerminatorByte;
}
private static ReadOnlySpan<char> G3_EN =>
@@ -123,11 +192,11 @@ public static byte SetG3Char(char chr, bool jp)
'ミ', 'ム', 'メ', 'モ', 'ヤ', 'ユ', 'ヨ', 'ラ', 'リ', 'ル', 'レ', 'ロ', 'ワ', 'ヲ', 'ン', 'ァ', // 7
'ィ', 'ゥ', 'ェ', 'ォ', 'ャ', 'ュ', 'ョ', 'ガ', 'ギ', 'グ', 'ゲ', 'ゴ', 'ザ', 'ジ', 'ズ', 'ゼ', // 8
'ゾ', 'ダ', 'ヂ', 'ヅ', 'デ', 'ド', 'バ', 'ビ', 'ブ', 'ベ', 'ボ', 'パ', 'ピ', 'プ', 'ペ', 'ポ', // 9
'ッ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '?', '.', '-', '・',// A
'⑬', '“', '”', '‘', '’', '♂', '♀', '$', ',', '⑧', '/', 'A', 'B', 'C', 'D', 'E', // B
'ッ', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '?', '.', '-', '･',// A
'⑬', '“', '”', '‘', '’', '⑭', '⑮', '$', ',', '⑧', '/', 'A', 'B', 'C', 'D', 'E', // B
'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', // C
'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', // D
'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '▶', // E
'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '►', // E
':', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F
// Make the total length 256 so that any byte access is always within the array
@@ -147,11 +216,11 @@ public static byte SetG3Char(char chr, bool jp)
'ィ', 'ゥ', 'ェ', 'ォ', 'ャ', 'ュ', 'ョ', 'ガ', 'ギ', 'グ', 'ゲ', 'ゴ', 'ザ', 'ジ', 'ズ', 'ゼ', // 8
'ゾ', 'ダ', 'ヂ', 'ヅ', 'デ', 'ド', 'バ', 'ビ', 'ブ', 'ベ', 'ボ', 'パ', 'ピ', 'プ', 'ペ', 'ポ', // 9
'ッ', '０', '１', '２', '３', '４', '５', '６', '７', '８', '９', '！', '？', '。', 'ー', '・', // A
'⋯', '『', '』', '「', '」', '♂', '♀', '$', '.', '⑧', '/', 'Ａ', 'Ｂ', 'Ｃ', 'Ｄ', 'Ｅ', // B
'…', '『', '』', '「', '」', '♂', '♀', '円', '．', '×', '／', 'Ａ', 'Ｂ', 'Ｃ', 'Ｄ', 'Ｅ', // B
'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', 'Ｕ', // C
'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ', 'ｂ', 'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', 'ｋ', // D
'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', '▶', // E
':', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F
'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', '►', // E
'：', 'Ä', 'Ö', 'Ü', 'ä', 'ö', 'ü', // F
// Make the total length 256 so that any byte access is always within the array
Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator, Terminator,

103
PKHeX.Core/PKM/Strings/StringConverter345.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Buffers;
using static System.Buffers.Binary.BinaryPrimitives;
namespace PKHeX.Core;
@@ -23,16 +24,99 @@ public static class StringConverter345
_ => [], // No Trash Bytes
};
/// <summary>
/// Remaps Gen3 Glyphs to Gen4 Glyphs.
/// </summary>
public static void TransferGlyphs34(ReadOnlySpan<byte> data, int language, int maxLength, Span<byte> dest)
{
Span<char> result = stackalloc char[data.Length];
int count = TransferGlyphs34(data, result, language, maxLength);
StringConverter4.SetString(dest, result[..count], maxLength, StringConverterOption.None);
}
private static int TransferGlyphs34(ReadOnlySpan<byte> data, Span<char> result, int language, int maxLength)
{
var i = 0;
for (; i < data.Length; i++)
{
var c = data[i];
if (c == StringConverter3.TerminatorByte)
break;
// If the encoded character is in the affected range, treat it as Japanese.
var lang = IsJapanese3(c) ? (int)LanguageID.Japanese : language;
var b = StringConverter3.GetG3Char(c, lang);
// If an invalid character (0xF7-0xFE) is present, the nickname/OT is replaced with all question marks.
// Based on the Gen4 game's language: "？？？？？" in Japanese, "??????????" for nicknames/"???????" for OTs in EFIGS, "?????" in Korean
// Since we can't tell the language of the Gen4 game from the PKM alone, use the PKM language instead.
if (b == StringConverter3.TerminatorByte)
{
var question = language == (int)LanguageID.Japanese ? '？' : '?';
result[..maxLength].Fill(question);
return maxLength;
}
result[i] = b;
}
return i;
}
/// <summary>
/// Remaps Gen3 Glyphs to Gen4 Glyphs.
/// </summary>
public static void TransferGlyphs34(ReadOnlySpan<char> input, int language, Span<char> result)
{
if (language == (int)LanguageID.Japanese)
{
input.CopyTo(result);
return;
}
for (var i = 0; i < input.Length; i++)
{
var b = input[i];
// If the encoded character is in the affected range, reinterpret it as Japanese.
var c = StringConverter3.SetG3Char(b, language);
result[i] = IsJapanese3(c) ? StringConverter3.GetG3Char(c, (int)LanguageID.Japanese) : b;
}
}
// Pal Park always uses the Japanese character set for kana and '円' regardless of language.
// Only legitimately affects Spanish in-game trades and default player names, where Á/Í/Ú become い/コ/つ in Gen4.
private const byte JapaneseStartGlyph = 0x01; // 'あ'
private const byte JapaneseEndGlyph = 0xA0; // 'ッ'
private const byte JapaneseYenGlyph = 0xB7; // '円'
private static bool IsJapanese3(byte glyph) => glyph is (>= JapaneseStartGlyph and <= JapaneseEndGlyph) or JapaneseYenGlyph;
/// <summary>
/// Remaps Gen4 Glyphs to Gen5 Glyphs.
/// </summary>
/// <param name="input">Input characters to transfer in place</param>
public static void TransferGlyphs45(Span<char> input)
{
for (int i = 0; i < input.Length; i++)
{
if (IsInvalid45(input[i]))
input[i] = '?';
}
}
// These characters are converted to halfwidth question marks upon transfer to Gen5, in addition to the empty/invalid characters in Gen4.
// None of these are user-enterable. Note that halfwidth '&' transfers properly, despite it not being user-enterable either.
private static bool IsInvalid45(char c) => c == StringConverter4.Terminator || InvalidSearchVal45.Contains(c);
private static readonly SearchValues<char> InvalidSearchVal45 = SearchValues.Create("$_ª°ºÂÃÅÆÊËÎÏÐÔÕØÛÝÞãåæðõøýþÿŒœŞş←↑→↓⑧⑨⑩⑪⑫⒅⒆⒇⒈⒉⒊⒋⒌⒍⒎⒏►♈♉♊♋♌♍♎♏円＆＿");
/// <summary>
/// Remaps Gen5 Glyphs to unicode codepoint.
/// </summary>
/// <param name="buffer">Input characters to transfer in place</param>
/// <returns>Remapped string</returns>
public static void TransferGlyphs56(Span<byte> buffer)
/// <param name="input">Input characters to transfer in place</param>
public static void TransferGlyphs56(Span<byte> input)
{
for (int i = 0; i < buffer.Length; i += 2)
for (int i = 0; i < input.Length; i += 2)
{
var span = buffer[i..];
var span = input[i..];
var c = ReadUInt16LittleEndian(span);
if (IsPrivateUseChar(c))
WriteUInt16LittleEndian(span, GetMigratedPrivateChar(c));
@@ -42,13 +126,12 @@ public static void TransferGlyphs56(Span<byte> buffer)
/// <summary>
/// Remaps private use unicode codepoint back to Gen5 private use codepoint.
/// </summary>
/// <param name="buffer">Input characters to transfer in place</param>
/// <returns>Remapped string</returns>
public static void TransferGlyphs65(Span<byte> buffer)
/// <param name="input">Input characters to transfer in place</param>
public static void TransferGlyphs65(Span<byte> input)
{
for (int i = 0; i < buffer.Length; i += 2)
for (int i = 0; i < input.Length; i += 2)
{
var span = buffer[i..];
var span = input[i..];
var c = ReadUInt16LittleEndian(span);
if (IsPrivateUseCharUnicode(c))
WriteUInt16LittleEndian(span, GetUnmigratedPrivateChar(c));

28
PKHeX.Core/PKM/Strings/StringConverter4Util.cs
View File

@@ -63,7 +63,7 @@ public static void StripDiacriticsFR4(Span<char> input)
#region Conversion Data
private const int TableINTEnd = 0x01DE; // Ignore chars that can't be entered.
private const int TableINTEnd = 0x01EC;
private const int TableKORStart = 0x400;
private const int TableKOREnd = 0xD65;
@@ -71,7 +71,6 @@ public static void StripDiacriticsFR4(Span<char> input)
private const ushort SaveInvalidAs = 0x1AC; // '?'
private const char NUL = (char)StringConverter4.Terminator;
private const char EMP = NUL; // Empty, not available on keyboard.
private const char YEN = NUL; // Yen, not available on keyboard.
public static ReadOnlySpan<char> TableINT =>
[
@@ -89,22 +88,23 @@ public static void StripDiacriticsFR4(Span<char> input)
'Ｅ', 'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', // 0B0-0BF
'Ｕ', 'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ', 'ｂ', 'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', // 0C0-0CF
'ｋ', 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｘ', 'ｙ', 'ｚ', // 0D0-0DF
EMP, '！', '？', '、', '。', '⋯', '・', '／', '「', '」', '『', '』', '（', '）', '⑭', '⑮', // 0E0-0EF
'＋', 'ー', '⑧', '⑨', '＝', '～', '：', '；', '．', '，', '♠', '♣', '♥', '♦', '★', '◎', // 0F0-0FF
'○', '□', '△', '◇', '＠', '♪', '％', '☀', '☁', '☂', '☃', '①', '②', '③', '☊', '⑤', // 100-10F
'⑥', '⑦', YEN, '♈', '♉', '♊', '♋', '♌', '♍', '♎', '♏', '←', '↑', '↓', '→', '‣', // 110-11F
EMP, '！', '？', '、', '。', '…', '・', '／', '「', '」', '『', '』', '（', '）', '♂', '♀', // 0E0-0EF
'＋', 'ー', '×', '÷', '＝', '～', '：', '；', '．', '，', '♠', '♣', '♥', '♦', '★', '◎', // 0F0-0FF
'○', '□', '△', '◇', '＠', '♪', '％', '☀', '☁', '☂', '☃', '①', '②', '③', '④', '⑤', // 100-10F
'⑥', '⑦', '円', '♈', '♉', '♊', '♋', '♌', '♍', '♎', '♏', '←', '↑', '↓', '→', '►', // 110-11F
'＆', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', // 120-12F
'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', // 130-13F
'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', // 140-14F
'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'À', // 150-15F
'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 'Ð', // 160-16F
'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×', 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 'à', // 170-17F
'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '⑧', 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 'à', // 170-17F
'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 'ð', // 180-18F
'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ', 'Œ', // 190-19F
'œ', 'Ş', 'ş', 'ª', 'º', '¹', '²', '³', '$', '¡', '¿', '!', '?', ',', '.', '⑬', // 1A0-1AF
'･', '/', '‘', '’', '“', '”', '„', '《', '》', '(', ')', '♂', '♀', '+', '-', '*', // 1B0-1BF
'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '⑨', 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ', 'Œ', // 190-19F
'œ', 'Ş', 'ş', 'ª', 'º', '⑩', '⑪', '⑫', '$', '¡', '¿', '!', '?', ',', '.', '⑬', // 1A0-1AF
'･', '/', '‘', '’', '“', '”', '„', '«', '»', '(', ')', '⑭', '⑮', '+', '-', '*', // 1B0-1BF
'#', '=', '&', '~', ':', ';', '⑯', '⑰', '⑱', '⑲', '⑳', '⑴', '⑵', '⑶', '⑷', '⑸', // 1C0-1CF
'@', '⑹', '%', '⑺', '⑻', '⑼', '⑽', '⑾', '⑿', '⒀', '⒁', '⒂', '⒃', '⒄', ' ', // 1D0-1DE*
'@', '⑹', '%', '⑺', '⑻', '⑼', '⑽', '⑾', '⑿', '⒀', '⒁', '⒂', '⒃', '⒄', ' ', '⒅', // 1D0-1DF
'⒆', '⒇', '⒈', '⒉', '⒊', '⒋', '⒌', '⒍', '°', '_', '＿', '⒎', '⒏', // 1E0-1EC*
];
public static ReadOnlySpan<char> TableKOR =>
@@ -258,9 +258,9 @@ public static void StripDiacriticsFR4(Span<char> input)
'흩', '희', '흰', '흴', '흼', '흽', '힁', '히', '힉', '힌', '힐', '힘', '힙', '힛', '힝', NUL, // D20-D2F
// 33 Jamo characters
NUL, 'ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', // D30-D3F
'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅭ', 'ᅮ', 'ᅲ', // D40-D4F
'ᅳ', 'ᅵ', NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, // D50-D5F
NUL, 'ㄱ', 'ㄲ', 'ㄴ', 'ㄷ', 'ㄸ', 'ㄹ', 'ㅁ', 'ㅂ', 'ㅃ', 'ㅅ', 'ㅆ', 'ㅇ', 'ㅈ', 'ㅉ', 'ㅊ', // D30-D3F
'ㅋ', 'ㅌ', 'ㅍ', 'ㅎ', 'ㅏ', 'ㅐ', 'ㅑ', 'ㅒ', 'ㅓ', 'ㅔ', 'ㅕ', 'ㅖ', 'ㅗ', 'ㅛ', 'ㅜ', 'ㅠ', // D40-D4F
'ㅡ', 'ㅣ', NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, NUL, // D50-D5F
// 5 additional syllables
NUL, '뢔', '쌰', '쎼', '쓔', '쬬', // D60-D65

2
PKHeX.Core/Saves/Storage/BulkStorage.cs
View File

@@ -53,7 +53,7 @@ protected BulkStorage(byte[] data, Type t, int start, int slotsPerBox = 30) : ba
public sealed override int GetPartyOffset(int slot) => int.MinValue;
public override string GetString(ReadOnlySpan<byte> data)
=> StringConverter.GetString(data, Generation, blank.Japanese, IsBigEndian);
=> StringConverter.GetString(data, Generation, blank.Japanese, IsBigEndian, Language);
public override int SetString(Span<byte> destBuffer, ReadOnlySpan<char> value, int maxLength, StringConverterOption option)
=> StringConverter.SetString(destBuffer, value, maxLength, option: option, generation: Generation, jp: blank.Japanese, isBigEndian: IsBigEndian, language: Language);

4
PKHeX.Core/Saves/Substructures/Gen3/SecretBase3.cs
View File

@@ -31,8 +31,8 @@ public int RegistryStatus
public string OriginalTrainerName
{
get => StringConverter3.GetString(Data.Slice(2, 7), Japanese);
set => StringConverter3.SetString(Data.Slice(2, 7), value, 7, Japanese, StringConverterOption.ClearFF);
get => StringConverter3.GetString(Data.Slice(2, 7), Language);
set => StringConverter3.SetString(Data.Slice(2, 7), value, 7, Language, StringConverterOption.ClearFF);
}
public uint OT_ID