From cedef43489b971b355dfea1129600528fa44fdbc Mon Sep 17 00:00:00 2001 From: Rachel Date: Wed, 30 Jul 2025 23:10:03 -0700 Subject: [PATCH] Implement JSON text banks in msgenc and fix program errors --- .../rapidjson_patch/000-pr719.patch | 22 ++ .../rapidjson_patch/001-pr1137.patch | 56 ++++ .../rapidjson_patch/002-pr1323.patch | 108 +++++++ subprojects/rapidjson.wrap | 2 +- tools/datagen/meson.build | 1 - tools/meson.build | 2 + tools/msgenc/Json.cpp | 285 ++++++++++++++++++ tools/msgenc/Json.h | 58 ++++ tools/msgenc/MessagesConverter.cpp | 15 + tools/msgenc/MessagesConverter.h | 12 +- tools/msgenc/MessagesDecoder.cpp | 13 +- tools/msgenc/MessagesDecoder.h | 1 + tools/msgenc/MessagesEncoder.cpp | 20 +- tools/msgenc/MessagesEncoder.h | 1 + tools/msgenc/Options.cpp | 8 +- tools/msgenc/Options.h | 11 +- tools/msgenc/meson.build | 13 +- tools/msgenc/msgenc.cpp | 15 +- 18 files changed, 614 insertions(+), 29 deletions(-) create mode 100644 subprojects/packagefiles/rapidjson_patch/000-pr719.patch create mode 100644 subprojects/packagefiles/rapidjson_patch/001-pr1137.patch create mode 100644 subprojects/packagefiles/rapidjson_patch/002-pr1323.patch create mode 100644 tools/msgenc/Json.cpp create mode 100644 tools/msgenc/Json.h diff --git a/subprojects/packagefiles/rapidjson_patch/000-pr719.patch b/subprojects/packagefiles/rapidjson_patch/000-pr719.patch new file mode 100644 index 0000000000..055bddc5ef --- /dev/null +++ b/subprojects/packagefiles/rapidjson_patch/000-pr719.patch @@ -0,0 +1,22 @@ +diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h +index e3e20dfb..19f5a6a5 100644 +--- a/include/rapidjson/document.h ++++ b/include/rapidjson/document.h +@@ -316,8 +316,6 @@ struct GenericStringRef { + + GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {} + +- GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; } +- + //! implicit conversion to plain CharType pointer + operator const Ch *() const { return s; } + +@@ -328,6 +326,8 @@ private: + //! Disallow construction from non-const array + template + GenericStringRef(CharType (&str)[N]) /* = delete */; ++ //! Copy assignment operator not permitted - immutable type ++ GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */; + }; + + //! Mark a character pointer as constant string diff --git a/subprojects/packagefiles/rapidjson_patch/001-pr1137.patch b/subprojects/packagefiles/rapidjson_patch/001-pr1137.patch new file mode 100644 index 0000000000..2b19494fcb --- /dev/null +++ b/subprojects/packagefiles/rapidjson_patch/001-pr1137.patch @@ -0,0 +1,56 @@ +diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h +index 094a07e8..eb6d7dcb 100644 +--- a/include/rapidjson/document.h ++++ b/include/rapidjson/document.h +@@ -45,7 +45,7 @@ RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_N + #endif // __GNUC__ + + #ifndef RAPIDJSON_NOMEMBERITERATORCLASS +-#include // std::iterator, std::random_access_iterator_tag ++#include // std::random_access_iterator_tag + #endif + + #if RAPIDJSON_HAS_CXX11_RVALUE_REFS +@@ -98,16 +98,13 @@ struct GenericMember { + \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator + */ + template +-class GenericMemberIterator +- : public std::iterator >::Type> { ++class GenericMemberIterator { + + friend class GenericValue; + template friend class GenericMemberIterator; + + typedef GenericMember PlainType; + typedef typename internal::MaybeAddConst::Type ValueType; +- typedef std::iterator BaseType; + + public: + //! Iterator type itself +@@ -117,12 +114,21 @@ public: + //! Non-constant iterator type + typedef GenericMemberIterator NonConstIterator; + ++ /** \name std::iterator_traits support */ ++ //@{ ++ typedef ValueType value_type; ++ typedef ValueType * pointer; ++ typedef ValueType & reference; ++ typedef std::ptrdiff_t difference_type; ++ typedef std::random_access_iterator_tag iterator_category; ++ //@} ++ + //! Pointer to (const) GenericMember +- typedef typename BaseType::pointer Pointer; ++ typedef pointer Pointer; + //! Reference to (const) GenericMember +- typedef typename BaseType::reference Reference; ++ typedef reference Reference; + //! Signed integer type (e.g. \c ptrdiff_t) +- typedef typename BaseType::difference_type DifferenceType; ++ typedef difference_type DifferenceType; + + //! Default constructor (singular value) + /*! Creates an iterator pointing to no element. diff --git a/subprojects/packagefiles/rapidjson_patch/002-pr1323.patch b/subprojects/packagefiles/rapidjson_patch/002-pr1323.patch new file mode 100644 index 0000000000..e8253ffc81 --- /dev/null +++ b/subprojects/packagefiles/rapidjson_patch/002-pr1323.patch @@ -0,0 +1,108 @@ +diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h +index d25c5c0a..bf9e6fdb 100644 +--- a/include/rapidjson/document.h ++++ b/include/rapidjson/document.h +@@ -1513,7 +1513,7 @@ public: + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); +- std::memmove(&*pos, &*last, static_cast(MemberEnd() - last) * sizeof(Member)); ++ std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; + } +@@ -1716,8 +1716,8 @@ public: + RAPIDJSON_ASSERT(last <= End()); + ValueIterator pos = Begin() + (first - Begin()); + for (ValueIterator itr = pos; itr != last; ++itr) +- itr->~GenericValue(); +- std::memmove(pos, last, static_cast(End() - last) * sizeof(GenericValue)); ++ itr->~GenericValue(); ++ std::memmove(static_cast(pos), last, static_cast(End() - last) * sizeof(GenericValue)); + data_.a.size -= static_cast(last - first); + return pos; + } +@@ -2032,7 +2032,7 @@ private: + if (count) { + GenericValue* e = static_cast(allocator.Malloc(count * sizeof(GenericValue))); + SetElementsPointer(e); +- std::memcpy(e, values, count * sizeof(GenericValue)); ++ std::memcpy(static_cast(e), values, count * sizeof(GenericValue)); + } + else + SetElementsPointer(0); +@@ -2050,7 +2045,7 @@ RAPIDJSON_DIAG_POP + if (count) { + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + SetMembersPointer(m); +- std::memcpy(m, members, count * sizeof(Member)); ++ std::memcpy(static_cast(m), members, count * sizeof(Member)); + } + else + SetMembersPointer(0); +diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h +index dc0af780..fa0d696e 100644 +--- a/include/rapidjson/schema.h ++++ b/include/rapidjson/schema.h +@@ -464,7 +464,7 @@ public: + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; +- char buffer[256 + 24]; ++ char buffer[256u + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); +diff --git a/test/perftest/schematest.cpp b/test/perftest/schematest.cpp +index 468f5fe6..7d27344b 100644 +--- a/test/perftest/schematest.cpp ++++ b/test/perftest/schematest.cpp +@@ -11,6 +11,11 @@ + + using namespace rapidjson; + ++RAPIDJSON_DIAG_PUSH ++#if defined(__GNUC__) && __GNUC__ >= 7 ++RAPIDJSON_DIAG_OFF(format-overflow) ++#endif ++ + template + static char* ReadFile(const char* filename, Allocator& allocator) { + const char *paths[] = { +@@ -42,6 +47,8 @@ static char* ReadFile(const char* filename, Allocator& allocator) { + return json; + } + ++RAPIDJSON_DIAG_POP ++ + class Schema : public PerfTest { + public: + Schema() {} +diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp +index 0c61a8a9..32610697 100644 +--- a/test/unittest/schematest.cpp ++++ b/test/unittest/schematest.cpp +@@ -1762,7 +1762,7 @@ private: + typename DocumentType::AllocatorType documentAllocator_; + typename SchemaDocumentType::AllocatorType schemaAllocator_; + char documentBuffer_[16384]; +- char schemaBuffer_[128 * 1024]; ++ char schemaBuffer_[128u * 1024]; + }; + + TEST(SchemaValidator, TestSuite) { +diff --git a/test/unittest/simdtest.cpp b/test/unittest/simdtest.cpp +index 7b58cd05..c60c85b2 100644 +--- a/test/unittest/simdtest.cpp ++++ b/test/unittest/simdtest.cpp +@@ -109,8 +109,8 @@ struct ScanCopyUnescapedStringHandler : BaseReaderHandler, ScanCopyUnesca + + template + void TestScanCopyUnescapedString() { +- char buffer[1024 + 5 + 32]; +- char backup[1024 + 5 + 32]; ++ char buffer[1024u + 5 + 32]; ++ char backup[1024u + 5 + 32]; + + // Test "ABCDABCD...\\" + for (size_t offset = 0; offset < 32; offset++) { diff --git a/subprojects/rapidjson.wrap b/subprojects/rapidjson.wrap index 041d711be6..2ff4a6c633 100644 --- a/subprojects/rapidjson.wrap +++ b/subprojects/rapidjson.wrap @@ -9,7 +9,7 @@ patch_hash = c1480d0ecef09dbaa4b4d85d86090205386fb2c7e87f4f158b20dbbda14c9afc source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/rapidjson_1.1.0-2/rapidjson-1.1.0.tar.gz wrapdb_version = 1.1.0-2 -diff_files = rapidjson_patch/gcc14fix.patch +diff_files = rapidjson_patch/000-pr719.patch, rapidjson_patch/001-pr1137.patch, rapidjson_patch/002-pr1323.patch [provide] rapidjson = rapidjson_dep diff --git a/tools/datagen/meson.build b/tools/datagen/meson.build index 2065de35ed..2bb973db72 100644 --- a/tools/datagen/meson.build +++ b/tools/datagen/meson.build @@ -10,7 +10,6 @@ subproject('narc') subproject('rapidjson') -rapidjson_dep = dependency('rapidjson') libnarc_dep = dependency('libnarc') datagen_cpp_args = [ diff --git a/tools/meson.build b/tools/meson.build index 876bb0eb13..250d0e3945 100644 --- a/tools/meson.build +++ b/tools/meson.build @@ -1,3 +1,5 @@ +rapidjson_dep = dependency('rapidjson') + # Native tools subdir('csv2bin') subdir('datagen') diff --git a/tools/msgenc/Json.cpp b/tools/msgenc/Json.cpp new file mode 100644 index 0000000000..c9e5cddba5 --- /dev/null +++ b/tools/msgenc/Json.cpp @@ -0,0 +1,285 @@ +#include "Json.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rapidjson/document.h" +#include "rapidjson/error/en.h" +#include "rapidjson/error/error.h" +#include "rapidjson/prettywriter.h" +#include "rapidjson/rapidjson.h" +#include "rapidjson/stringbuffer.h" + +#include "MessagesConverter.h" + +string ReadWholeFile(const string_view& fpath) { + constexpr size_t read_size = 4096; + + ifstream stream(fpath.data()); + stream.exceptions(ios_base::badbit); + if (!stream) { + throw ios_base::failure("file does not exist"); + } + + string out; + string buf(read_size, '\0'); + while (stream.read(&buf[0], read_size)) { + out.append(buf, 0, stream.gcount()); + } + + out.append(buf, 0, stream.gcount()); + return out; +} + +// Read header constants from the supplied file. +// Constants are expected to be of the format `#define +{name} +{int value}`, +// and the integer value should be sequential starting from 0. +void Json::ReadHeader(const string &_filename) { + string hstring = ReadWholeFile(_filename); + regex pattern(R"(#define\s+(\w+)\s+([0-9]+))"); + smatch results; + id_strings.clear(); + while (regex_search(hstring, results, pattern)) { + id_strings.emplace_back(results[1]); + hstring = results.suffix().str(); + } +} + +// Write header constants to the supplied file. +// Print constants in the format `#define {name} {int value}` such that the +// integer values are sequentially ordered starting from 0. +void Json::WriteHeader(const string &_filename) { + ofstream hstrm(_filename); + string guard(_filename); + for (auto &c : guard) { + switch (c) { + case '/': + case '.': + case '-': + c = '_'; + break; + + default: + c = toupper(c); + break; + } + } + + hstrm << "/***************************************************\n"; + hstrm << " * WARNING: This file was autogenerated by msgenc. *\n"; + hstrm << " * DO NOT MODIFY *\n"; + hstrm << " ***************************************************/\n"; + hstrm << "\n"; + hstrm << "#ifndef MSGENC_" << guard << "\n"; + hstrm << "#define MSGENC_" << guard << "\n"; + hstrm << "\n"; + for (size_t i = 0; i < id_strings.size(); i++) { + vector message_lines = SplitMessage(messages[i]); + for (const auto& line : message_lines) { + hstrm << "// " << line << "\n"; + } + hstrm << "#define " << id_strings[i] << " " << i << "\n"; + } + + hstrm << "\n"; + hstrm << "#endif // MSGENC_" << guard << "\n"; + hstrm.flush(); +} + +// Read messages from JSON into memory to be converted +int Json::FromFile(MessagesConverter &converter) { + static char errbuf[256] = {0}; + + string json = ReadWholeFile(filename); + rapidjson::ParseResult result = doc.Parse(json.c_str()); + if (!result) { + throw runtime_error(rapidjson::GetParseError_En(result.Code())); + } + + if (!doc.IsObject()) { + throw runtime_error("Input document is not an object."); + } + + int i = 0; + int key = JSON_KEY_NOT_DEFINED; + + if (auto membKey = doc.FindMember("key"); membKey != doc.MemberEnd()) { + if (!membKey->value.IsInt()) { + throw runtime_error("Non-numeric value for `key`."); + } + + key = membKey->value.GetInt(); + key &= 0xFFFF; + key |= 0x10000; + } + + auto membMessages = doc.FindMember("messages"); + if (membMessages == doc.MemberEnd()) { + throw runtime_error("Bank: missing required member `messages`."); + } else if (!membMessages->value.IsArray()) { + throw runtime_error("Bank.messages: expected an array."); + } + + for (const auto &entry : membMessages->value.GetArray()) { + if (!entry.IsObject()) { + sprintf(errbuf, "Bank.messages[%d]: expected an object.", i); + throw runtime_error(errbuf); + } + + string message; + auto membId = entry.FindMember("id"); + auto membGarbage = entry.FindMember("garbage"); + auto membText = entry.FindMember("en_US"); // TODO: Multi-language support + + if (membId == entry.MemberEnd()) { + sprintf(errbuf, "Bank.messages[%d]: missing required member `id`.", i); + throw runtime_error(errbuf); + } else if (!membId->value.IsString()) { + sprintf(errbuf, "Bank.messages[%d].id: expected a string.", i); + throw runtime_error(errbuf); + } + + if (membGarbage != entry.MemberEnd()) { + if (!membGarbage->value.IsInt()) { + sprintf(errbuf, "Bank.messages[%d].garbage: expected an integer.", i); + throw runtime_error(errbuf); + } + + message.resize(membGarbage->value.GetInt(), ' '); + goto enqueue_message; + } + + if (membText == entry.MemberEnd()) { + sprintf(errbuf, "Bank.messages[%d]: missing required member `en_US`.", i); + throw runtime_error(errbuf); + } else if (!membText->value.IsString() && !membText->value.IsArray()) { + sprintf(errbuf, "Bank.messages[%d].en_US: expected a string or multi-line array of strings.", i); + throw runtime_error(errbuf); + } + + if (membText->value.IsString()) { + message.assign(membText->value.GetString()); + } else if (membText->value.IsArray()) { + for (const auto &line : membText->value.GetArray()) { + if (!line.IsString()) { + goto content_error; + } + + message.append(line.GetString()); + } + } else { + content_error: + sprintf(errbuf, "Bank.messages[%d].en_US: expected a string or array of strings.", i); + throw runtime_error(errbuf); + } + + enqueue_message: + converter.GetDecodedMessages().emplace_back(message); // emplace a copy + id_strings.emplace_back(membId->value.GetString()); + messages.push_back(message); + i++; + IncRowNoBuf(); + } + + if (!converter.GetHeaderFilename().empty()) { + WriteHeader(converter.GetHeaderFilename()); + } + + return key; +} + +void Json::ToFile(MessagesConverter &converter) { + if (!converter.GetHeaderFilename().empty()) { + ReadHeader(converter.GetHeaderFilename()); + } + + auto it = id_strings.cbegin(); + doc.SetObject(); + doc.AddMember("key", converter.GetKey(), doc.GetAllocator()); + + rapidjson::Value messages(rapidjson::kArrayType); + + char keybuf[256]; + string prefix = filename.substr(filename.find_last_of('/') + 1); + prefix = prefix.substr(0, prefix.find_first_of('.')); + for (const auto &message : converter.GetDecodedMessages()) { + rapidjson::Value entry_name(rapidjson::kStringType); + if (it != id_strings.cend()) { + entry_name.SetString(it->c_str(), doc.GetAllocator()); + } else { + sprintf(keybuf, "%s_%s", prefix.c_str(), row_no_buf); + entry_name.SetString(keybuf, doc.GetAllocator()); + } + + rapidjson::Value entry(rapidjson::kObjectType); + entry.AddMember("id", entry_name, doc.GetAllocator()); + + if (message.find_first_not_of(' ') == string::npos) { + entry.AddMember("garbage", message.size(), doc.GetAllocator()); + } else { + vector message_lines = SplitMessage(message, true); + if (message_lines.size() == 1) { + rapidjson::Value entry_message(rapidjson::kStringType); + entry_message.SetString(message.c_str(), message.size(), doc.GetAllocator()); + entry.AddMember("en_US", entry_message, doc.GetAllocator()); + } else { + rapidjson::Value entry_lines(rapidjson::kArrayType); + for (const auto& line : message_lines) { + rapidjson::Value entry_line(rapidjson::kStringType); + entry_line.SetString(line.c_str(), line.size(), doc.GetAllocator()); + entry_lines.PushBack(entry_line, doc.GetAllocator()); + } + entry.AddMember("en_US", entry_lines, doc.GetAllocator()); + } + } + + messages.PushBack(entry, doc.GetAllocator()); + IncRowNoBuf(); + it++; + } + + doc.AddMember("messages", messages, doc.GetAllocator()); + + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + doc.Accept(writer); + + ofstream jstrm(filename); + jstrm << buffer.GetString() << endl; +} + +vector Json::SplitMessage(const string &message, bool preserve) { + vector v; + + size_t start = 0, i = 0; + for (; i < message.size(); i++) { + if (message[i] == '\r' || message[i] == '\n' || message[i] == '\f') { + if (preserve) { + v.push_back(message.substr(start, i - start + 1)); + } else { + v.push_back(message.substr(start, i - start)); + } + + start = i + 1; + } + } + + if (start < message.size()) { + if (preserve) { + v.push_back(message.substr(start, i - start + 1)); + } else { + v.push_back(message.substr(start, i - start)); + } + } + + return v; +} diff --git a/tools/msgenc/Json.h b/tools/msgenc/Json.h new file mode 100644 index 0000000000..8b4f8db244 --- /dev/null +++ b/tools/msgenc/Json.h @@ -0,0 +1,58 @@ +#ifndef GUARD_JSON_H +#define GUARD_JSON_H + +// JSON-encoded structures are a port of the original GMM structure previously +// used by the project. A valid JSON file contains a numeric key, and all other +// keys should be interpreted in-order as the entries for a text bank. Each +// bank entry is sub-divided into locales to allow for future multi-language +// extensions. + +#include +#include +#include + +#include "rapidjson/document.h" + +#include "MessagesConverter.h" + +#define JSON_KEY_NOT_DEFINED -1 + +class Json { + static constexpr int _row_no_buf_ndigit = 5; + + char row_no_buf[_row_no_buf_ndigit + 1] = {0}; + string filename; + rapidjson::Document doc; + vector id_strings; + vector messages; + + void WriteHeader(const string &_filename); + void ReadHeader(const string &_filename); + void IncRowNoBuf() { + for (int i = _row_no_buf_ndigit - 1; i >= 0; i--) { + row_no_buf[i]++; + if (row_no_buf[i] > '9') { + if (i == 0) { + throw runtime_error("message count overflow"); + } + + row_no_buf[i] = '0'; + } else { + break; + } + } + } + +public: + Json(string &_filename, ios::openmode _openmode) : filename(_filename) { + memset(row_no_buf, '0', _row_no_buf_ndigit); + } + + int FromFile(MessagesConverter &converter); + void ToFile(MessagesConverter &converter); + +private: + vector SplitMessage(const string &message, bool preserve = false); +}; + +#endif // GUARD_JSON_H diff --git a/tools/msgenc/MessagesConverter.cpp b/tools/msgenc/MessagesConverter.cpp index ff75f7d78c..d57e1b6e9a 100644 --- a/tools/msgenc/MessagesConverter.cpp +++ b/tools/msgenc/MessagesConverter.cpp @@ -61,6 +61,21 @@ void MessagesConverter::ReadCharmap() { if (code[0] == '{' && code[code.length() - 1] == '}') { code = code.substr(1, code.length() - 2); CmdmapRegisterCommand(code, value_i); + } else if (code[0] == '\\' && code.length() == 2) { + string literal; + switch (code[1]) { + case 'n': literal = '\n'; break; + case 'r': literal = '\r'; break; + case 'f': literal = '\f'; break; + + default: + stringstream s; + s << "charmap syntax error at " << (lineno + 1); + throw runtime_error(s.str()); + } + + CharmapRegisterCharacter(code, value_i); + CharmapRegisterCharacter(literal, value_i); } else { CharmapRegisterCharacter(code, value_i); } diff --git a/tools/msgenc/MessagesConverter.h b/tools/msgenc/MessagesConverter.h index 8e39ae4bf6..c0feb81643 100644 --- a/tools/msgenc/MessagesConverter.h +++ b/tools/msgenc/MessagesConverter.h @@ -58,6 +58,7 @@ protected: string charmapfilename; string binfilename; string headerfilename; + TextFormat text_format = FMT_PLAIN_TEXT; MsgArcHeader header = {}; vector alloc_table; @@ -72,19 +73,12 @@ protected: static string ReadTextFile(string& filename); static void WriteTextFile(string& filename, string const & contents); -public: - typedef int txtfmt; - static const txtfmt PlainText = 0; - static const txtfmt GamefreakGMM = 1; -protected: - txtfmt text_format = PlainText; - public: MessagesConverter(Options &options) : - textfilename(options.posargs[0]), - binfilename(options.posargs[1]), mode(options.mode), + textfilename(options.posargs[0]), charmapfilename(options.charmap), + binfilename(options.posargs[1]), headerfilename(options.gmm_header), text_format(options.textFormat) { diff --git a/tools/msgenc/MessagesDecoder.cpp b/tools/msgenc/MessagesDecoder.cpp index 9ab04139f3..99cc5df6a8 100644 --- a/tools/msgenc/MessagesDecoder.cpp +++ b/tools/msgenc/MessagesDecoder.cpp @@ -1,6 +1,8 @@ #include #include "MessagesDecoder.h" #include "Gmm.h" +#include "Json.h" +#include "Options.h" void MessagesDecoder::CmdmapRegisterCommand(string &command, uint16_t value) { @@ -168,6 +170,10 @@ void MessagesDecoder::WriteMessagesToGMM(string &filename) { GMM(filename, std::ios::out).ToFile(*this); } +void MessagesDecoder::WriteMessagesToJson(string &filename) { + Json(filename, std::ios::out).ToFile(*this); +} + // Public virtual functions void MessagesDecoder::ReadInput() @@ -187,11 +193,14 @@ void MessagesDecoder::Convert() void MessagesDecoder::WriteOutput() { switch (text_format) { - case PlainText: + case FMT_PLAIN_TEXT: WriteMessagesToText(textfilename); break; - case GamefreakGMM: + case FMT_GAMEFREAK_GMM: WriteMessagesToGMM(textfilename); break; + case FMT_JSON: + WriteMessagesToJson(textfilename); + break; } } diff --git a/tools/msgenc/MessagesDecoder.h b/tools/msgenc/MessagesDecoder.h index 2896c38501..45818f99f5 100644 --- a/tools/msgenc/MessagesDecoder.h +++ b/tools/msgenc/MessagesDecoder.h @@ -19,6 +19,7 @@ class MessagesDecoder : public MessagesConverter void ReadMessagesFromBin(string& filename); void WriteMessagesToText(string& filename); void WriteMessagesToGMM(string& filename); + void WriteMessagesToJson(string& filename); template void WriteBinaryFile(string& filename, T& data); static u16string DecodeTrainerNameMessage(u16string const &message); string DecodeMessage(u16string& message, int& i); diff --git a/tools/msgenc/MessagesEncoder.cpp b/tools/msgenc/MessagesEncoder.cpp index 1d89cb6d14..866fc79258 100644 --- a/tools/msgenc/MessagesEncoder.cpp +++ b/tools/msgenc/MessagesEncoder.cpp @@ -1,5 +1,7 @@ #include "MessagesEncoder.h" #include "Gmm.h" +#include "Json.h" +#include "Options.h" void MessagesEncoder::CmdmapRegisterCommand(string &command, uint16_t value) { @@ -42,6 +44,17 @@ void MessagesEncoder::ReadMessagesFromGMM(string& filename) { debug_printf("%d lines\n", header.count); } +void MessagesEncoder::ReadMessagesFromJson(string& filename) { + int key_from_file = Json(filename, std::ios::in).FromFile(*this); + + if (key_from_file != JSON_KEY_NOT_DEFINED) { + header.key = key_from_file; + } + + header.count = vec_decoded.size(); + debug_printf("%d lines\n", header.count); +} + u16string MessagesEncoder::EncodeMessage(const string & message, int & i) { u16string encoded; bool is_trname = false; @@ -158,12 +171,15 @@ void MessagesEncoder::WriteMessagesToBin(string& filename) { void MessagesEncoder::ReadInput() { switch (text_format) { - case PlainText: + case FMT_PLAIN_TEXT: ReadMessagesFromText(textfilename); break; - case GamefreakGMM: + case FMT_GAMEFREAK_GMM: ReadMessagesFromGMM(textfilename); break; + case FMT_JSON: + ReadMessagesFromJson(textfilename); + break; } } diff --git a/tools/msgenc/MessagesEncoder.h b/tools/msgenc/MessagesEncoder.h index 38a8b0e60e..1856f0b817 100644 --- a/tools/msgenc/MessagesEncoder.h +++ b/tools/msgenc/MessagesEncoder.h @@ -11,6 +11,7 @@ class MessagesEncoder : public MessagesConverter void ReadMessagesFromText(string& filename); void ReadMessagesFromGMM(string& filename); + void ReadMessagesFromJson(string& filename); void WriteMessagesToBin(string& filename); u16string EncodeMessage(const string& message, int & i); void CharmapRegisterCharacter(string& code, uint16_t value) override; diff --git a/tools/msgenc/Options.cpp b/tools/msgenc/Options.cpp index 64492d0f9c..ce97f7abcf 100644 --- a/tools/msgenc/Options.cpp +++ b/tools/msgenc/Options.cpp @@ -1,5 +1,7 @@ #include "Options.h" +static string POSARGS[] = {"TEXTFILE", "BINFILE"}; + Options::Options(int argc, char **argv) { for (int i = 1; i < argc; i++) { string arg(argv[i]); @@ -24,8 +26,10 @@ Options::Options(int argc, char **argv) { charmap = argv[++i]; } else if (arg == "-D") { dumpBinary = argv[++i]; + } else if (arg == "--json") { + textFormat = FMT_JSON; } else if (arg == "--gmm") { - textFormat = GamefreakGMM; + textFormat = FMT_GAMEFREAK_GMM; } else if (arg[0] != '-') { posargs.push_back(arg); } else { @@ -34,7 +38,7 @@ Options::Options(int argc, char **argv) { } } if (posargs.size() < 2) { - failReason = "missing required positional argument: " + (string[]){"INFILE", "OUTFILE"}[posargs.size()]; + failReason = "missing required positional argument: " + POSARGS[posargs.size()]; } if (mode == CONV_INVALID) { failReason = "missing mode flag: -d or -e is required"; diff --git a/tools/msgenc/Options.h b/tools/msgenc/Options.h index ba224b9eb3..77c61ac671 100644 --- a/tools/msgenc/Options.h +++ b/tools/msgenc/Options.h @@ -14,6 +14,12 @@ enum ConvertMode : uint8_t { CONV_INVALID = 0xFF, }; +enum TextFormat : uint8_t { + FMT_PLAIN_TEXT = 0, + FMT_GAMEFREAK_GMM, + FMT_JSON, +}; + struct Options { ConvertMode mode = CONV_INVALID; int key = 0; @@ -24,10 +30,7 @@ struct Options { bool printVersion = false; string dumpBinary; string gmm_header = ""; - typedef int txtfmt; - static const txtfmt PlainText = 0; - static const txtfmt GamefreakGMM = 1; - txtfmt textFormat = PlainText; + TextFormat textFormat = FMT_PLAIN_TEXT; Options(int argc, char ** argv); }; diff --git a/tools/msgenc/meson.build b/tools/msgenc/meson.build index 88d9ac349d..b269f4a488 100644 --- a/tools/msgenc/meson.build +++ b/tools/msgenc/meson.build @@ -8,11 +8,20 @@ msgenc_exe = executable('msgenc', 'MessagesDecoder.cpp', 'MessagesEncoder.cpp', 'Gmm.cpp', + 'Json.cpp', 'pugixml.cpp' ], cpp_args: [ '-DNDEBUG', - '-std=c++17' + '-g', + '-std=c++17', + '-Wall', + '-Wextra', + '-Wpedantic', + '-Wno-unused-parameter', ], - native: true + native: true, + dependencies: [ + rapidjson_dep, + ] ) diff --git a/tools/msgenc/msgenc.cpp b/tools/msgenc/msgenc.cpp index 15c0b7ce61..a6d21dd507 100644 --- a/tools/msgenc/msgenc.cpp +++ b/tools/msgenc/msgenc.cpp @@ -11,18 +11,19 @@ #include "Options.h" static const char* progname = "msgenc"; -static const char* version = "2021.12.21"; +static const char* version = "2025.08.12"; static inline void usage() { cout << progname << " v" << version << endl; - cout << "Usage: " << progname << " [-h] [-v] -d|-e [OPTIONS] -c CHARMAP INFILE OUTFILE" << endl; + cout << "Usage: " << progname << " [-h] [-v] -d|-e [OPTIONS] -c CHARMAP TEXTFILE BINFILE" << endl; cout << endl; - cout << "INFILE Required: Path to the input file to convert (-e: plaintext; -d: binary)." << endl; - cout << "OUTFILE Required: Path to the output file (-e: binary; -d: plaintext)." << endl; + cout << "TEXTFILE Required: Path to the text file (-e: input; -d: output)." << endl; + cout << "BINFILE Required: Path to the binary file (ie: output; -d: input)." << endl; cout << "-c CHARMAP Required: Path to a text file with a character mapping, for example pokeheartgold/charmap.txt." << endl; cout << "-d Decode from binary to text, also print the key" << endl; cout << "-e Encode from text to binary using the provided key" << endl; cout << "--gmm Text file is GMM (Gamefreak XML format)" << endl; + cout << "--json Text file is JSON" << endl; cout << "-H HEADER When operating in GMM mode, specify this header file to read/write C constant values" << endl; cout << "-k KEY The 16-bit encryption key for this message bank. Default: computes it from the binary file name" << endl; cout << "-v Print the program version and exit." << endl; @@ -73,8 +74,10 @@ int do_main(MessagesConverter* &converter, int argc, char ** argv) { } int main(int argc, char ** argv) { - MessagesConverter *converter; + MessagesConverter *converter = nullptr; int result = do_main(converter, argc, argv); - delete converter; + + if (converter) + delete converter; return result; }