Implement JSON text banks in msgenc and fix program errors

subprojects/packagefiles/rapidjson_patch/000-pr719.patch

@@ -0,0 +1,22 @@
+diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h
+index e3e20dfb..19f5a6a5 100644
+--- a/include/rapidjson/document.h
++++ b/include/rapidjson/document.h
+@@ -316,8 +316,6 @@ struct GenericStringRef {
+ 
+     GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {}
+ 
+-    GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; }
+-
+     //! implicit conversion to plain CharType pointer
+     operator const Ch *() const { return s; }
+ 
+@@ -328,6 +326,8 @@ private:
+     //! Disallow construction from non-const array
+     template<SizeType N>
+     GenericStringRef(CharType (&str)[N]) /* = delete */;
++    //! Copy assignment operator not permitted - immutable type
++    GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */;
+ };
+ 
+ //! Mark a character pointer as constant string

subprojects/packagefiles/rapidjson_patch/001-pr1137.patch

@@ -0,0 +1,56 @@
+diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h
+index 094a07e8..eb6d7dcb 100644
+--- a/include/rapidjson/document.h
++++ b/include/rapidjson/document.h
+@@ -45,7 +45,7 @@ RAPIDJSON_DIAG_OFF(terminate) // ignore throwing RAPIDJSON_ASSERT in RAPIDJSON_N
+ #endif // __GNUC__
+ 
+ #ifndef RAPIDJSON_NOMEMBERITERATORCLASS
+-#include <iterator> // std::iterator, std::random_access_iterator_tag
++#include <iterator> // std::random_access_iterator_tag
+ #endif
+ 
+ #if RAPIDJSON_HAS_CXX11_RVALUE_REFS
+@@ -98,16 +98,13 @@ struct GenericMember {
+     \see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator
+  */
+ template <bool Const, typename Encoding, typename Allocator>
+-class GenericMemberIterator
+-    : public std::iterator<std::random_access_iterator_tag
+-        , typename internal::MaybeAddConst<Const,GenericMember<Encoding,Allocator> >::Type> {
++class GenericMemberIterator {
+ 
+     friend class GenericValue<Encoding,Allocator>;
+     template <bool, typename, typename> friend class GenericMemberIterator;
+ 
+     typedef GenericMember<Encoding,Allocator> PlainType;
+     typedef typename internal::MaybeAddConst<Const,PlainType>::Type ValueType;
+-    typedef std::iterator<std::random_access_iterator_tag,ValueType> BaseType;
+ 
+ public:
+     //! Iterator type itself
+@@ -117,12 +114,21 @@ public:
+     //! Non-constant iterator type
+     typedef GenericMemberIterator<false,Encoding,Allocator> NonConstIterator;
+ 
++    /** \name std::iterator_traits support */
++    //@{
++    typedef ValueType      value_type;
++    typedef ValueType *    pointer;
++    typedef ValueType &    reference;
++    typedef std::ptrdiff_t difference_type;
++    typedef std::random_access_iterator_tag iterator_category;
++    //@}
++
+     //! Pointer to (const) GenericMember
+-    typedef typename BaseType::pointer         Pointer;
++    typedef pointer         Pointer;
+     //! Reference to (const) GenericMember
+-    typedef typename BaseType::reference       Reference;
++    typedef reference       Reference;
+     //! Signed integer type (e.g. \c ptrdiff_t)
+-    typedef typename BaseType::difference_type DifferenceType;
++    typedef difference_type DifferenceType;
+ 
+     //! Default constructor (singular value)
+     /*! Creates an iterator pointing to no element.

subprojects/packagefiles/rapidjson_patch/002-pr1323.patch

@@ -0,0 +1,108 @@
+diff --git a/include/rapidjson/document.h b/include/rapidjson/document.h
+index d25c5c0a..bf9e6fdb 100644
+--- a/include/rapidjson/document.h
++++ b/include/rapidjson/document.h
+@@ -1513,7 +1513,7 @@ public:
+         MemberIterator pos = MemberBegin() + (first - MemberBegin());
+         for (MemberIterator itr = pos; itr != last; ++itr)
+             itr->~Member();
+-        std::memmove(&*pos, &*last, static_cast<size_t>(MemberEnd() - last) * sizeof(Member));
++        std::memmove(static_cast<void*>(&*pos), &*last, static_cast<size_t>(MemberEnd() - last) * sizeof(Member));
+         data_.o.size -= static_cast<SizeType>(last - first);
+         return pos;
+     }
+@@ -1716,8 +1716,8 @@ public:
+         RAPIDJSON_ASSERT(last <= End());
+         ValueIterator pos = Begin() + (first - Begin());
+         for (ValueIterator itr = pos; itr != last; ++itr)
+-            itr->~GenericValue();       
+-        std::memmove(pos, last, static_cast<size_t>(End() - last) * sizeof(GenericValue));
++            itr->~GenericValue();
++        std::memmove(static_cast<void*>(pos), last, static_cast<size_t>(End() - last) * sizeof(GenericValue));
+         data_.a.size -= static_cast<SizeType>(last - first);
+         return pos;
+     }
+@@ -2032,7 +2032,7 @@ private:
+         if (count) {
+             GenericValue* e = static_cast<GenericValue*>(allocator.Malloc(count * sizeof(GenericValue)));
+             SetElementsPointer(e);
+-            std::memcpy(e, values, count * sizeof(GenericValue));
++            std::memcpy(static_cast<void*>(e), values, count * sizeof(GenericValue));
+         }
+         else
+             SetElementsPointer(0);
+@@ -2050,7 +2045,7 @@ RAPIDJSON_DIAG_POP
+         if (count) {
+             Member* m = static_cast<Member*>(allocator.Malloc(count * sizeof(Member)));
+             SetMembersPointer(m);
+-            std::memcpy(m, members, count * sizeof(Member));
++            std::memcpy(static_cast<void*>(m), members, count * sizeof(Member));
+         }
+         else
+             SetMembersPointer(0);
+diff --git a/include/rapidjson/schema.h b/include/rapidjson/schema.h
+index dc0af780..fa0d696e 100644
+--- a/include/rapidjson/schema.h
++++ b/include/rapidjson/schema.h
+@@ -464,7 +464,7 @@ public:
+                 enum_ = static_cast<uint64_t*>(allocator_->Malloc(sizeof(uint64_t) * v->Size()));
+                 for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) {
+                     typedef Hasher<EncodingType, MemoryPoolAllocator<> > EnumHasherType;
+-                    char buffer[256 + 24];
++                    char buffer[256u + 24];
+                     MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer));
+                     EnumHasherType h(&hasherAllocator, 256);
+                     itr->Accept(h);
+diff --git a/test/perftest/schematest.cpp b/test/perftest/schematest.cpp
+index 468f5fe6..7d27344b 100644
+--- a/test/perftest/schematest.cpp
++++ b/test/perftest/schematest.cpp
+@@ -11,6 +11,11 @@
+ 
+ using namespace rapidjson;
+ 
++RAPIDJSON_DIAG_PUSH
++#if defined(__GNUC__) && __GNUC__ >= 7
++RAPIDJSON_DIAG_OFF(format-overflow)
++#endif
++
+ template <typename Allocator>
+ static char* ReadFile(const char* filename, Allocator& allocator) {
+     const char *paths[] = {
+@@ -42,6 +47,8 @@ static char* ReadFile(const char* filename, Allocator& allocator) {
+     return json;
+ }
+ 
++RAPIDJSON_DIAG_POP
++
+ class Schema : public PerfTest {
+ public:
+     Schema() {}
+diff --git a/test/unittest/schematest.cpp b/test/unittest/schematest.cpp
+index 0c61a8a9..32610697 100644
+--- a/test/unittest/schematest.cpp
++++ b/test/unittest/schematest.cpp
+@@ -1762,7 +1762,7 @@ private:
+     typename DocumentType::AllocatorType documentAllocator_;
+     typename SchemaDocumentType::AllocatorType schemaAllocator_;
+     char documentBuffer_[16384];
+-    char schemaBuffer_[128 * 1024];
++    char schemaBuffer_[128u * 1024];
+ };
+ 
+ TEST(SchemaValidator, TestSuite) {
+diff --git a/test/unittest/simdtest.cpp b/test/unittest/simdtest.cpp
+index 7b58cd05..c60c85b2 100644
+--- a/test/unittest/simdtest.cpp
++++ b/test/unittest/simdtest.cpp
+@@ -109,8 +109,8 @@ struct ScanCopyUnescapedStringHandler : BaseReaderHandler<UTF8<>, ScanCopyUnesca
+ 
+ template <unsigned parseFlags, typename StreamType>
+ void TestScanCopyUnescapedString() {
+-    char buffer[1024 + 5 + 32];
+-    char backup[1024 + 5 + 32];
++    char buffer[1024u + 5 + 32];
++    char backup[1024u + 5 + 32];
+ 
+     // Test "ABCDABCD...\\"
+     for (size_t offset = 0; offset < 32; offset++) {

subprojects/rapidjson.wrap

@@ -9,7 +9,7 @@ patch_hash = c1480d0ecef09dbaa4b4d85d86090205386fb2c7e87f4f158b20dbbda14c9afc
 source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/rapidjson_1.1.0-2/rapidjson-1.1.0.tar.gz
 wrapdb_version = 1.1.0-2
 
-diff_files = rapidjson_patch/gcc14fix.patch
+diff_files = rapidjson_patch/000-pr719.patch, rapidjson_patch/001-pr1137.patch, rapidjson_patch/002-pr1323.patch
 
 [provide]
 rapidjson = rapidjson_dep

tools/datagen/meson.build

@@ -10,7 +10,6 @@
 subproject('narc')
 subproject('rapidjson')
 
-rapidjson_dep = dependency('rapidjson')
 libnarc_dep = dependency('libnarc')
 
 datagen_cpp_args = [

tools/meson.build

@@ -1,3 +1,5 @@
+rapidjson_dep = dependency('rapidjson')
+
 # Native tools
 subdir('csv2bin')
 subdir('datagen')

tools/msgenc/Json.cpp

@@ -0,0 +1,285 @@
+#include "Json.h"
+
+#include <cctype>
+#include <cstddef>
+#include <cstdio>
+#include <cstring>
+#include <fstream>
+#include <ios>
+#include <regex>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "rapidjson/document.h"
+#include "rapidjson/error/en.h"
+#include "rapidjson/error/error.h"
+#include "rapidjson/prettywriter.h"
+#include "rapidjson/rapidjson.h"
+#include "rapidjson/stringbuffer.h"
+
+#include "MessagesConverter.h"
+
+string ReadWholeFile(const string_view& fpath) {
+    constexpr size_t read_size = 4096;
+
+    ifstream stream(fpath.data());
+    stream.exceptions(ios_base::badbit);
+    if (!stream) {
+        throw ios_base::failure("file does not exist");
+    }
+
+    string out;
+    string buf(read_size, '\0');
+    while (stream.read(&buf[0], read_size)) {
+        out.append(buf, 0, stream.gcount());
+    }
+
+    out.append(buf, 0, stream.gcount());
+    return out;
+}
+
+// Read header constants from the supplied file.
+// Constants are expected to be of the format `#define +{name} +{int value}`,
+// and the integer value should be sequential starting from 0.
+void Json::ReadHeader(const string &_filename) {
+    string hstring = ReadWholeFile(_filename);
+    regex pattern(R"(#define\s+(\w+)\s+([0-9]+))");
+    smatch results;
+    id_strings.clear();
+    while (regex_search(hstring, results, pattern)) {
+        id_strings.emplace_back(results[1]);
+        hstring = results.suffix().str();
+    }
+}
+
+// Write header constants to the supplied file.
+// Print constants in the format `#define {name} {int value}` such that the
+// integer values are sequentially ordered starting from 0.
+void Json::WriteHeader(const string &_filename) {
+    ofstream hstrm(_filename);
+    string guard(_filename);
+    for (auto &c : guard) {
+        switch (c) {
+        case '/':
+        case '.':
+        case '-':
+            c = '_';
+            break;
+
+        default:
+            c = toupper(c);
+            break;
+        }
+    }
+
+    hstrm << "/***************************************************\n";
+    hstrm << " * WARNING: This file was autogenerated by msgenc. *\n";
+    hstrm << " *               DO NOT MODIFY                     *\n";
+    hstrm << " ***************************************************/\n";
+    hstrm << "\n";
+    hstrm << "#ifndef MSGENC_" << guard << "\n";
+    hstrm << "#define MSGENC_" << guard << "\n";
+    hstrm << "\n";
+    for (size_t i = 0; i < id_strings.size(); i++) {
+        vector<string> message_lines = SplitMessage(messages[i]);
+        for (const auto& line : message_lines) {
+            hstrm << "// " << line << "\n";
+        }
+        hstrm << "#define " << id_strings[i] << " " << i << "\n";
+    }
+
+    hstrm << "\n";
+    hstrm << "#endif // MSGENC_" << guard << "\n";
+    hstrm.flush();
+}
+
+// Read messages from JSON into memory to be converted
+int Json::FromFile(MessagesConverter &converter) {
+    static char errbuf[256] = {0};
+
+    string json = ReadWholeFile(filename);
+    rapidjson::ParseResult result = doc.Parse(json.c_str());
+    if (!result) {
+        throw runtime_error(rapidjson::GetParseError_En(result.Code()));
+    }
+
+    if (!doc.IsObject()) {
+        throw runtime_error("Input document is not an object.");
+    }
+
+    int i = 0;
+    int key = JSON_KEY_NOT_DEFINED;
+
+    if (auto membKey = doc.FindMember("key"); membKey != doc.MemberEnd()) {
+        if (!membKey->value.IsInt()) {
+            throw runtime_error("Non-numeric value for `key`.");
+        }
+
+        key = membKey->value.GetInt();
+        key &= 0xFFFF;
+        key |= 0x10000;
+    }
+
+    auto membMessages = doc.FindMember("messages");
+    if (membMessages == doc.MemberEnd()) {
+        throw runtime_error("Bank: missing required member `messages`.");
+    } else if (!membMessages->value.IsArray()) {
+        throw runtime_error("Bank.messages: expected an array.");
+    }
+
+    for (const auto &entry : membMessages->value.GetArray()) {
+        if (!entry.IsObject()) {
+            sprintf(errbuf, "Bank.messages[%d]: expected an object.", i);
+            throw runtime_error(errbuf);
+        }
+
+        string message;
+        auto membId = entry.FindMember("id");
+        auto membGarbage = entry.FindMember("garbage");
+        auto membText = entry.FindMember("en_US"); // TODO: Multi-language support
+
+        if (membId == entry.MemberEnd()) {
+            sprintf(errbuf, "Bank.messages[%d]: missing required member `id`.", i);
+            throw runtime_error(errbuf);
+        } else if (!membId->value.IsString()) {
+            sprintf(errbuf, "Bank.messages[%d].id: expected a string.", i);
+            throw runtime_error(errbuf);
+        }
+
+        if (membGarbage != entry.MemberEnd()) {
+            if (!membGarbage->value.IsInt()) {
+                sprintf(errbuf, "Bank.messages[%d].garbage: expected an integer.", i);
+                throw runtime_error(errbuf);
+            }
+
+            message.resize(membGarbage->value.GetInt(), ' ');
+            goto enqueue_message;
+        }
+
+        if (membText == entry.MemberEnd()) {
+            sprintf(errbuf, "Bank.messages[%d]: missing required member `en_US`.", i);
+            throw runtime_error(errbuf);
+        } else if (!membText->value.IsString() && !membText->value.IsArray()) {
+            sprintf(errbuf, "Bank.messages[%d].en_US: expected a string or multi-line array of strings.", i);
+            throw runtime_error(errbuf);
+        }
+
+        if (membText->value.IsString()) {
+            message.assign(membText->value.GetString());
+        } else if (membText->value.IsArray()) {
+            for (const auto &line : membText->value.GetArray()) {
+                if (!line.IsString()) {
+                    goto content_error;
+                }
+
+                message.append(line.GetString());
+            }
+        } else {
+        content_error:
+            sprintf(errbuf, "Bank.messages[%d].en_US: expected a string or array of strings.", i);
+            throw runtime_error(errbuf);
+        }
+
+    enqueue_message:
+        converter.GetDecodedMessages().emplace_back(message); // emplace a copy
+        id_strings.emplace_back(membId->value.GetString());
+        messages.push_back(message);
+        i++;
+        IncRowNoBuf();
+    }
+
+    if (!converter.GetHeaderFilename().empty()) {
+        WriteHeader(converter.GetHeaderFilename());
+    }
+
+    return key;
+}
+
+void Json::ToFile(MessagesConverter &converter) {
+    if (!converter.GetHeaderFilename().empty()) {
+        ReadHeader(converter.GetHeaderFilename());
+    }
+
+    auto it = id_strings.cbegin();
+    doc.SetObject();
+    doc.AddMember("key", converter.GetKey(), doc.GetAllocator());
+
+    rapidjson::Value messages(rapidjson::kArrayType);
+
+    char keybuf[256];
+    string prefix = filename.substr(filename.find_last_of('/') + 1);
+    prefix = prefix.substr(0, prefix.find_first_of('.'));
+    for (const auto &message : converter.GetDecodedMessages()) {
+        rapidjson::Value entry_name(rapidjson::kStringType);
+        if (it != id_strings.cend()) {
+            entry_name.SetString(it->c_str(), doc.GetAllocator());
+        } else {
+            sprintf(keybuf, "%s_%s", prefix.c_str(), row_no_buf);
+            entry_name.SetString(keybuf, doc.GetAllocator());
+        }
+
+        rapidjson::Value entry(rapidjson::kObjectType);
+        entry.AddMember("id", entry_name, doc.GetAllocator());
+
+        if (message.find_first_not_of(' ') == string::npos) {
+            entry.AddMember("garbage", message.size(), doc.GetAllocator());
+        } else {
+            vector<string> message_lines = SplitMessage(message, true);
+            if (message_lines.size() == 1) {
+                rapidjson::Value entry_message(rapidjson::kStringType);
+                entry_message.SetString(message.c_str(), message.size(), doc.GetAllocator());
+                entry.AddMember("en_US", entry_message, doc.GetAllocator());
+            } else {
+                rapidjson::Value entry_lines(rapidjson::kArrayType);
+                for (const auto& line : message_lines) {
+                    rapidjson::Value entry_line(rapidjson::kStringType);
+                    entry_line.SetString(line.c_str(), line.size(), doc.GetAllocator());
+                    entry_lines.PushBack(entry_line, doc.GetAllocator());
+                }
+                entry.AddMember("en_US", entry_lines, doc.GetAllocator());
+            }
+        }
+
+        messages.PushBack(entry, doc.GetAllocator());
+        IncRowNoBuf();
+        it++;
+    }
+
+    doc.AddMember("messages", messages, doc.GetAllocator());
+
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    doc.Accept(writer);
+
+    ofstream jstrm(filename);
+    jstrm << buffer.GetString() << endl;
+}
+
+vector<string> Json::SplitMessage(const string &message, bool preserve) {
+    vector<string> v;
+
+    size_t start = 0, i = 0;
+    for (; i < message.size(); i++) {
+        if (message[i] == '\r' || message[i] == '\n' || message[i] == '\f') {
+            if (preserve) {
+                v.push_back(message.substr(start, i - start + 1));
+            } else {
+                v.push_back(message.substr(start, i - start));
+            }
+
+            start = i + 1;
+        }
+    }
+
+    if (start < message.size()) {
+        if (preserve) {
+            v.push_back(message.substr(start, i - start + 1));
+        } else {
+            v.push_back(message.substr(start, i - start));
+        }
+    }
+
+    return v;
+}

tools/msgenc/Json.h

@@ -0,0 +1,58 @@
+#ifndef GUARD_JSON_H
+#define GUARD_JSON_H
+
+// JSON-encoded structures are a port of the original GMM structure previously
+// used by the project. A valid JSON file contains a numeric key, and all other
+// keys should be interpreted in-order as the entries for a text bank. Each
+// bank entry is sub-divided into locales to allow for future multi-language
+// extensions.
+
+#include <cstring>
+#include <string>
+#include <vector>
+
+#include "rapidjson/document.h"
+
+#include "MessagesConverter.h"
+
+#define JSON_KEY_NOT_DEFINED -1
+
+class Json {
+    static constexpr int _row_no_buf_ndigit = 5;
+
+    char row_no_buf[_row_no_buf_ndigit + 1] = {0};
+    string filename;
+    rapidjson::Document doc;
+    vector<string> id_strings;
+    vector<string> messages;
+
+    void WriteHeader(const string &_filename);
+    void ReadHeader(const string &_filename);
+    void IncRowNoBuf() {
+        for (int i = _row_no_buf_ndigit - 1; i >= 0; i--) {
+            row_no_buf[i]++;
+            if (row_no_buf[i] > '9') {
+                if (i == 0) {
+                    throw runtime_error("message count overflow");
+                }
+
+                row_no_buf[i] = '0';
+            } else {
+                break;
+            }
+        }
+    }
+
+public:
+    Json(string &_filename, ios::openmode _openmode) : filename(_filename) {
+        memset(row_no_buf, '0', _row_no_buf_ndigit);
+    }
+
+    int FromFile(MessagesConverter &converter);
+    void ToFile(MessagesConverter &converter);
+
+private:
+    vector<string> SplitMessage(const string &message, bool preserve = false);
+};
+
+#endif // GUARD_JSON_H

tools/msgenc/MessagesConverter.cpp

@@ -61,6 +61,21 @@ void MessagesConverter::ReadCharmap() {
         if (code[0] == '{' && code[code.length() - 1] == '}') {
             code = code.substr(1, code.length() - 2);
             CmdmapRegisterCommand(code, value_i);
+        } else if (code[0] == '\\' && code.length() == 2) {
+            string literal;
+            switch (code[1]) {
+            case 'n': literal = '\n'; break;
+            case 'r': literal = '\r'; break;
+            case 'f': literal = '\f'; break;
+
+            default:
+                stringstream s;
+                s << "charmap syntax error at " << (lineno + 1);
+                throw runtime_error(s.str());
+            }
+
+            CharmapRegisterCharacter(code, value_i);
+            CharmapRegisterCharacter(literal, value_i);
         } else {
             CharmapRegisterCharacter(code, value_i);
         }

tools/msgenc/MessagesConverter.h

@@ -58,6 +58,7 @@ protected:
     string charmapfilename;
     string binfilename;
     string headerfilename;
+    TextFormat text_format = FMT_PLAIN_TEXT;
 
     MsgArcHeader header = {};
     vector<MsgAlloc> alloc_table;
@@ -72,19 +73,12 @@ protected:
     static string ReadTextFile(string& filename);
     static void WriteTextFile(string& filename, string const & contents);
 
-public:
-    typedef int txtfmt;
-    static const txtfmt PlainText = 0;
-    static const txtfmt GamefreakGMM = 1;
-protected:
-    txtfmt text_format = PlainText;
-
 public:
     MessagesConverter(Options &options) :
-        textfilename(options.posargs[0]),
-        binfilename(options.posargs[1]),
         mode(options.mode),
+        textfilename(options.posargs[0]),
         charmapfilename(options.charmap),
+        binfilename(options.posargs[1]),
         headerfilename(options.gmm_header),
         text_format(options.textFormat)
     {

tools/msgenc/MessagesDecoder.cpp

@@ -1,6 +1,8 @@
 #include <algorithm>
 #include "MessagesDecoder.h"
 #include "Gmm.h"
+#include "Json.h"
+#include "Options.h"
 
 void MessagesDecoder::CmdmapRegisterCommand(string &command, uint16_t value)
 {
@@ -168,6 +170,10 @@ void MessagesDecoder::WriteMessagesToGMM(string &filename) {
     GMM(filename, std::ios::out).ToFile(*this);
 }
 
+void MessagesDecoder::WriteMessagesToJson(string &filename) {
+    Json(filename, std::ios::out).ToFile(*this);
+}
+
 // Public virtual functions
 
 void MessagesDecoder::ReadInput()
@@ -187,11 +193,14 @@ void MessagesDecoder::Convert()
 void MessagesDecoder::WriteOutput()
 {
     switch (text_format) {
-    case PlainText:
+    case FMT_PLAIN_TEXT:
         WriteMessagesToText(textfilename);
         break;
-    case GamefreakGMM:
+    case FMT_GAMEFREAK_GMM:
         WriteMessagesToGMM(textfilename);
         break;
+    case FMT_JSON:
+        WriteMessagesToJson(textfilename);
+        break;
     }
 }

tools/msgenc/MessagesDecoder.h

@@ -19,6 +19,7 @@ class MessagesDecoder : public MessagesConverter
     void ReadMessagesFromBin(string& filename);
     void WriteMessagesToText(string& filename);
     void WriteMessagesToGMM(string& filename);
+    void WriteMessagesToJson(string& filename);
     template <typename T> void WriteBinaryFile(string& filename, T& data);
     static u16string DecodeTrainerNameMessage(u16string const &message);
     string DecodeMessage(u16string& message, int& i);

tools/msgenc/MessagesEncoder.cpp

@@ -1,5 +1,7 @@
 #include "MessagesEncoder.h"
 #include "Gmm.h"
+#include "Json.h"
+#include "Options.h"
 
 void MessagesEncoder::CmdmapRegisterCommand(string &command, uint16_t value)
 {
@@ -42,6 +44,17 @@ void MessagesEncoder::ReadMessagesFromGMM(string& filename) {
     debug_printf("%d lines\n", header.count);
 }
 
+void MessagesEncoder::ReadMessagesFromJson(string& filename) {
+    int key_from_file = Json(filename, std::ios::in).FromFile(*this);
+
+    if (key_from_file != JSON_KEY_NOT_DEFINED) {
+        header.key = key_from_file;
+    }
+
+    header.count = vec_decoded.size();
+    debug_printf("%d lines\n", header.count);
+}
+
 u16string MessagesEncoder::EncodeMessage(const string & message, int & i) {
     u16string encoded;
     bool is_trname = false;
@@ -158,12 +171,15 @@ void MessagesEncoder::WriteMessagesToBin(string& filename) {
 void MessagesEncoder::ReadInput()
 {
     switch (text_format) {
-    case PlainText:
+    case FMT_PLAIN_TEXT:
         ReadMessagesFromText(textfilename);
         break;
-    case GamefreakGMM:
+    case FMT_GAMEFREAK_GMM:
         ReadMessagesFromGMM(textfilename);
         break;
+    case FMT_JSON:
+        ReadMessagesFromJson(textfilename);
+        break;
     }
 }
 

tools/msgenc/MessagesEncoder.h

@@ -11,6 +11,7 @@ class MessagesEncoder : public MessagesConverter
 
     void ReadMessagesFromText(string& filename);
     void ReadMessagesFromGMM(string& filename);
+    void ReadMessagesFromJson(string& filename);
     void WriteMessagesToBin(string& filename);
     u16string EncodeMessage(const string& message, int & i);
     void CharmapRegisterCharacter(string& code, uint16_t value) override;

tools/msgenc/Options.cpp

@@ -1,5 +1,7 @@
 #include "Options.h"
 
+static string POSARGS[] = {"TEXTFILE", "BINFILE"};
+
 Options::Options(int argc, char **argv) {
     for (int i = 1; i < argc; i++) {
         string arg(argv[i]);
@@ -24,8 +26,10 @@ Options::Options(int argc, char **argv) {
             charmap = argv[++i];
         } else if (arg == "-D") {
             dumpBinary = argv[++i];
+        } else if (arg == "--json") {
+            textFormat = FMT_JSON;
         } else if (arg == "--gmm") {
-            textFormat = GamefreakGMM;
+            textFormat = FMT_GAMEFREAK_GMM;
         } else if (arg[0] != '-') {
             posargs.push_back(arg);
         } else {
@@ -34,7 +38,7 @@ Options::Options(int argc, char **argv) {
         }
     }
     if (posargs.size() < 2) {
-        failReason = "missing required positional argument: " + (string[]){"INFILE", "OUTFILE"}[posargs.size()];
+        failReason = "missing required positional argument: " + POSARGS[posargs.size()];
     }
     if (mode == CONV_INVALID) {
         failReason = "missing mode flag: -d or -e is required";

tools/msgenc/Options.h

@@ -14,6 +14,12 @@ enum ConvertMode : uint8_t {
     CONV_INVALID = 0xFF,
 };
 
+enum TextFormat : uint8_t {
+    FMT_PLAIN_TEXT = 0,
+    FMT_GAMEFREAK_GMM,
+    FMT_JSON,
+};
+
 struct Options {
     ConvertMode mode = CONV_INVALID;
     int key = 0;
@@ -24,10 +30,7 @@ struct Options {
     bool printVersion = false;
     string dumpBinary;
     string gmm_header = "";
-    typedef int txtfmt;
-    static const txtfmt PlainText = 0;
-    static const txtfmt GamefreakGMM = 1;
-    txtfmt textFormat = PlainText;
+    TextFormat textFormat = FMT_PLAIN_TEXT;
     Options(int argc, char ** argv);
 };
 

tools/msgenc/meson.build

@@ -8,11 +8,20 @@ msgenc_exe = executable('msgenc',
         'MessagesDecoder.cpp',
         'MessagesEncoder.cpp',
         'Gmm.cpp',
+        'Json.cpp',
         'pugixml.cpp'
     ],
     cpp_args: [
         '-DNDEBUG',
-        '-std=c++17'
+        '-g',
+        '-std=c++17',
+        '-Wall',
+        '-Wextra',
+        '-Wpedantic',
+        '-Wno-unused-parameter',
     ],
-    native: true
+    native: true,
+    dependencies: [
+        rapidjson_dep,
+    ]
 )

tools/msgenc/msgenc.cpp

@@ -11,18 +11,19 @@
 #include "Options.h"
 
 static const char* progname = "msgenc";
-static const char* version = "2021.12.21";
+static const char* version = "2025.08.12";
 
 static inline void usage() {
     cout << progname << " v" << version << endl;
-    cout << "Usage: " << progname << " [-h] [-v] -d|-e [OPTIONS] -c CHARMAP INFILE OUTFILE" << endl;
+    cout << "Usage: " << progname << " [-h] [-v] -d|-e [OPTIONS] -c CHARMAP TEXTFILE BINFILE" << endl;
     cout << endl;
-    cout << "INFILE        Required: Path to the input file to convert (-e: plaintext; -d: binary)." << endl;
-    cout << "OUTFILE       Required: Path to the output file (-e: binary; -d: plaintext)." << endl;
+    cout << "TEXTFILE      Required: Path to the text file (-e: input; -d: output)." << endl;
+    cout << "BINFILE       Required: Path to the binary file (ie: output; -d: input)." << endl;
     cout << "-c CHARMAP    Required: Path to a text file with a character mapping, for example pokeheartgold/charmap.txt." << endl;
     cout << "-d            Decode from binary to text, also print the key" << endl;
     cout << "-e            Encode from text to binary using the provided key" << endl;
     cout << "--gmm         Text file is GMM (Gamefreak XML format)" << endl;
+    cout << "--json        Text file is JSON" << endl;
     cout << "-H HEADER     When operating in GMM mode, specify this header file to read/write C constant values" << endl;
     cout << "-k KEY        The 16-bit encryption key for this message bank. Default: computes it from the binary file name" << endl;
     cout << "-v            Print the program version and exit." << endl;
@@ -73,8 +74,10 @@ int do_main(MessagesConverter* &converter, int argc, char ** argv) {
 }
 
 int main(int argc, char ** argv) {
-    MessagesConverter *converter;
+    MessagesConverter *converter = nullptr;
     int result = do_main(converter, argc, argv);
-    delete converter;
+
+    if (converter)
+        delete converter;
     return result;
 }