From ef6eb69c72b2ff90cd343b87daed12d25b7cd52f Mon Sep 17 00:00:00 2001
From: GriffinR <griffin.g.richards@gmail.com>
Date: Wed, 19 Mar 2025 14:27:56 -0400
Subject: [PATCH] Add file cache to ParserUtil

---
 include/core/parseutil.h |  6 +++-
 include/project.h        |  1 +
 src/core/parseutil.cpp   | 57 +++++++++++++++++++++++++++--------
 src/project.cpp          | 64 ++++++++++++++++++++++++----------------
 4 files changed, 90 insertions(+), 38 deletions(-)

diff --git a/include/core/parseutil.h b/include/core/parseutil.h
index b5fc5dd2..4c19a27c 100644
--- a/include/core/parseutil.h
+++ b/include/core/parseutil.h
@@ -45,7 +45,8 @@ public:
     ParseUtil();
     void set_root(const QString &dir);
     static QString readTextFile(const QString &path, QString *error = nullptr);
-    void invalidateTextFile(const QString &path);
+    bool cacheFile(const QString &path, QString *error = nullptr);
+    void clearFileCache() { this->fileCache.clear(); }
     static int textFileLineCount(const QString &path);
     QList<QStringList> parseAsm(const QString &filename);
     QStringList readCArray(const QString &filename, const QString &label);
@@ -87,6 +88,7 @@ private:
     QString text;
     QString file;
     QString curDefine;
+    QHash<QString, QString> fileCache;
     QHash<QString, QStringList> errorMap;
     int evaluateDefine(const QString&, const QString &, QMap<QString, int>*, QMap<QString, QString>*);
     QList<Token> tokenizeExpression(QString, QMap<QString, int>*, QMap<QString, QString>*);
@@ -105,6 +107,8 @@ private:
     QMap<QString, int> evaluateCDefines(const QString &filename, const QSet<QString> &filterList, bool useRegex, QString *error);
     bool defineNameMatchesFilter(const QString &name, const QSet<QString> &filterList) const;
     bool defineNameMatchesFilter(const QString &name, const QSet<QRegularExpression> &filterList) const;
+    QString loadTextFile(const QString &path, QString *error = nullptr);
+    QString pathWithRoot(const QString &path);
 
     static const QRegularExpression re_incScriptLabel;
     static const QRegularExpression re_globalIncScriptLabel;
diff --git a/include/project.h b/include/project.h
index f7a4b5cc..5fcb579c 100644
--- a/include/project.h
+++ b/include/project.h
@@ -279,6 +279,7 @@ private:
 
     void ignoreWatchedFileTemporarily(QString filepath);
     void recordFileChange(const QString &filepath);
+    void resetFileCache();
 
     QString findSpeciesIconPath(const QStringList &names) const;
 
diff --git a/src/core/parseutil.cpp b/src/core/parseutil.cpp
index 22880562..62735db3 100644
--- a/src/core/parseutil.cpp
+++ b/src/core/parseutil.cpp
@@ -37,6 +37,12 @@ void ParseUtil::set_root(const QString &dir) {
     this->root = dir;
 }
 
+QString ParseUtil::pathWithRoot(const QString &path) {
+    if (this->root.isEmpty()) return path;
+    if (path.startsWith(this->root)) return path;
+    return QString("%1/%2").arg(this->root).arg(path);
+}
+
 void ParseUtil::recordError(const QString &message) {
     this->errorMap[this->curDefine].append(message);
 }
@@ -85,6 +91,34 @@ QString ParseUtil::readTextFile(const QString &path, QString *error) {
     return text;
 }
 
+// Load the specified text file, either from the cache or by reading the file.
+// Note that this doesn't insert any parsed files into the file cache, and we don't
+// want it to (we read a lot of files only once, storing them all is a waste of memory).
+QString ParseUtil::loadTextFile(const QString &path, QString *error) {
+    auto it = this->fileCache.constFind(path);
+    if (it != this->fileCache.constEnd()) {
+        // Load text file from cache
+        //logWarn(QString("CACHE HIT ON %1").arg(path));
+        return it.value();
+    }
+
+/* TODO: Remove
+    static QSet<QString> parsedFiles;
+    if (parsedFiles.contains(path)) {
+        logWarn(QString("CACHE MISS ON %1").arg(path));
+    } else {
+        parsedFiles.insert(path);
+    }
+*/
+
+    return readTextFile(pathWithRoot(path), error);
+}
+
+bool ParseUtil::cacheFile(const QString &path, QString *error) {
+    this->fileCache.insert(path, readTextFile(pathWithRoot(path), error));
+    return !error || error->isEmpty();
+}
+
 int ParseUtil::textFileLineCount(const QString &path) {
     const QString text = readTextFile(path);
     return text.split('\n').count() + 1;
@@ -93,7 +127,7 @@ int ParseUtil::textFileLineCount(const QString &path) {
 QList<QStringList> ParseUtil::parseAsm(const QString &filename) {
     QList<QStringList> parsed;
 
-    this->text = readTextFile(this->root + '/' + filename);
+    this->text = loadTextFile(filename);
     const QStringList lines = removeLineComments(this->text, "@").split('\n');
     for (const auto &line : lines) {
         const QString trimmedLine = line.trimmed();
@@ -295,7 +329,7 @@ QString ParseUtil::readCIncbin(const QString &filename, const QString &label) {
         return path;
     }
 
-    this->text = readTextFile(this->root + "/" + filename);
+    this->text = loadTextFile(filename);
 
     QRegularExpression re(QString(
         "\\b%1\\b"
@@ -316,7 +350,7 @@ QMap<QString, QString> ParseUtil::readCIncbinMulti(const QString &filepath) {
     QMap<QString, QString> incbinMap;
 
     this->file = filepath;
-    this->text = readTextFile(this->root + "/" + filepath);
+    this->text = loadTextFile(filepath);
 
     static const QRegularExpression regex("(?<label>[A-Za-z0-9_]+)\\s*\\[?\\s*\\]?\\s*=\\s*INCBIN_[US][0-9][0-9]?\\(\\s*\\\"(?<path>[^\\\\\"]*)\\\"\\s*\\)");
 
@@ -338,7 +372,7 @@ QStringList ParseUtil::readCIncbinArray(const QString &filename, const QString &
         return paths;
     }
 
-    this->text = readTextFile(this->root + "/" + filename);
+    this->text = loadTextFile(filename);
 
     bool found = false;
     QString arrayText;
@@ -388,8 +422,7 @@ ParseUtil::ParsedDefines ParseUtil::readCDefines(const QString &filename, const
         return result;
     }
 
-    QString filepath = this->root + "/" + this->file;
-    this->text = readTextFile(filepath, error);
+    this->text = loadTextFile(filename, error);
     if (this->text.isNull())
         return result;
 
@@ -507,7 +540,7 @@ QStringList ParseUtil::readCArray(const QString &filename, const QString &label)
     }
 
     this->file = filename;
-    this->text = readTextFile(this->root + "/" + filename);
+    this->text = loadTextFile(filename);
 
     QRegularExpression re(QString(R"(\b%1\b\s*(\[?[^\]]*\])?\s*=\s*\{([^\}]*)\})").arg(label));
     QRegularExpressionMatch match = re.match(this->text);
@@ -530,7 +563,7 @@ QMap<QString, QStringList> ParseUtil::readCArrayMulti(const QString &filename) {
     QMap<QString, QStringList> map;
 
     this->file = filename;
-    this->text = readTextFile(this->root + "/" + filename);
+    this->text = loadTextFile(filename);
 
     static const QRegularExpression regex(R"((?<label>\b[A-Za-z0-9_]+\b)\s*(\[[^\]]*\])?\s*=\s*\{(?<body>[^\}]*)\})");
 
@@ -556,7 +589,7 @@ QMap<QString, QStringList> ParseUtil::readCArrayMulti(const QString &filename) {
 }
 
 QMap<QString, QString> ParseUtil::readNamedIndexCArray(const QString &filename, const QString &label, QString *error) {
-    this->text = readTextFile(this->root + "/" + filename, error);
+    this->text = loadTextFile(filename, error);
     QMap<QString, QString> map;
 
     QRegularExpression re_text(QString(R"(\b%1\b\s*(\[?[^\]]*\])?\s*=\s*\{([^\}]*)\})").arg(label));
@@ -589,7 +622,7 @@ bool ParseUtil::gameStringToBool(const QString &gameString, bool * ok) {
 }
 
 tsl::ordered_map<QString, QHash<QString, QString>> ParseUtil::readCStructs(const QString &filename, const QString &label, const QHash<int, QString> &memberMap) {
-    QString filePath = this->root + "/" + filename;
+    QString filePath = pathWithRoot(filename);
     auto cParser = fex::Parser();
     auto tokens = fex::Lexer().LexFile(filePath);
     auto topLevelObjects = cParser.ParseTopLevelObjects(tokens);
@@ -657,7 +690,7 @@ QStringList ParseUtil::getLabelValues(const QList<QStringList> &list, const QStr
 }
 
 bool ParseUtil::tryParseJsonFile(QJsonDocument *out, const QString &filepath, QString *error) {
-    QFile file(filepath);
+    QFile file(pathWithRoot(filepath));
     if (!file.open(QIODevice::ReadOnly)) {
         if (error) *error = file.errorString();
         return false;
@@ -678,7 +711,7 @@ bool ParseUtil::tryParseJsonFile(QJsonDocument *out, const QString &filepath, QS
 
 bool ParseUtil::tryParseOrderedJsonFile(poryjson::Json::object *out, const QString &filepath, QString *error) {
     QString err;
-    QString jsonTxt = readTextFile(filepath, error);
+    QString jsonTxt = loadTextFile(filepath, error);
     if (error && !error->isEmpty()) {
         return false;
     }
diff --git a/src/project.cpp b/src/project.cpp
index 6590aac7..ae2959a2 100644
--- a/src/project.cpp
+++ b/src/project.cpp
@@ -78,6 +78,7 @@ bool Project::sanityCheck() {
 }
 
 bool Project::load() {
+    resetFileCache();
     this->disabledSettingsNames.clear();
     bool success = readMapLayouts()
                 && readRegionMapSections()
@@ -119,6 +120,26 @@ bool Project::load() {
     return success;
 }
 
+void Project::resetFileCache() {
+    this->parser.clearFileCache();
+
+    const QSet<QString> filepaths = {
+        // Whenever we load a tileset we'll need to parse some data from these files, so we cache them to avoid the overhead of opening the files.
+        // We don't know yet whether the project uses C or asm tileset data, so try to cache both (we'll ignore errors from missing files).
+        projectConfig.getFilePath(ProjectFilePath::tilesets_headers_asm),
+        projectConfig.getFilePath(ProjectFilePath::tilesets_graphics_asm),
+        projectConfig.getFilePath(ProjectFilePath::tilesets_metatiles_asm),
+        projectConfig.getFilePath(ProjectFilePath::tilesets_headers),
+        projectConfig.getFilePath(ProjectFilePath::tilesets_graphics),
+        projectConfig.getFilePath(ProjectFilePath::tilesets_metatiles),
+        // We need separate sets of constants from these files
+        projectConfig.getFilePath(ProjectFilePath::constants_map_types),
+    };
+    for (const auto &path : filepaths) {
+        this->parser.cacheFile(path);
+    }
+}
+
 QString Project::getProjectTitle() const {
     if (!root.isNull()) {
         return root.section('/', -1);
@@ -200,7 +221,7 @@ void Project::initTopLevelMapFields() {
 bool Project::readMapJson(const QString &mapName, QJsonDocument * out) {
     const QString mapFilepath = QString("%1%2/map.json").arg(projectConfig.getFilePath(ProjectFilePath::data_map_folders)).arg(mapName);
     QString error;
-    if (!parser.tryParseJsonFile(out, QString("%1/%2").arg(this->root).arg(mapFilepath), &error)) {
+    if (!parser.tryParseJsonFile(out, mapFilepath, &error)) {
         logError(QString("Failed to read map data from '%1': %2").arg(mapFilepath).arg(error));
         return false;
     }
@@ -477,12 +498,11 @@ bool Project::readMapLayouts() {
     clearMapLayouts();
 
     const QString layoutsFilepath = projectConfig.getFilePath(ProjectFilePath::json_layouts);
-    const QString fullFilepath = QString("%1/%2").arg(this->root).arg(layoutsFilepath);
-    fileWatcher.addPath(fullFilepath);
+    fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(layoutsFilepath));
     QJsonDocument layoutsDoc;
     QString error;
-    if (!parser.tryParseJsonFile(&layoutsDoc, fullFilepath, &error)) {
-        logError(QString("Failed to read map layouts from '%1': %2").arg(fullFilepath).arg(error));
+    if (!parser.tryParseJsonFile(&layoutsDoc, layoutsFilepath, &error)) {
+        logError(QString("Failed to read map layouts from '%1': %2").arg(layoutsFilepath).arg(error));
         return false;
     }
 
@@ -979,9 +999,6 @@ bool Project::loadLayoutTilesets(Layout *layout) {
     return true;
 }
 
-// TODO: We are parsing the tileset headers file whenever we load a tileset for the first time.
-//       At a minimum this means we're parsing the file three times per session (twice here for the first map's tilesets, once on launch in Project::readTilesetLabels).
-//       We can cache the header data instead and only parse it once on launch.
 Tileset* Project::loadTileset(QString label, Tileset *tileset) {
     auto memberMap = Tileset::getHeaderMemberMap(this->usingAsmTilesets);
     if (this->usingAsmTilesets) {
@@ -1634,15 +1651,14 @@ bool Project::readWildMonData() {
     this->pokemonMaxLevel = qMax(this->pokemonMinLevel, this->pokemonMaxLevel);
 
     // Read encounter data
-    const QString wildMonJsonBaseFilepath = projectConfig.getFilePath(ProjectFilePath::json_wild_encounters);
-    QString wildMonJsonFilepath = QString("%1/%2").arg(root).arg(wildMonJsonBaseFilepath);
-    fileWatcher.addPath(wildMonJsonFilepath);
+    const QString wildMonJsonFilepath = projectConfig.getFilePath(ProjectFilePath::json_wild_encounters);
+    fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(wildMonJsonFilepath));
 
     OrderedJson::object wildMonObj;
     QString error;
     if (!parser.tryParseOrderedJsonFile(&wildMonObj, wildMonJsonFilepath, &error)) {
         // Failing to read wild encounters data is not a critical error, the encounter editor will just be disabled
-        logWarn(QString("Failed to read wild encounters from '%1': %2").arg(wildMonJsonBaseFilepath).arg(error));
+        logWarn(QString("Failed to read wild encounters from '%1': %2").arg(wildMonJsonFilepath).arg(error));
         return true;
     }
 
@@ -1768,8 +1784,8 @@ bool Project::readMapGroups() {
 
     this->initTopLevelMapFields();
 
-    const QString filepath = root + "/" + projectConfig.getFilePath(ProjectFilePath::json_map_groups);
-    fileWatcher.addPath(filepath);
+    const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_map_groups);
+    fileWatcher.addPath(root + "/" + filepath);
     QJsonDocument mapGroupsDoc;
     QString error;
     if (!parser.tryParseJsonFile(&mapGroupsDoc, filepath, &error)) {
@@ -2277,14 +2293,13 @@ bool Project::readRegionMapSections() {
     const QString requiredPrefix = projectConfig.getIdentifier(ProjectIdentifier::define_map_section_prefix);
 
     QJsonDocument doc;
-    const QString baseFilepath = projectConfig.getFilePath(ProjectFilePath::json_region_map_entries);
-    const QString filepath = QString("%1/%2").arg(this->root).arg(baseFilepath);
+    const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_region_map_entries);
     QString error;
     if (!parser.tryParseJsonFile(&doc, filepath, &error)) {
-        logError(QString("Failed to read region map sections from '%1': %2").arg(baseFilepath).arg(error));
+        logError(QString("Failed to read region map sections from '%1': %2").arg(filepath).arg(error));
         return false;
     }
-    fileWatcher.addPath(filepath);
+    fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(filepath));
 
     QJsonArray mapSections = doc.object()["map_sections"].toArray();
     for (int i = 0; i < mapSections.size(); i++) {
@@ -2300,13 +2315,13 @@ bool Project::readRegionMapSections() {
                 // ignoring everything here and then wiping the file's data when we save later.
                 idField = oldIdField;
             } else {
-                logWarn(QString("Ignoring data for map section %1 in '%2'. Missing required field \"%3\"").arg(i).arg(baseFilepath).arg(idField));
+                logWarn(QString("Ignoring data for map section %1 in '%2'. Missing required field \"%3\"").arg(i).arg(filepath).arg(idField));
                 continue;
             }
         }
         const QString idName = ParseUtil::jsonToQString(mapSectionObj[idField]);
         if (!idName.startsWith(requiredPrefix)) {
-            logWarn(QString("Ignoring data for map section '%1' in '%2'. IDs must start with the prefix '%3'").arg(idName).arg(baseFilepath).arg(requiredPrefix));
+            logWarn(QString("Ignoring data for map section '%1' in '%2'. IDs must start with the prefix '%3'").arg(idName).arg(filepath).arg(requiredPrefix));
             continue;
         }
 
@@ -2404,21 +2419,20 @@ bool Project::readHealLocations() {
     clearHealLocations();
 
     QJsonDocument doc;
-    const QString baseFilepath = projectConfig.getFilePath(ProjectFilePath::json_heal_locations);
-    const QString filepath = QString("%1/%2").arg(this->root).arg(baseFilepath);
+    const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_heal_locations);
     QString error;
     if (!parser.tryParseJsonFile(&doc, filepath, &error)) {
-        logError(QString("Failed to read heal locations from '%1': %2").arg(baseFilepath).arg(error));
+        logError(QString("Failed to read heal locations from '%1': %2").arg(filepath).arg(error));
         return false;
     }
-    fileWatcher.addPath(filepath);
+    fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(filepath));
 
     QJsonArray healLocations = doc.object()["heal_locations"].toArray();
     for (int i = 0; i < healLocations.size(); i++) {
         QJsonObject healLocationObj = healLocations.at(i).toObject();
         static const QString mapField = QStringLiteral("map");
         if (!healLocationObj.contains(mapField)) {
-            logWarn(QString("Ignoring data for heal location %1 in '%2'. Missing required field \"%3\"").arg(i).arg(baseFilepath).arg(mapField));
+            logWarn(QString("Ignoring data for heal location %1 in '%2'. Missing required field \"%3\"").arg(i).arg(filepath).arg(mapField));
             continue;
         }