mirror of
https://github.com/huderlem/porymap.git
synced 2026-04-03 16:04:55 -05:00
Add file cache to ParserUtil
This commit is contained in:
GriffinR
parent
d42887e815
commit
ef6eb69c72
|
|
@ -45,7 +45,8 @@ public:
|
|||
ParseUtil();
|
||||
void set_root(const QString &dir);
|
||||
static QString readTextFile(const QString &path, QString *error = nullptr);
|
||||
void invalidateTextFile(const QString &path);
|
||||
bool cacheFile(const QString &path, QString *error = nullptr);
|
||||
void clearFileCache() { this->fileCache.clear(); }
|
||||
static int textFileLineCount(const QString &path);
|
||||
QList<QStringList> parseAsm(const QString &filename);
|
||||
QStringList readCArray(const QString &filename, const QString &label);
|
||||
|
|
@ -87,6 +88,7 @@ private:
|
|||
QString text;
|
||||
QString file;
|
||||
QString curDefine;
|
||||
QHash<QString, QString> fileCache;
|
||||
QHash<QString, QStringList> errorMap;
|
||||
int evaluateDefine(const QString&, const QString &, QMap<QString, int>*, QMap<QString, QString>*);
|
||||
QList<Token> tokenizeExpression(QString, QMap<QString, int>*, QMap<QString, QString>*);
|
||||
|
|
@ -105,6 +107,8 @@ private:
|
|||
QMap<QString, int> evaluateCDefines(const QString &filename, const QSet<QString> &filterList, bool useRegex, QString *error);
|
||||
bool defineNameMatchesFilter(const QString &name, const QSet<QString> &filterList) const;
|
||||
bool defineNameMatchesFilter(const QString &name, const QSet<QRegularExpression> &filterList) const;
|
||||
QString loadTextFile(const QString &path, QString *error = nullptr);
|
||||
QString pathWithRoot(const QString &path);
|
||||
|
||||
static const QRegularExpression re_incScriptLabel;
|
||||
static const QRegularExpression re_globalIncScriptLabel;
|
||||
|
|
|
|||
|
|
@ -279,6 +279,7 @@ private:
|
|||
|
||||
void ignoreWatchedFileTemporarily(QString filepath);
|
||||
void recordFileChange(const QString &filepath);
|
||||
void resetFileCache();
|
||||
|
||||
QString findSpeciesIconPath(const QStringList &names) const;
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,12 @@ void ParseUtil::set_root(const QString &dir) {
|
|||
this->root = dir;
|
||||
}
|
||||
|
||||
QString ParseUtil::pathWithRoot(const QString &path) {
|
||||
if (this->root.isEmpty()) return path;
|
||||
if (path.startsWith(this->root)) return path;
|
||||
return QString("%1/%2").arg(this->root).arg(path);
|
||||
}
|
||||
|
||||
void ParseUtil::recordError(const QString &message) {
|
||||
this->errorMap[this->curDefine].append(message);
|
||||
}
|
||||
|
|
@ -85,6 +91,34 @@ QString ParseUtil::readTextFile(const QString &path, QString *error) {
|
|||
return text;
|
||||
}
|
||||
|
||||
// Load the specified text file, either from the cache or by reading the file.
|
||||
// Note that this doesn't insert any parsed files into the file cache, and we don't
|
||||
// want it to (we read a lot of files only once, storing them all is a waste of memory).
|
||||
QString ParseUtil::loadTextFile(const QString &path, QString *error) {
|
||||
auto it = this->fileCache.constFind(path);
|
||||
if (it != this->fileCache.constEnd()) {
|
||||
// Load text file from cache
|
||||
//logWarn(QString("CACHE HIT ON %1").arg(path));
|
||||
return it.value();
|
||||
}
|
||||
|
||||
/* TODO: Remove
|
||||
static QSet<QString> parsedFiles;
|
||||
if (parsedFiles.contains(path)) {
|
||||
logWarn(QString("CACHE MISS ON %1").arg(path));
|
||||
} else {
|
||||
parsedFiles.insert(path);
|
||||
}
|
||||
*/
|
||||
|
||||
return readTextFile(pathWithRoot(path), error);
|
||||
}
|
||||
|
||||
bool ParseUtil::cacheFile(const QString &path, QString *error) {
|
||||
this->fileCache.insert(path, readTextFile(pathWithRoot(path), error));
|
||||
return !error || error->isEmpty();
|
||||
}
|
||||
|
||||
int ParseUtil::textFileLineCount(const QString &path) {
|
||||
const QString text = readTextFile(path);
|
||||
return text.split('\n').count() + 1;
|
||||
|
|
@ -93,7 +127,7 @@ int ParseUtil::textFileLineCount(const QString &path) {
|
|||
QList<QStringList> ParseUtil::parseAsm(const QString &filename) {
|
||||
QList<QStringList> parsed;
|
||||
|
||||
this->text = readTextFile(this->root + '/' + filename);
|
||||
this->text = loadTextFile(filename);
|
||||
const QStringList lines = removeLineComments(this->text, "@").split('\n');
|
||||
for (const auto &line : lines) {
|
||||
const QString trimmedLine = line.trimmed();
|
||||
|
|
@ -295,7 +329,7 @@ QString ParseUtil::readCIncbin(const QString &filename, const QString &label) {
|
|||
return path;
|
||||
}
|
||||
|
||||
this->text = readTextFile(this->root + "/" + filename);
|
||||
this->text = loadTextFile(filename);
|
||||
|
||||
QRegularExpression re(QString(
|
||||
"\\b%1\\b"
|
||||
|
|
@ -316,7 +350,7 @@ QMap<QString, QString> ParseUtil::readCIncbinMulti(const QString &filepath) {
|
|||
QMap<QString, QString> incbinMap;
|
||||
|
||||
this->file = filepath;
|
||||
this->text = readTextFile(this->root + "/" + filepath);
|
||||
this->text = loadTextFile(filepath);
|
||||
|
||||
static const QRegularExpression regex("(?<label>[A-Za-z0-9_]+)\\s*\\[?\\s*\\]?\\s*=\\s*INCBIN_[US][0-9][0-9]?\\(\\s*\\\"(?<path>[^\\\\\"]*)\\\"\\s*\\)");
|
||||
|
||||
|
|
@ -338,7 +372,7 @@ QStringList ParseUtil::readCIncbinArray(const QString &filename, const QString &
|
|||
return paths;
|
||||
}
|
||||
|
||||
this->text = readTextFile(this->root + "/" + filename);
|
||||
this->text = loadTextFile(filename);
|
||||
|
||||
bool found = false;
|
||||
QString arrayText;
|
||||
|
|
@ -388,8 +422,7 @@ ParseUtil::ParsedDefines ParseUtil::readCDefines(const QString &filename, const
|
|||
return result;
|
||||
}
|
||||
|
||||
QString filepath = this->root + "/" + this->file;
|
||||
this->text = readTextFile(filepath, error);
|
||||
this->text = loadTextFile(filename, error);
|
||||
if (this->text.isNull())
|
||||
return result;
|
||||
|
||||
|
|
@ -507,7 +540,7 @@ QStringList ParseUtil::readCArray(const QString &filename, const QString &label)
|
|||
}
|
||||
|
||||
this->file = filename;
|
||||
this->text = readTextFile(this->root + "/" + filename);
|
||||
this->text = loadTextFile(filename);
|
||||
|
||||
QRegularExpression re(QString(R"(\b%1\b\s*(\[?[^\]]*\])?\s*=\s*\{([^\}]*)\})").arg(label));
|
||||
QRegularExpressionMatch match = re.match(this->text);
|
||||
|
|
@ -530,7 +563,7 @@ QMap<QString, QStringList> ParseUtil::readCArrayMulti(const QString &filename) {
|
|||
QMap<QString, QStringList> map;
|
||||
|
||||
this->file = filename;
|
||||
this->text = readTextFile(this->root + "/" + filename);
|
||||
this->text = loadTextFile(filename);
|
||||
|
||||
static const QRegularExpression regex(R"((?<label>\b[A-Za-z0-9_]+\b)\s*(\[[^\]]*\])?\s*=\s*\{(?<body>[^\}]*)\})");
|
||||
|
||||
|
|
@ -556,7 +589,7 @@ QMap<QString, QStringList> ParseUtil::readCArrayMulti(const QString &filename) {
|
|||
}
|
||||
|
||||
QMap<QString, QString> ParseUtil::readNamedIndexCArray(const QString &filename, const QString &label, QString *error) {
|
||||
this->text = readTextFile(this->root + "/" + filename, error);
|
||||
this->text = loadTextFile(filename, error);
|
||||
QMap<QString, QString> map;
|
||||
|
||||
QRegularExpression re_text(QString(R"(\b%1\b\s*(\[?[^\]]*\])?\s*=\s*\{([^\}]*)\})").arg(label));
|
||||
|
|
@ -589,7 +622,7 @@ bool ParseUtil::gameStringToBool(const QString &gameString, bool * ok) {
|
|||
}
|
||||
|
||||
tsl::ordered_map<QString, QHash<QString, QString>> ParseUtil::readCStructs(const QString &filename, const QString &label, const QHash<int, QString> &memberMap) {
|
||||
QString filePath = this->root + "/" + filename;
|
||||
QString filePath = pathWithRoot(filename);
|
||||
auto cParser = fex::Parser();
|
||||
auto tokens = fex::Lexer().LexFile(filePath);
|
||||
auto topLevelObjects = cParser.ParseTopLevelObjects(tokens);
|
||||
|
|
@ -657,7 +690,7 @@ QStringList ParseUtil::getLabelValues(const QList<QStringList> &list, const QStr
|
|||
}
|
||||
|
||||
bool ParseUtil::tryParseJsonFile(QJsonDocument *out, const QString &filepath, QString *error) {
|
||||
QFile file(filepath);
|
||||
QFile file(pathWithRoot(filepath));
|
||||
if (!file.open(QIODevice::ReadOnly)) {
|
||||
if (error) *error = file.errorString();
|
||||
return false;
|
||||
|
|
@ -678,7 +711,7 @@ bool ParseUtil::tryParseJsonFile(QJsonDocument *out, const QString &filepath, QS
|
|||
|
||||
bool ParseUtil::tryParseOrderedJsonFile(poryjson::Json::object *out, const QString &filepath, QString *error) {
|
||||
QString err;
|
||||
QString jsonTxt = readTextFile(filepath, error);
|
||||
QString jsonTxt = loadTextFile(filepath, error);
|
||||
if (error && !error->isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ bool Project::sanityCheck() {
|
|||
}
|
||||
|
||||
bool Project::load() {
|
||||
resetFileCache();
|
||||
this->disabledSettingsNames.clear();
|
||||
bool success = readMapLayouts()
|
||||
&& readRegionMapSections()
|
||||
|
|
@ -119,6 +120,26 @@ bool Project::load() {
|
|||
return success;
|
||||
}
|
||||
|
||||
void Project::resetFileCache() {
|
||||
this->parser.clearFileCache();
|
||||
|
||||
const QSet<QString> filepaths = {
|
||||
// Whenever we load a tileset we'll need to parse some data from these files, so we cache them to avoid the overhead of opening the files.
|
||||
// We don't know yet whether the project uses C or asm tileset data, so try to cache both (we'll ignore errors from missing files).
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_headers_asm),
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_graphics_asm),
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_metatiles_asm),
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_headers),
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_graphics),
|
||||
projectConfig.getFilePath(ProjectFilePath::tilesets_metatiles),
|
||||
// We need separate sets of constants from these files
|
||||
projectConfig.getFilePath(ProjectFilePath::constants_map_types),
|
||||
};
|
||||
for (const auto &path : filepaths) {
|
||||
this->parser.cacheFile(path);
|
||||
}
|
||||
}
|
||||
|
||||
QString Project::getProjectTitle() const {
|
||||
if (!root.isNull()) {
|
||||
return root.section('/', -1);
|
||||
|
|
@ -200,7 +221,7 @@ void Project::initTopLevelMapFields() {
|
|||
bool Project::readMapJson(const QString &mapName, QJsonDocument * out) {
|
||||
const QString mapFilepath = QString("%1%2/map.json").arg(projectConfig.getFilePath(ProjectFilePath::data_map_folders)).arg(mapName);
|
||||
QString error;
|
||||
if (!parser.tryParseJsonFile(out, QString("%1/%2").arg(this->root).arg(mapFilepath), &error)) {
|
||||
if (!parser.tryParseJsonFile(out, mapFilepath, &error)) {
|
||||
logError(QString("Failed to read map data from '%1': %2").arg(mapFilepath).arg(error));
|
||||
return false;
|
||||
}
|
||||
|
|
@ -477,12 +498,11 @@ bool Project::readMapLayouts() {
|
|||
clearMapLayouts();
|
||||
|
||||
const QString layoutsFilepath = projectConfig.getFilePath(ProjectFilePath::json_layouts);
|
||||
const QString fullFilepath = QString("%1/%2").arg(this->root).arg(layoutsFilepath);
|
||||
fileWatcher.addPath(fullFilepath);
|
||||
fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(layoutsFilepath));
|
||||
QJsonDocument layoutsDoc;
|
||||
QString error;
|
||||
if (!parser.tryParseJsonFile(&layoutsDoc, fullFilepath, &error)) {
|
||||
logError(QString("Failed to read map layouts from '%1': %2").arg(fullFilepath).arg(error));
|
||||
if (!parser.tryParseJsonFile(&layoutsDoc, layoutsFilepath, &error)) {
|
||||
logError(QString("Failed to read map layouts from '%1': %2").arg(layoutsFilepath).arg(error));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -979,9 +999,6 @@ bool Project::loadLayoutTilesets(Layout *layout) {
|
|||
return true;
|
||||
}
|
||||
|
||||
// TODO: We are parsing the tileset headers file whenever we load a tileset for the first time.
|
||||
// At a minimum this means we're parsing the file three times per session (twice here for the first map's tilesets, once on launch in Project::readTilesetLabels).
|
||||
// We can cache the header data instead and only parse it once on launch.
|
||||
Tileset* Project::loadTileset(QString label, Tileset *tileset) {
|
||||
auto memberMap = Tileset::getHeaderMemberMap(this->usingAsmTilesets);
|
||||
if (this->usingAsmTilesets) {
|
||||
|
|
@ -1634,15 +1651,14 @@ bool Project::readWildMonData() {
|
|||
this->pokemonMaxLevel = qMax(this->pokemonMinLevel, this->pokemonMaxLevel);
|
||||
|
||||
// Read encounter data
|
||||
const QString wildMonJsonBaseFilepath = projectConfig.getFilePath(ProjectFilePath::json_wild_encounters);
|
||||
QString wildMonJsonFilepath = QString("%1/%2").arg(root).arg(wildMonJsonBaseFilepath);
|
||||
fileWatcher.addPath(wildMonJsonFilepath);
|
||||
const QString wildMonJsonFilepath = projectConfig.getFilePath(ProjectFilePath::json_wild_encounters);
|
||||
fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(wildMonJsonFilepath));
|
||||
|
||||
OrderedJson::object wildMonObj;
|
||||
QString error;
|
||||
if (!parser.tryParseOrderedJsonFile(&wildMonObj, wildMonJsonFilepath, &error)) {
|
||||
// Failing to read wild encounters data is not a critical error, the encounter editor will just be disabled
|
||||
logWarn(QString("Failed to read wild encounters from '%1': %2").arg(wildMonJsonBaseFilepath).arg(error));
|
||||
logWarn(QString("Failed to read wild encounters from '%1': %2").arg(wildMonJsonFilepath).arg(error));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -1768,8 +1784,8 @@ bool Project::readMapGroups() {
|
|||
|
||||
this->initTopLevelMapFields();
|
||||
|
||||
const QString filepath = root + "/" + projectConfig.getFilePath(ProjectFilePath::json_map_groups);
|
||||
fileWatcher.addPath(filepath);
|
||||
const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_map_groups);
|
||||
fileWatcher.addPath(root + "/" + filepath);
|
||||
QJsonDocument mapGroupsDoc;
|
||||
QString error;
|
||||
if (!parser.tryParseJsonFile(&mapGroupsDoc, filepath, &error)) {
|
||||
|
|
@ -2277,14 +2293,13 @@ bool Project::readRegionMapSections() {
|
|||
const QString requiredPrefix = projectConfig.getIdentifier(ProjectIdentifier::define_map_section_prefix);
|
||||
|
||||
QJsonDocument doc;
|
||||
const QString baseFilepath = projectConfig.getFilePath(ProjectFilePath::json_region_map_entries);
|
||||
const QString filepath = QString("%1/%2").arg(this->root).arg(baseFilepath);
|
||||
const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_region_map_entries);
|
||||
QString error;
|
||||
if (!parser.tryParseJsonFile(&doc, filepath, &error)) {
|
||||
logError(QString("Failed to read region map sections from '%1': %2").arg(baseFilepath).arg(error));
|
||||
logError(QString("Failed to read region map sections from '%1': %2").arg(filepath).arg(error));
|
||||
return false;
|
||||
}
|
||||
fileWatcher.addPath(filepath);
|
||||
fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(filepath));
|
||||
|
||||
QJsonArray mapSections = doc.object()["map_sections"].toArray();
|
||||
for (int i = 0; i < mapSections.size(); i++) {
|
||||
|
|
@ -2300,13 +2315,13 @@ bool Project::readRegionMapSections() {
|
|||
// ignoring everything here and then wiping the file's data when we save later.
|
||||
idField = oldIdField;
|
||||
} else {
|
||||
logWarn(QString("Ignoring data for map section %1 in '%2'. Missing required field \"%3\"").arg(i).arg(baseFilepath).arg(idField));
|
||||
logWarn(QString("Ignoring data for map section %1 in '%2'. Missing required field \"%3\"").arg(i).arg(filepath).arg(idField));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const QString idName = ParseUtil::jsonToQString(mapSectionObj[idField]);
|
||||
if (!idName.startsWith(requiredPrefix)) {
|
||||
logWarn(QString("Ignoring data for map section '%1' in '%2'. IDs must start with the prefix '%3'").arg(idName).arg(baseFilepath).arg(requiredPrefix));
|
||||
logWarn(QString("Ignoring data for map section '%1' in '%2'. IDs must start with the prefix '%3'").arg(idName).arg(filepath).arg(requiredPrefix));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -2404,21 +2419,20 @@ bool Project::readHealLocations() {
|
|||
clearHealLocations();
|
||||
|
||||
QJsonDocument doc;
|
||||
const QString baseFilepath = projectConfig.getFilePath(ProjectFilePath::json_heal_locations);
|
||||
const QString filepath = QString("%1/%2").arg(this->root).arg(baseFilepath);
|
||||
const QString filepath = projectConfig.getFilePath(ProjectFilePath::json_heal_locations);
|
||||
QString error;
|
||||
if (!parser.tryParseJsonFile(&doc, filepath, &error)) {
|
||||
logError(QString("Failed to read heal locations from '%1': %2").arg(baseFilepath).arg(error));
|
||||
logError(QString("Failed to read heal locations from '%1': %2").arg(filepath).arg(error));
|
||||
return false;
|
||||
}
|
||||
fileWatcher.addPath(filepath);
|
||||
fileWatcher.addPath(QString("%1/%2").arg(this->root).arg(filepath));
|
||||
|
||||
QJsonArray healLocations = doc.object()["heal_locations"].toArray();
|
||||
for (int i = 0; i < healLocations.size(); i++) {
|
||||
QJsonObject healLocationObj = healLocations.at(i).toObject();
|
||||
static const QString mapField = QStringLiteral("map");
|
||||
if (!healLocationObj.contains(mapField)) {
|
||||
logWarn(QString("Ignoring data for heal location %1 in '%2'. Missing required field \"%3\"").arg(i).arg(baseFilepath).arg(mapField));
|
||||
logWarn(QString("Ignoring data for heal location %1 in '%2'. Missing required field \"%3\"").arg(i).arg(filepath).arg(mapField));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user