Map MD5 hash names for afp/bsi/geo folders

.vscode/c_cpp_properties.json

@@ -9,7 +9,7 @@
             "compilerPath": "/usr/bin/i686-w64-mingw32-gcc",
             "cStandard": "c17",
             "cppStandard": "gnu++17",
-            "intelliSenseMode": "linux-gcc-x86",
+            "intelliSenseMode": "windows-gcc-x86",
             "compileCommands": "${workspaceFolder}/build32/compile_commands.json"
         },
         {
@@ -21,7 +21,7 @@
             "compilerPath": "/usr/bin/x86_64-w64-mingw32-gcc",
             "cStandard": "c17",
             "cppStandard": "gnu++17",
-            "intelliSenseMode": "linux-gcc-x64",
+            "intelliSenseMode": "windows-gcc-x64",
             "compileCommands": "${workspaceFolder}/build64/compile_commands.json"
         }
     ],

meson.build

@@ -1,4 +1,4 @@
-project('layeredfs', 'c', 'cpp', version: '3.5',
+project('layeredfs', 'c', 'cpp', version: '3.6_BETA',
     default_options: [
         'cpp_std=c++20',
         'buildtype=release',

src/avs_standalone.cpp

@@ -23,7 +23,7 @@ static void log_writer(const char *chars, uint32_t nchars, void *ctx);
     X("XCgsqzn0000159", void *, avs_filesys_ramfs)
 
 #define AVS_FUNC_PTR(obfus_name, ret_type, name, ...) ret_type (*name)(__VA_ARGS__);
-    FOREACH_EXTRA_FUNC(AVS_FUNC_PTR)
+FOREACH_EXTRA_FUNC(AVS_FUNC_PTR)
 
 static bool g_print_logs = true;
 static size_t g_boot_cfg_offset;

src/hook.cpp

@@ -59,7 +59,7 @@ class AvsHookFile : public HookFile {
         }
     }
 };
-class AvsOpenHookFile : public AvsHookFile {
+class AvsOpenHookFile final : public AvsHookFile {
     private:
     uint16_t mode;
     int flags;
@@ -79,7 +79,7 @@ class AvsOpenHookFile : public AvsHookFile {
     }
 };
 
-class AvsLstatHookFile : public AvsHookFile {
+class AvsLstatHookFile final : public AvsHookFile {
     private:
     struct avs_stat *st;
 
@@ -95,7 +95,7 @@ class AvsLstatHookFile : public AvsHookFile {
     }
 };
 
-class AvsConvertPathHookFile : public AvsHookFile {
+class AvsConvertPathHookFile final : public AvsHookFile {
     private:
     char *dest_name;
 
@@ -111,7 +111,7 @@ class AvsConvertPathHookFile : public AvsHookFile {
     }
 };
 
-class PkfsHookFile : public HookFile {
+class PkfsHookFile final : public HookFile {
     public:
     PkfsHookFile(const std::string path, const std::string norm_path)
         : HookFile(path, norm_path)
@@ -302,9 +302,11 @@ uint32_t handle_file_open(HookFile &file) {
 
     if (string_ends_with(file.path, "texturelist.xml")) {
         parse_texturelist(file);
-    }
-    else {
+    } else if(string_ends_with(file.path, "afplist.xml")) {
+        parse_afplist(file);
+    } else {
         handle_texture(file);
+        handle_afp(file);
     }
 
     auto ret = file.call_real();

src/imagefs.cpp

@@ -3,6 +3,8 @@
 #include <inttypes.h>
 #include <map>
 #include <fstream>
+#include <memory>
+#include <sstream>
 
 #include "3rd_party/lodepng.h"
 #include "3rd_party/stb_dxt.h"
@@ -39,14 +41,22 @@ typedef struct image {
     string ifs_mod_path;
     int width;
     int height;
-    const string cache_folder() { return CACHE_FOLDER + "/" + ifs_mod_path;  }
-    const string cache_file() { return cache_folder() + "/" + name_md5; };
+    string cache_folder() const { return CACHE_FOLDER + "/" + ifs_mod_path;  }
+    string cache_file() const { return cache_folder() + "/" + name_md5; };
 } image_t;
 
+typedef struct afp {
+    string mod_path;
+} afp_t;
+
 // ifs_textures["data/graphics/ver04/logo.ifs/tex/4f754d4f424f092637a49a5527ece9bb"] will be "konami"
-static std::map<string, image_t, CaseInsensitiveCompare> ifs_textures;
+static std::map<string, std::shared_ptr<image_t>, CaseInsensitiveCompare> ifs_textures;
 static CriticalSectionLock ifs_textures_mtx;
 
+static std::map<std::string, std::shared_ptr<afp_t>, CaseInsensitiveCompare> afp_md5_names;
+static CriticalSectionLock afp_md5_names_mtx;
+
+
 void rapidxml_dump_to_file(const string& out, const rapidxml::xml_document<> &xml) {
     std::ofstream out_file;
     out_file.open(out.c_str());
@@ -173,7 +183,7 @@ bool add_images_to_list(string_set &extra_pngs, rapidxml::xml_node<> *texturelis
 
             auto md5_path = ifs_path + "/tex/" + image_info.name_md5;
             ifs_textures_mtx.lock();
-            ifs_textures[md5_path] = image_info;
+            ifs_textures[md5_path] = std::make_shared<image_t>(std::move(image_info));
             ifs_textures_mtx.unlock();
         }
     }
@@ -280,12 +290,12 @@ void parse_texturelist(HookFile &file) {
             image_info.width = (dimensions[1] - dimensions[0]) / 2;
             image_info.height = (dimensions[3] - dimensions[2]) / 2;
 
+            extra_pngs.erase(image_info.name);
+
             auto md5_path = ifs_path + "/tex/" + image_info.name_md5;
             ifs_textures_mtx.lock();
-            ifs_textures[md5_path] = image_info;
+            ifs_textures[md5_path] = std::make_shared<image_t>(std::move(image_info));
             ifs_textures_mtx.unlock();
-
-            extra_pngs.erase(image_info.name);
         }
     }
 
@@ -306,7 +316,7 @@ void parse_texturelist(HookFile &file) {
     }
 }
 
-bool cache_texture(string const&png_path, image_t &tex) {
+bool cache_texture(string const&png_path, image_t const&tex) {
     string cache_path = tex.cache_folder();
     if (!mkdir_p(cache_path)) {
         log_warning("Couldn't create texture cache folder");
@@ -407,12 +417,87 @@ bool cache_texture(string const&png_path, image_t &tex) {
     return true;
 }
 
-void handle_texture(HookFile &file) {
+void parse_afplist(HookFile &file) {
+    // get a reasonable base path
+    auto ifs_path = file.norm_path;
+    // truncate
+    ifs_path.resize(ifs_path.size() - strlen("/tex/afplist.xml"));
+    // log_misc("Reading ifs %s", ifs_path.c_str());
+    auto ifs_mod_path = ifs_path;
+    string_replace(ifs_mod_path, ".ifs", "_ifs");
+
+    if (!find_first_modfolder(ifs_mod_path)) {
+        // hide this print - the texturelist.xml will catch it
+        // log_verbose("mod folder doesn't exist, skipping");
+        return;
+    }
+
+    // open the correct file
+    auto path_to_open = file.get_path_to_open();
+    rapidxml::xml_document<> afplist;
+    auto success = rapidxml_from_avs_filepath(path_to_open, afplist, afplist);
+    if (!success)
+        return;
+
+    auto afplist_node = afplist.first_node("afplist");
+
+    if (!afplist_node) {
+        log_warning("afplist has no afplist node");
+        return;
+    }
+
+    int mapped = 0;
+
+    for(auto afp = afplist_node->first_node("afp");
+            afp;
+            afp = afp->next_sibling("afp")) {
+
+        auto name = afp->first_attribute("name");
+        if (!name) {
+            log_warning("AFP missing name %s", path_to_open.c_str());
+            continue;
+        }
+
+        // <geo __type="u16" __count="5">5 8 11 16 19</geo>
+        auto geo = afp->first_node("geo");
+        if (!geo) {
+            log_warning("AFP missing geo %s", path_to_open.c_str());
+            continue;
+        }
+
+        auto add_mapping = [&](std::string folder, std::string file) {
+            auto md5_path = ifs_path + folder + MD5()(file);
+            afp_md5_names[md5_path] = std::make_shared<afp_t>(afp_t {
+                .mod_path = ifs_mod_path + folder + file,
+            });
+            mapped++;
+            // log_info("AFP %s -> %s", md5_path.c_str(), (ifs_mod_path + folder + file).c_str());
+        };
+
+        afp_md5_names_mtx.lock();
+
+        add_mapping("/afp/", name->value());
+        add_mapping("/afp/bsi/", name->value());
+
+        // iterate geos
+        std::string index;
+        std::stringstream ss(geo->value());
+        while(ss >> index) {
+            add_mapping("/geo/", std::string(name->value()) + "_shape" + index);
+        }
+
+        afp_md5_names_mtx.unlock();
+    }
+
+    log_misc("Mapped %d AFP filenames", mapped);
+}
+
+std::optional<std::tuple<std::string, std::shared_ptr<image_t>>> lookup_png_from_md5(HookFile &file) {
     ifs_textures_mtx.lock();
     auto tex_search = ifs_textures.find(file.norm_path);
     if (tex_search == ifs_textures.end()) {
         ifs_textures_mtx.unlock();
-        return;
+        return std::nullopt;
     }
 
     //log_misc("Mapped file %s is found!", norm_path.c_str());
@@ -420,30 +505,65 @@ void handle_texture(HookFile &file) {
     ifs_textures_mtx.unlock(); // is it safe to unlock this early? Time will tell...
 
     // remove the /tex/, it's nicer to navigate
-    auto png_path = find_first_modfile(tex.ifs_mod_path + "/" + tex.name + ".png");
+    auto png_path = find_first_modfile(tex->ifs_mod_path + "/" + tex->name + ".png");
     if (!png_path) {
         // but maybe they used it anyway
-        png_path = find_first_modfile(tex.ifs_mod_path + "/tex/" + tex.name + ".png");
+        png_path = find_first_modfile(tex->ifs_mod_path + "/tex/" + tex->name + ".png");
         if (!png_path)
-            return;
+            return std::nullopt;
     }
 
-    if (tex.compression == UNSUPPORTED_COMPRESS) {
-        log_warning("Unsupported compression for %s", png_path->c_str());
+    return std::make_tuple(*png_path, tex);
+}
+
+void handle_texture(HookFile &file) {
+    auto lookup = lookup_png_from_md5(file);
+    if(!lookup)
+        return;
+
+    auto &[png_path, tex] = *lookup;
+
+    if (tex->compression == UNSUPPORTED_COMPRESS) {
+        log_warning("Unsupported compression for %s", png_path.c_str());
         return;
     }
-    if (tex.format == UNSUPPORTED_FORMAT) {
-        log_warning("Unsupported texture format for %s", png_path->c_str());
+    if (tex->format == UNSUPPORTED_FORMAT) {
+        log_warning("Unsupported texture format for %s", png_path.c_str());
         return;
     }
 
-    log_verbose("Mapped file %s found!", png_path->c_str());
-    if (cache_texture(*png_path, tex)) {
-        file.mod_path = tex.cache_file();
+    log_verbose("Mapped file %s found!", png_path.c_str());
+    if (cache_texture(png_path, *tex)) {
+        file.mod_path = tex->cache_file();
     }
     return;
 }
 
+std::optional<std::string> lookup_afp_from_md5(HookFile &file) {
+    afp_md5_names_mtx.lock();
+    auto afp_search = afp_md5_names.find(file.norm_path);
+    if (afp_search == afp_md5_names.end()) {
+        afp_md5_names_mtx.unlock();
+        return std::nullopt;
+    }
+
+    //log_misc("Mapped file %s is found!", norm_path.c_str());
+    auto afp = afp_search->second;
+    afp_md5_names_mtx.unlock(); // is it safe to unlock this early? Time will tell...
+
+    return find_first_modfile(afp->mod_path);
+}
+
+void handle_afp(HookFile &file) {
+    auto lookup = lookup_afp_from_md5(file);
+    if(!lookup)
+        return;
+
+    log_verbose("Mapped file %s found!", lookup->c_str());
+    file.mod_path = *lookup;
+    return;
+}
+
 void merge_xmls(HookFile &file) {
     auto start = time();
     // initialize since we're GOTO-ing like naughty people

src/imagefs.hpp

@@ -1,5 +1,15 @@
+#include <memory>
+#include <optional>
+#include <tuple>
+
 #include "hook.h"
 
 void handle_texture(HookFile &file);
+void handle_afp(HookFile &file);
 void parse_texturelist(HookFile &file);
+void parse_afplist(HookFile &file);
 void merge_xmls(HookFile &file);
+// only exported to test the MD5 lookup machinery
+struct image;
+std::optional<std::tuple<std::string, std::shared_ptr<struct image>>> lookup_png_from_md5(HookFile &file);
+std::optional<std::string> lookup_afp_from_md5(HookFile &file);

src/tests.cpp

@@ -3,12 +3,23 @@
 
 #include "config.hpp"
 #include "hook.h"
+#include "imagefs.hpp"
 #include "avs_standalone.hpp"
 #include "modpath_handler.h"
 
 using ::testing::Contains;
 using ::testing::Optional;
 
+#define FOREACH_EXTRA_FUNC(X)                                                                                                                              \
+    X("XCgsqzn000004d", void, avs_fs_umount_by_desc, uint32_t desc) \
+
+
+#define AVS_FUNC_PTR(obfus_name, ret_type, name, ...) ret_type (*name)(__VA_ARGS__);
+FOREACH_EXTRA_FUNC(AVS_FUNC_PTR)
+
+#define LOAD_FUNC(obfus_name, ret_type, name, ...)                 \
+   ASSERT_TRUE((name = (decltype(name))GetProcAddress(avs, obfus_name))); \
+
 class Environment : public ::testing::Environment {
     public:
      ~Environment() override {}
@@ -16,9 +27,15 @@ class Environment : public ::testing::Environment {
      // Override this to define how to set up the environment.
      void SetUp() override {
         ASSERT_TRUE(avs_standalone::boot(false));
+
+        auto avs = GetModuleHandleA("avs2-core.dll");
+        ASSERT_TRUE(avs);
+        FOREACH_EXTRA_FUNC(LOAD_FUNC);
+
         ASSERT_EQ(init(), 0);
 
         config.mod_folder = "./testcases_data_mods";
+        config.verbose_logs = true;
         print_config();
         cache_mods();
 
@@ -44,6 +61,21 @@ class DevModeOnOff : public testing::TestWithParam<bool> {
 };
 INSTANTIATE_TEST_SUITE_P(DevModeOnOffInstance, DevModeOnOff, testing::Bool());
 
+class TestHookFile final : public HookFile {
+   using HookFile::HookFile;
+
+   public:
+
+   bool ramfs_demangle() override {return true;};
+
+   uint32_t call_real() override {
+       throw "Unimplemented";
+   }
+
+   std::optional<std::vector<uint8_t>> load_to_vec() override {
+      throw "Unimplemented";
+   }
+};
 
 TEST_P(DevModeOnOff, MissingFileNullopt) {
    ASSERT_EQ(find_first_modfile("doesn't exist"), std::nullopt);
@@ -58,3 +90,62 @@ TEST_P(DevModeOnOff, CaseInsensitiveFolders) {
    EXPECT_THAT(find_first_modfolder("OhNO"), Optional(config.mod_folder + "/Case_Sensitive/OhNO/"));
    EXPECT_THAT(find_first_modfolder("ohno"), Optional(config.mod_folder + "/Case_Sensitive/OhNO/"));
 }
+
+TEST(ImageFs, MD5DemanglingWorks) {
+   std::string mount = "/afp/data/mount/test.ifs";
+   auto desc = hook_avs_fs_mount(mount.c_str(), "./data/test.ifs", "imagefs", NULL);
+   ASSERT_GT(desc, 0);
+
+   // load all the xml files to load md5 mappings
+   auto check_load = [](std::string path) {
+      auto f = hook_avs_fs_open(path.c_str(), avs_open_mode_read(), 420);
+      EXPECT_GT(f, 0);
+      if(f > 0)
+         avs_fs_close(f);
+   };
+   check_load(mount + "/tex/texturelist.xml");
+   check_load(mount + "/afp/afplist.xml");
+
+   auto lookup_tex = [&](std::string folder, std::string fname) {
+      MD5 md5;
+      auto hash = md5(fname);
+      auto path = mount + "/" + folder + "/" + hash;
+      auto norm = normalise_path(path);
+      EXPECT_NE(norm, std::nullopt);
+      if(!norm) return std::string();
+
+      TestHookFile file(path, *norm);
+      log_info("Lookup %s norm %s", path.c_str(), norm->c_str());
+      auto lookup = lookup_png_from_md5(file);
+      EXPECT_NE(lookup, std::nullopt);
+      if(!lookup) return std::string();
+      auto &[png, tex] = *lookup;
+
+      return png;
+   };
+
+   auto lookup_afp = [&](std::string folder, std::string fname) {
+      auto hash = MD5()(fname);
+      auto path = mount + "/" + folder + "/" + hash;
+      auto norm = normalise_path(path);
+      EXPECT_NE(norm, std::nullopt);
+      if(!norm) return std::string();
+
+      TestHookFile file(path, *norm);
+      log_info("Lookup %s norm %s", path.c_str(), norm->c_str());
+      auto lookup = lookup_afp_from_md5(file);
+      EXPECT_NE(lookup, std::nullopt);
+      if(!lookup) return std::string();
+
+      return *lookup;
+   };
+
+   EXPECT_EQ(lookup_tex("tex", "inner"),               config.mod_folder + "/md5_lookup/test_ifs/tex/inner.png");
+   EXPECT_EQ(lookup_tex("tex", "outer"),               config.mod_folder + "/md5_lookup/test_ifs/outer.png");
+   EXPECT_EQ(lookup_afp("afp", "confirm_all"),         config.mod_folder + "/md5_lookup/test_ifs/afp/confirm_all");
+   EXPECT_EQ(lookup_afp("afp/bsi", "confirm_all"),     config.mod_folder + "/md5_lookup/test_ifs/afp/bsi/confirm_all");
+   EXPECT_EQ(lookup_afp("geo", "confirm_all_shape5"),  config.mod_folder + "/md5_lookup/test_ifs/geo/confirm_all_shape5");
+   EXPECT_EQ(lookup_afp("geo", "confirm_all_shape11"), config.mod_folder + "/md5_lookup/test_ifs/geo/confirm_all_shape11");
+
+   avs_fs_umount_by_desc(desc);
+}