ARC: fixup some insane AI-made choices and support merged XMLs in a terribly hacky way. Will rework later.

src/hook.cpp

@@ -51,72 +51,6 @@ unsigned int (*pkfs_fs_read)(unsigned int f, void *buf, int sz);
 unsigned int (*pkfs_fs_close)(unsigned int f);
 void (*pkfs_clear_hdd_error)();
 
-class AvsHookFile : public HookFile {
-    using HookFile::HookFile;
-
-    std::optional<std::vector<uint8_t>> load_to_vec() override {
-        AVS_FILE f = avs_fs_open(get_path_to_open().c_str(), avs_open_mode_read(), 420);
-        if (f >= 0) {
-            auto ret = avs_file_to_vec(f);
-            avs_fs_close(f);
-            return ret;
-        } else {
-            return nullopt;
-        }
-    }
-};
-class AvsOpenHookFile final : public AvsHookFile {
-    private:
-    uint16_t mode;
-    int flags;
-
-    public:
-    AvsOpenHookFile(const std::string path, const std::string norm_path, uint16_t mode, int flags)
-        : AvsHookFile(path, norm_path)
-        , mode(mode)
-        , flags(flags)
-    {}
-
-    bool ramfs_demangle() override {return true;};
-
-    uint32_t call_real() override {
-        log_if_modfile();
-        return (uint32_t)avs_fs_open(get_path_to_open().c_str(), mode, flags);
-    }
-};
-
-class AvsLstatHookFile final : public AvsHookFile {
-    private:
-    struct avs_stat *st;
-
-    public:
-    AvsLstatHookFile(const std::string path, const std::string norm_path, struct avs_stat *st)
-        : AvsHookFile(path, norm_path)
-        , st(st)
-    {}
-
-    uint32_t call_real() override {
-        log_if_modfile();
-        return (uint32_t)avs_fs_lstat(get_path_to_open().c_str(), st);
-    }
-};
-
-class AvsConvertPathHookFile final : public AvsHookFile {
-    private:
-    char *dest_name;
-
-    public:
-    AvsConvertPathHookFile(const std::string path, const std::string norm_path, char *dest_name)
-        : AvsHookFile(path, norm_path)
-        , dest_name(dest_name)
-    {}
-
-    uint32_t call_real() override {
-        log_if_modfile();
-        return (uint32_t)avs_fs_convert_path(dest_name, get_path_to_open().c_str());
-    }
-};
-
 class PkfsHookFile final : public HookFile {
     public:
     PkfsHookFile(const std::string path, const std::string norm_path)
@@ -192,7 +126,8 @@ string_set list_pngs(string const&folder) {
 
 struct ArcModScan {
     // Per-entry overrides. Keyed by relative path within the arc, e.g. "shader/foo.bin".
-    std::map<string, string> files;
+    // Tuple of actual path / norm path
+    std::map<string, std::pair<string, string>> files;
     // Inner-ifs paths inside the arc, derived from "*_ifs" mod subdirs. Paths
     // are arc-relative with the dot restored: a mod dir "sub/inner_ifs/" yields
     // "sub/inner.ifs". These get registered with the demangler so the game's
@@ -202,9 +137,9 @@ struct ArcModScan {
     std::set<string> inner_ifs_paths;
 };
 
-static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string const& rel_prefix) {
+static void scan_arc_mod_onefolder(ArcModScan &out, string const& mod_folder, string const& folder, string const& rel_prefix) {
     WIN32_FIND_DATAA fd;
-    HANDLE hFind = FindFirstFileA((folder + "/*").c_str(), &fd);
+    HANDLE hFind = FindFirstFileA((mod_folder + "/" + folder + "/*").c_str(), &fd);
     if (hFind == INVALID_HANDLE_VALUE)
         return;
     do {
@@ -218,12 +153,16 @@ static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string
                 continue;
             }
             scan_arc_mod_onefolder(out,
+                mod_folder,
                 folder + "/" + fd.cFileName,
                 rel_prefix + fd.cFileName + "/");
         } else {
             string rel = rel_prefix + fd.cFileName;
             if (out.files.find(rel) == out.files.end())
-                out.files[rel] = folder + "/" + fd.cFileName;
+                out.files[rel] = std::make_pair(
+                    mod_folder + "/" + folder + "/" + fd.cFileName,
+                    folder + "/" + fd.cFileName
+                );
         }
     } while (FindNextFileA(hFind, &fd));
     FindClose(hFind);
@@ -232,7 +171,7 @@ static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string
 static ArcModScan scan_arc_mod_folder(string const& folder) {
     ArcModScan ret;
     for (auto &mod : available_mods()) {
-        scan_arc_mod_onefolder(ret, mod + "/" + folder, "");
+        scan_arc_mod_onefolder(ret, mod, folder, "");
     }
     return ret;
 }
@@ -271,7 +210,7 @@ void handle_arc(HookFile &file) {
     auto starting = file.get_path_to_open();
     cache_hasher.add(starting);
     for (auto &[_, path] : scan.files) {
-        cache_hasher.add(path);
+        cache_hasher.add(path.first);
     }
     cache_hasher.finish();
 
@@ -303,20 +242,92 @@ void handle_arc(HookFile &file) {
         return;
     }
 
+    // TODO: this is a terrible hack really. XML merging assumes the file
+    // exists on disk, so we need to make it so if it only exists inside of
+    // the original .arc file
+    for (const auto &arc_file : arc.files) {
+        if (!string_ends_with(arc_file.first, ".xml")) {
+            continue;
+        }
+
+        // do we have an overlaid base xml? If so, nothing to do
+        bool found = false;
+        for (auto &[name, path] : scan.files) {
+            // TODO: more hacks, make these paths less insane
+            auto parent_pos = path.second.find("/");
+            if (parent_pos == path.second.npos) {
+                continue;
+            }
+            auto path_noparent = path.second.substr(parent_pos + 1);
+
+            if (_stricmp(arc_file.first.c_str(), path_noparent.c_str())) {
+                continue;
+            }
+            found = true;
+        }
+
+        if (found)
+            continue;
+
+
+        // Do we have XMLs to merge? Only then, extract to cache and add a fake
+        // extra entry. We don't need to add this particular file to the cache
+        // because it gets its key off the original .arc file
+        string merged_fname = arc_file.first;
+        string_replace(merged_fname, ".xml", ".merged.xml");
+        decltype(scan.files) extra_files;
+        for (auto &[name, path] : scan.files) {
+            // TODO: more hacks, make these paths less insane
+            auto parent_pos = path.second.find("/");
+            if (parent_pos == path.second.npos) {
+                continue;
+            }
+            auto path_noparent = path.second.substr(parent_pos + 1);
+            auto path_justparent = path.second.substr(0, parent_pos);
+
+            if (_stricmp(merged_fname.c_str(), path_noparent.c_str())) {
+                continue;
+            }
+
+            auto xml_orig = CACHE_FOLDER + "/" + file.norm_path + "/" + arc_file.first + ".orig";
+            auto xml_orig_norm = file.norm_path + "/" + arc_file.first;
+            string_replace(xml_orig, ".arc", "_arc");
+            string_replace(xml_orig_norm, ".arc", "_arc");
+
+            auto out_folder = xml_orig.substr(0, xml_orig.rfind("/"));
+            if (!mkdir_p(out_folder)) {
+                log_warning("Couldn't create arc xml cache folder %s", out_folder.c_str());
+                continue;
+            }
+
+            FILE *f = fopen(xml_orig.c_str(), "wb");
+            if (!f) {
+                log_warning("Couldn't create arc xml base at %s", xml_orig.c_str());
+                continue;
+            }
+            fwrite(arc_file.second.data(), 1, arc_file.second.size(), f);
+            fclose(f);
+            extra_files.emplace(arc_file.first, std::make_pair(xml_orig, xml_orig_norm));
+        }
+        for (auto &[name, path] : extra_files)
+            scan.files.emplace(name, std::move(path));
+    }
+
     for (auto &[name, path] : scan.files) {
-        std::ifstream f(path, std::ios::binary | std::ios::ate);
-        if (!f) {
-            log_warning("arc: couldn't open mod file '%s'", path.c_str());
+        if (string_ends_with(name, ".merged.xml"))
+            continue;
+
+        AvsOpenHookFile f(path.first, path.second, 0, 0);
+
+        if (string_ends_with(name, ".xml"))
+            merge_xmls(f);
+
+        auto data = f.load_to_vec();
+        if (!data) {
+            log_warning("arc: couldn't load mod file '%s'", path.first.c_str());
             continue;
         }
-        auto size = f.tellg();
-        f.seekg(0);
-        std::vector<uint8_t> data(size);
-        if (!f.read(reinterpret_cast<char*>(data.data()), size)) {
-            log_warning("arc: couldn't read mod file '%s'", path.c_str());
-            continue;
-        }
-        arc.add_or_replace(name, std::move(data));
+        arc.add_or_replace(name, std::move(*data));
     }
 
     if (!arc.save(out.c_str())) {

src/hook.h

@@ -75,5 +75,72 @@ class HookFile {
     virtual ~HookFile() {}
 };
 
+class AvsHookFile : public HookFile {
+    using HookFile::HookFile;
+
+    public:
+    std::optional<std::vector<uint8_t>> load_to_vec() override {
+        AVS_FILE f = avs_fs_open(get_path_to_open().c_str(), avs_open_mode_read(), 420);
+        if (f >= 0) {
+            auto ret = avs_file_to_vec(f);
+            avs_fs_close(f);
+            return ret;
+        } else {
+            return std::nullopt;
+        }
+    }
+};
+class AvsOpenHookFile final : public AvsHookFile {
+    private:
+    uint16_t mode;
+    int flags;
+
+    public:
+    AvsOpenHookFile(const std::string path, const std::string norm_path, uint16_t mode, int flags)
+        : AvsHookFile(path, norm_path)
+        , mode(mode)
+        , flags(flags)
+    {}
+
+    bool ramfs_demangle() override {return true;};
+
+    uint32_t call_real() override {
+        log_if_modfile();
+        return (uint32_t)avs_fs_open(get_path_to_open().c_str(), mode, flags);
+    }
+};
+
+class AvsLstatHookFile final : public AvsHookFile {
+    private:
+    struct avs_stat *st;
+
+    public:
+    AvsLstatHookFile(const std::string path, const std::string norm_path, struct avs_stat *st)
+        : AvsHookFile(path, norm_path)
+        , st(st)
+    {}
+
+    uint32_t call_real() override {
+        log_if_modfile();
+        return (uint32_t)avs_fs_lstat(get_path_to_open().c_str(), st);
+    }
+};
+
+class AvsConvertPathHookFile final : public AvsHookFile {
+    private:
+    char *dest_name;
+
+    public:
+    AvsConvertPathHookFile(const std::string path, const std::string norm_path, char *dest_name)
+        : AvsHookFile(path, norm_path)
+        , dest_name(dest_name)
+    {}
+
+    uint32_t call_real() override {
+        log_if_modfile();
+        return (uint32_t)avs_fs_convert_path(dest_name, get_path_to_open().c_str());
+    }
+};
+
 // for the tests
 void handle_arc(HookFile &file);

src/tests.cpp

@@ -26,12 +26,24 @@ FOREACH_EXTRA_FUNC(AVS_FUNC_PTR)
 #define LOAD_FUNC(obfus_name, ret_type, name, ...)                 \
    ASSERT_TRUE((name = (decltype(name))GetProcAddress(avs, obfus_name))); \
 
+class LogTestName : public testing::EmptyTestEventListener {
+    void OnTestStart(const testing::TestInfo& info) override {
+        log_misc("--------- Running %s.%s", info.test_suite_name(), info.name());
+    }
+};
+
 class Environment : public ::testing::Environment {
     public:
      ~Environment() override {}
 
      // Override this to define how to set up the environment.
      void SetUp() override {
+        testing::UnitTest::GetInstance()->listeners().Append(new LogTestName);
+
+        char exe_path[MAX_PATH];
+        ASSERT_TRUE(GetModuleFileNameA(NULL, exe_path, MAX_PATH));
+        dll_time = file_time(exe_path);
+
         ASSERT_TRUE(avs_standalone::boot(false));
 
         auto avs = GetModuleHandleA("avs2-core.dll");
@@ -266,17 +278,6 @@ TEST(RamFs, DemanglingWorksNabla) {
    EXPECT_EQ(path, "/data/graphics/ver07/logo.ifs/tex/texturelist.xml");
 }
 
-class ArcTestHookFile final : public HookFile {
-    std::optional<std::vector<uint8_t>> original;
-public:
-    ArcTestHookFile(std::string path, std::string norm_path,
-                    std::optional<std::vector<uint8_t>> original = std::nullopt)
-        : HookFile(path, norm_path), original(std::move(original)) {}
-
-    uint32_t call_real() override { return 0; }
-    std::optional<std::vector<uint8_t>> load_to_vec() override { return original; }
-};
-
 static std::vector<uint8_t> read_arc_file(std::string const& arc_path, std::string const& name) {
     std::ifstream f(arc_path, std::ios::binary);
     if (!f) return {};
@@ -288,7 +289,7 @@ static std::vector<uint8_t> read_arc_file(std::string const& arc_path, std::stri
 }
 
 TEST(ArcArchive, ScratchFromTwoMods) {
-    ArcTestHookFile file("scratch.arc", "scratch.arc");
+    AvsOpenHookFile file("scratch.arc", "scratch.arc", avs_open_mode_read(), 420);
     handle_arc(file);
     ASSERT_TRUE(file.mod_path.has_value());
 
@@ -298,6 +299,82 @@ TEST(ArcArchive, ScratchFromTwoMods) {
     EXPECT_EQ(read_arc_file(*file.mod_path, "file_b"), expected_b);
 }
 
+TEST(ArcArchive, MergedXmlInsideArc) {
+    AvsOpenHookFile file("xml_merge.arc", "xml_merge.arc", avs_open_mode_read(), 42);
+    // note: only two tests where we have an original .arc so we need a tiny bit of
+    // copied extra work from handle_file
+    auto norm_copy = file.norm_path;
+    file.mod_path = find_first_modfile(norm_copy);
+    handle_arc(file);
+    ASSERT_TRUE(file.mod_path.has_value());
+
+    auto base_xml = read_arc_file(*file.mod_path, "merged/base.xml");
+    ASSERT_FALSE(base_xml.empty());
+    EXPECT_TRUE(read_arc_file(*file.mod_path, "merged/base.merged.xml").empty());
+
+    base_xml.push_back('\0');
+    rapidxml::xml_document<> xml_doc;
+    xml_doc.parse<rapidxml::parse_no_utf8>((char*)base_xml.data());
+
+    auto node = xml_doc.first_node();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afplist");
+
+    node = node->first_node();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afp");
+    auto attr = node->first_attribute("name");
+    ASSERT_NE(attr, nullptr);
+    EXPECT_STREQ(attr->value(), "hare");
+
+    node = node->next_sibling();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afp");
+    attr = node->first_attribute("name");
+    ASSERT_NE(attr, nullptr);
+    EXPECT_STREQ(attr->value(), "hare2");
+
+    EXPECT_EQ(node->next_sibling(), nullptr);
+}
+
+TEST(ArcArchive, MergedXmlInsideArcWithOriginalInsideArc) {
+    AvsOpenHookFile file("xml_merge.arc", "xml_merge.arc", avs_open_mode_read(), 42);
+    // note: only two tests where we have an original .arc so we need a tiny bit of
+    // copied extra work from handle_file
+    auto norm_copy = file.norm_path;
+    file.mod_path = find_first_modfile(norm_copy);
+    handle_arc(file);
+    ASSERT_TRUE(file.mod_path.has_value());
+
+    auto base_xml = read_arc_file(*file.mod_path, "merged_only/base.xml");
+    ASSERT_FALSE(base_xml.empty());
+    EXPECT_TRUE(read_arc_file(*file.mod_path, "merged_only/base.merged.xml").empty());
+
+    base_xml.push_back('\0');
+    rapidxml::xml_document<> xml_doc;
+    xml_doc.parse<rapidxml::parse_no_utf8>((char*)base_xml.data());
+
+    auto node = xml_doc.first_node();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afplist");
+
+    node = node->first_node();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afp");
+    auto attr = node->first_attribute("name");
+    ASSERT_NE(attr, nullptr);
+    EXPECT_STREQ(attr->value(), "hare");
+
+    node = node->next_sibling();
+    ASSERT_NE(node, nullptr);
+    EXPECT_STREQ(node->name(), "afp");
+    attr = node->first_attribute("name");
+    ASSERT_NE(attr, nullptr);
+    EXPECT_STREQ(attr->value(), "hare2");
+
+    EXPECT_EQ(node->next_sibling(), nullptr);
+}
+
 // Drives the demangler mount chain end-to-end after handle_arc has registered
 // the inner-ifs basename: simulates the ramfs/imagefs mount sequence and
 // asserts that a path inside the inner ifs's imagefs mountpoint demangles back
@@ -321,11 +398,11 @@ TEST(ArcArchive, IfsOnlySubtreeSkipsRepack) {
     // Mod folder has only an _ifs subtree (no per-entry overrides). No repack
     // should happen — the original arc is left alone — but the inner-ifs
     // basename still gets registered with the demangler.
-    ArcTestHookFile file("ifs_only_repack.arc", "ifs_only_repack.arc");
+    AvsOpenHookFile file("ifs_only_repack.arc", "ifs_only_repack.arc", avs_open_mode_read(), 42);
     handle_arc(file);
     EXPECT_FALSE(file.mod_path.has_value());
 
-    exercise_inner_ifs_demangle(file.path);
+    exercise_inner_ifs_demangle(file.norm_path);
 }
 
 TEST(ArcArchive, IfsWithExtraOverrides) {
@@ -338,10 +415,8 @@ TEST(ArcArchive, IfsWithExtraOverrides) {
     std::string tmp_path = std::string(config.get_mod_folder()) + "/_cache/ifs_with_extras_orig.arc";
     mkdir_p(std::string(config.get_mod_folder()) + "/_cache");
     ASSERT_TRUE(orig.save(tmp_path.c_str()));
-    std::ifstream f(tmp_path, std::ios::binary);
-    std::vector<uint8_t> orig_bytes(std::istreambuf_iterator<char>(f), {});
 
-    ArcTestHookFile file("ifs_with_extras.arc", "ifs_with_extras.arc", std::move(orig_bytes));
+    AvsOpenHookFile file(tmp_path, "ifs_with_extras.arc", avs_open_mode_read(), 42);
     handle_arc(file);
     ASSERT_TRUE(file.mod_path.has_value());
 
@@ -350,7 +425,7 @@ TEST(ArcArchive, IfsWithExtraOverrides) {
     std::vector<uint8_t> expected_override = {'o','v','e','r','r','i','d','d','e','n','_','o','t','h','e','r','_','d','a','t','a'};
     EXPECT_EQ(read_arc_file(*file.mod_path, "other.bin"), expected_override);
 
-    exercise_inner_ifs_demangle(file.path);
+    exercise_inner_ifs_demangle(file.norm_path);
 }
 
 TEST(ArcArchive, OverlayWithTwoMods) {
@@ -363,11 +438,7 @@ TEST(ArcArchive, OverlayWithTwoMods) {
     mkdir_p(std::string(config.get_mod_folder()) + "/_cache");
     ASSERT_TRUE(orig.save(tmp_path.c_str()));
 
-    std::ifstream f(tmp_path, std::ios::binary);
-    ASSERT_TRUE(f.good());
-    std::vector<uint8_t> orig_bytes(std::istreambuf_iterator<char>(f), {});
-
-    ArcTestHookFile file("overlay_test.arc", "overlay_test.arc", std::move(orig_bytes));
+    AvsOpenHookFile file(tmp_path, "overlay_test.arc", avs_open_mode_read(), 42);
     handle_arc(file);
     ASSERT_TRUE(file.mod_path.has_value());
 

src/utils.cpp

@@ -266,7 +266,7 @@ CacheHasher::CacheHasher(std::string hash_file): hash_file(hash_file) {
     }
 }
 
-void CacheHasher::add(std::string &path) {
+void CacheHasher::add(const std::string &path) {
     digest.add(path.c_str(), path.length());
 
     auto ts = file_time(path.c_str());

src/utils.hpp

@@ -43,7 +43,7 @@ class CacheHasher {
     public:
     CacheHasher(std::string hash_file);
     // add a path and its timestamp to the hash. Should not be called after `finish`
-    void add(std::string &path);
+    void add(const std::string &path);
     // complete the hashing op
     void finish();
     // check if the hashfile matches

testcases_data_mods/arc_mod1/scratch_arc/merged/base.merged.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare2">
+    <geo __type="u16" __count="3">20 25 30</geo>
+  </afp>
+</afplist>

testcases_data_mods/arc_mod1/scratch_arc/merged/base.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare">
+    <geo __type="u16" __count="3">5 10 15</geo>
+  </afp>
+</afplist>

testcases_data_mods/arc_mod1/scratch_arc/merged_only/base.merged.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare2">
+    <geo __type="u16" __count="3">20 25 30</geo>
+  </afp>
+</afplist>

testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.merged.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare2">
+    <geo __type="u16" __count="3">20 25 30</geo>
+  </afp>
+</afplist>

testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare">
+    <geo __type="u16" __count="3">5 10 15</geo>
+  </afp>
+</afplist>

testcases_data_mods/arc_mod1/xml_merge_arc/merged_only/base.merged.xml

@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<afplist>
+  <afp name="hare2">
+    <geo __type="u16" __count="3">20 25 30</geo>
+  </afp>
+</afplist>