diff --git a/src/hook.cpp b/src/hook.cpp index 5e64434..d0aa74f 100644 --- a/src/hook.cpp +++ b/src/hook.cpp @@ -51,72 +51,6 @@ unsigned int (*pkfs_fs_read)(unsigned int f, void *buf, int sz); unsigned int (*pkfs_fs_close)(unsigned int f); void (*pkfs_clear_hdd_error)(); -class AvsHookFile : public HookFile { - using HookFile::HookFile; - - std::optional> load_to_vec() override { - AVS_FILE f = avs_fs_open(get_path_to_open().c_str(), avs_open_mode_read(), 420); - if (f >= 0) { - auto ret = avs_file_to_vec(f); - avs_fs_close(f); - return ret; - } else { - return nullopt; - } - } -}; -class AvsOpenHookFile final : public AvsHookFile { - private: - uint16_t mode; - int flags; - - public: - AvsOpenHookFile(const std::string path, const std::string norm_path, uint16_t mode, int flags) - : AvsHookFile(path, norm_path) - , mode(mode) - , flags(flags) - {} - - bool ramfs_demangle() override {return true;}; - - uint32_t call_real() override { - log_if_modfile(); - return (uint32_t)avs_fs_open(get_path_to_open().c_str(), mode, flags); - } -}; - -class AvsLstatHookFile final : public AvsHookFile { - private: - struct avs_stat *st; - - public: - AvsLstatHookFile(const std::string path, const std::string norm_path, struct avs_stat *st) - : AvsHookFile(path, norm_path) - , st(st) - {} - - uint32_t call_real() override { - log_if_modfile(); - return (uint32_t)avs_fs_lstat(get_path_to_open().c_str(), st); - } -}; - -class AvsConvertPathHookFile final : public AvsHookFile { - private: - char *dest_name; - - public: - AvsConvertPathHookFile(const std::string path, const std::string norm_path, char *dest_name) - : AvsHookFile(path, norm_path) - , dest_name(dest_name) - {} - - uint32_t call_real() override { - log_if_modfile(); - return (uint32_t)avs_fs_convert_path(dest_name, get_path_to_open().c_str()); - } -}; - class PkfsHookFile final : public HookFile { public: PkfsHookFile(const std::string path, const std::string norm_path) @@ -192,7 +126,8 @@ string_set list_pngs(string const&folder) { struct ArcModScan { // Per-entry overrides. Keyed by relative path within the arc, e.g. "shader/foo.bin". - std::map files; + // Tuple of actual path / norm path + std::map> files; // Inner-ifs paths inside the arc, derived from "*_ifs" mod subdirs. Paths // are arc-relative with the dot restored: a mod dir "sub/inner_ifs/" yields // "sub/inner.ifs". These get registered with the demangler so the game's @@ -202,9 +137,9 @@ struct ArcModScan { std::set inner_ifs_paths; }; -static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string const& rel_prefix) { +static void scan_arc_mod_onefolder(ArcModScan &out, string const& mod_folder, string const& folder, string const& rel_prefix) { WIN32_FIND_DATAA fd; - HANDLE hFind = FindFirstFileA((folder + "/*").c_str(), &fd); + HANDLE hFind = FindFirstFileA((mod_folder + "/" + folder + "/*").c_str(), &fd); if (hFind == INVALID_HANDLE_VALUE) return; do { @@ -218,12 +153,16 @@ static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string continue; } scan_arc_mod_onefolder(out, + mod_folder, folder + "/" + fd.cFileName, rel_prefix + fd.cFileName + "/"); } else { string rel = rel_prefix + fd.cFileName; if (out.files.find(rel) == out.files.end()) - out.files[rel] = folder + "/" + fd.cFileName; + out.files[rel] = std::make_pair( + mod_folder + "/" + folder + "/" + fd.cFileName, + folder + "/" + fd.cFileName + ); } } while (FindNextFileA(hFind, &fd)); FindClose(hFind); @@ -232,7 +171,7 @@ static void scan_arc_mod_onefolder(ArcModScan &out, string const& folder, string static ArcModScan scan_arc_mod_folder(string const& folder) { ArcModScan ret; for (auto &mod : available_mods()) { - scan_arc_mod_onefolder(ret, mod + "/" + folder, ""); + scan_arc_mod_onefolder(ret, mod, folder, ""); } return ret; } @@ -271,7 +210,7 @@ void handle_arc(HookFile &file) { auto starting = file.get_path_to_open(); cache_hasher.add(starting); for (auto &[_, path] : scan.files) { - cache_hasher.add(path); + cache_hasher.add(path.first); } cache_hasher.finish(); @@ -303,20 +242,92 @@ void handle_arc(HookFile &file) { return; } + // TODO: this is a terrible hack really. XML merging assumes the file + // exists on disk, so we need to make it so if it only exists inside of + // the original .arc file + for (const auto &arc_file : arc.files) { + if (!string_ends_with(arc_file.first, ".xml")) { + continue; + } + + // do we have an overlaid base xml? If so, nothing to do + bool found = false; + for (auto &[name, path] : scan.files) { + // TODO: more hacks, make these paths less insane + auto parent_pos = path.second.find("/"); + if (parent_pos == path.second.npos) { + continue; + } + auto path_noparent = path.second.substr(parent_pos + 1); + + if (_stricmp(arc_file.first.c_str(), path_noparent.c_str())) { + continue; + } + found = true; + } + + if (found) + continue; + + + // Do we have XMLs to merge? Only then, extract to cache and add a fake + // extra entry. We don't need to add this particular file to the cache + // because it gets its key off the original .arc file + string merged_fname = arc_file.first; + string_replace(merged_fname, ".xml", ".merged.xml"); + decltype(scan.files) extra_files; + for (auto &[name, path] : scan.files) { + // TODO: more hacks, make these paths less insane + auto parent_pos = path.second.find("/"); + if (parent_pos == path.second.npos) { + continue; + } + auto path_noparent = path.second.substr(parent_pos + 1); + auto path_justparent = path.second.substr(0, parent_pos); + + if (_stricmp(merged_fname.c_str(), path_noparent.c_str())) { + continue; + } + + auto xml_orig = CACHE_FOLDER + "/" + file.norm_path + "/" + arc_file.first + ".orig"; + auto xml_orig_norm = file.norm_path + "/" + arc_file.first; + string_replace(xml_orig, ".arc", "_arc"); + string_replace(xml_orig_norm, ".arc", "_arc"); + + auto out_folder = xml_orig.substr(0, xml_orig.rfind("/")); + if (!mkdir_p(out_folder)) { + log_warning("Couldn't create arc xml cache folder %s", out_folder.c_str()); + continue; + } + + FILE *f = fopen(xml_orig.c_str(), "wb"); + if (!f) { + log_warning("Couldn't create arc xml base at %s", xml_orig.c_str()); + continue; + } + fwrite(arc_file.second.data(), 1, arc_file.second.size(), f); + fclose(f); + extra_files.emplace(arc_file.first, std::make_pair(xml_orig, xml_orig_norm)); + } + for (auto &[name, path] : extra_files) + scan.files.emplace(name, std::move(path)); + } + for (auto &[name, path] : scan.files) { - std::ifstream f(path, std::ios::binary | std::ios::ate); - if (!f) { - log_warning("arc: couldn't open mod file '%s'", path.c_str()); + if (string_ends_with(name, ".merged.xml")) + continue; + + AvsOpenHookFile f(path.first, path.second, 0, 0); + + if (string_ends_with(name, ".xml")) + merge_xmls(f); + + auto data = f.load_to_vec(); + if (!data) { + log_warning("arc: couldn't load mod file '%s'", path.first.c_str()); continue; } - auto size = f.tellg(); - f.seekg(0); - std::vector data(size); - if (!f.read(reinterpret_cast(data.data()), size)) { - log_warning("arc: couldn't read mod file '%s'", path.c_str()); - continue; - } - arc.add_or_replace(name, std::move(data)); + arc.add_or_replace(name, std::move(*data)); } if (!arc.save(out.c_str())) { diff --git a/src/hook.h b/src/hook.h index 61aa419..4b9fa5c 100644 --- a/src/hook.h +++ b/src/hook.h @@ -75,5 +75,72 @@ class HookFile { virtual ~HookFile() {} }; +class AvsHookFile : public HookFile { + using HookFile::HookFile; + + public: + std::optional> load_to_vec() override { + AVS_FILE f = avs_fs_open(get_path_to_open().c_str(), avs_open_mode_read(), 420); + if (f >= 0) { + auto ret = avs_file_to_vec(f); + avs_fs_close(f); + return ret; + } else { + return std::nullopt; + } + } +}; +class AvsOpenHookFile final : public AvsHookFile { + private: + uint16_t mode; + int flags; + + public: + AvsOpenHookFile(const std::string path, const std::string norm_path, uint16_t mode, int flags) + : AvsHookFile(path, norm_path) + , mode(mode) + , flags(flags) + {} + + bool ramfs_demangle() override {return true;}; + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_open(get_path_to_open().c_str(), mode, flags); + } +}; + +class AvsLstatHookFile final : public AvsHookFile { + private: + struct avs_stat *st; + + public: + AvsLstatHookFile(const std::string path, const std::string norm_path, struct avs_stat *st) + : AvsHookFile(path, norm_path) + , st(st) + {} + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_lstat(get_path_to_open().c_str(), st); + } +}; + +class AvsConvertPathHookFile final : public AvsHookFile { + private: + char *dest_name; + + public: + AvsConvertPathHookFile(const std::string path, const std::string norm_path, char *dest_name) + : AvsHookFile(path, norm_path) + , dest_name(dest_name) + {} + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_convert_path(dest_name, get_path_to_open().c_str()); + } +}; + // for the tests void handle_arc(HookFile &file); diff --git a/src/tests.cpp b/src/tests.cpp index 5da4082..fdeb5ee 100644 --- a/src/tests.cpp +++ b/src/tests.cpp @@ -26,12 +26,24 @@ FOREACH_EXTRA_FUNC(AVS_FUNC_PTR) #define LOAD_FUNC(obfus_name, ret_type, name, ...) \ ASSERT_TRUE((name = (decltype(name))GetProcAddress(avs, obfus_name))); \ +class LogTestName : public testing::EmptyTestEventListener { + void OnTestStart(const testing::TestInfo& info) override { + log_misc("--------- Running %s.%s", info.test_suite_name(), info.name()); + } +}; + class Environment : public ::testing::Environment { public: ~Environment() override {} // Override this to define how to set up the environment. void SetUp() override { + testing::UnitTest::GetInstance()->listeners().Append(new LogTestName); + + char exe_path[MAX_PATH]; + ASSERT_TRUE(GetModuleFileNameA(NULL, exe_path, MAX_PATH)); + dll_time = file_time(exe_path); + ASSERT_TRUE(avs_standalone::boot(false)); auto avs = GetModuleHandleA("avs2-core.dll"); @@ -266,17 +278,6 @@ TEST(RamFs, DemanglingWorksNabla) { EXPECT_EQ(path, "/data/graphics/ver07/logo.ifs/tex/texturelist.xml"); } -class ArcTestHookFile final : public HookFile { - std::optional> original; -public: - ArcTestHookFile(std::string path, std::string norm_path, - std::optional> original = std::nullopt) - : HookFile(path, norm_path), original(std::move(original)) {} - - uint32_t call_real() override { return 0; } - std::optional> load_to_vec() override { return original; } -}; - static std::vector read_arc_file(std::string const& arc_path, std::string const& name) { std::ifstream f(arc_path, std::ios::binary); if (!f) return {}; @@ -288,7 +289,7 @@ static std::vector read_arc_file(std::string const& arc_path, std::stri } TEST(ArcArchive, ScratchFromTwoMods) { - ArcTestHookFile file("scratch.arc", "scratch.arc"); + AvsOpenHookFile file("scratch.arc", "scratch.arc", avs_open_mode_read(), 420); handle_arc(file); ASSERT_TRUE(file.mod_path.has_value()); @@ -298,6 +299,82 @@ TEST(ArcArchive, ScratchFromTwoMods) { EXPECT_EQ(read_arc_file(*file.mod_path, "file_b"), expected_b); } +TEST(ArcArchive, MergedXmlInsideArc) { + AvsOpenHookFile file("xml_merge.arc", "xml_merge.arc", avs_open_mode_read(), 42); + // note: only two tests where we have an original .arc so we need a tiny bit of + // copied extra work from handle_file + auto norm_copy = file.norm_path; + file.mod_path = find_first_modfile(norm_copy); + handle_arc(file); + ASSERT_TRUE(file.mod_path.has_value()); + + auto base_xml = read_arc_file(*file.mod_path, "merged/base.xml"); + ASSERT_FALSE(base_xml.empty()); + EXPECT_TRUE(read_arc_file(*file.mod_path, "merged/base.merged.xml").empty()); + + base_xml.push_back('\0'); + rapidxml::xml_document<> xml_doc; + xml_doc.parse((char*)base_xml.data()); + + auto node = xml_doc.first_node(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afplist"); + + node = node->first_node(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afp"); + auto attr = node->first_attribute("name"); + ASSERT_NE(attr, nullptr); + EXPECT_STREQ(attr->value(), "hare"); + + node = node->next_sibling(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afp"); + attr = node->first_attribute("name"); + ASSERT_NE(attr, nullptr); + EXPECT_STREQ(attr->value(), "hare2"); + + EXPECT_EQ(node->next_sibling(), nullptr); +} + +TEST(ArcArchive, MergedXmlInsideArcWithOriginalInsideArc) { + AvsOpenHookFile file("xml_merge.arc", "xml_merge.arc", avs_open_mode_read(), 42); + // note: only two tests where we have an original .arc so we need a tiny bit of + // copied extra work from handle_file + auto norm_copy = file.norm_path; + file.mod_path = find_first_modfile(norm_copy); + handle_arc(file); + ASSERT_TRUE(file.mod_path.has_value()); + + auto base_xml = read_arc_file(*file.mod_path, "merged_only/base.xml"); + ASSERT_FALSE(base_xml.empty()); + EXPECT_TRUE(read_arc_file(*file.mod_path, "merged_only/base.merged.xml").empty()); + + base_xml.push_back('\0'); + rapidxml::xml_document<> xml_doc; + xml_doc.parse((char*)base_xml.data()); + + auto node = xml_doc.first_node(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afplist"); + + node = node->first_node(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afp"); + auto attr = node->first_attribute("name"); + ASSERT_NE(attr, nullptr); + EXPECT_STREQ(attr->value(), "hare"); + + node = node->next_sibling(); + ASSERT_NE(node, nullptr); + EXPECT_STREQ(node->name(), "afp"); + attr = node->first_attribute("name"); + ASSERT_NE(attr, nullptr); + EXPECT_STREQ(attr->value(), "hare2"); + + EXPECT_EQ(node->next_sibling(), nullptr); +} + // Drives the demangler mount chain end-to-end after handle_arc has registered // the inner-ifs basename: simulates the ramfs/imagefs mount sequence and // asserts that a path inside the inner ifs's imagefs mountpoint demangles back @@ -321,11 +398,11 @@ TEST(ArcArchive, IfsOnlySubtreeSkipsRepack) { // Mod folder has only an _ifs subtree (no per-entry overrides). No repack // should happen — the original arc is left alone — but the inner-ifs // basename still gets registered with the demangler. - ArcTestHookFile file("ifs_only_repack.arc", "ifs_only_repack.arc"); + AvsOpenHookFile file("ifs_only_repack.arc", "ifs_only_repack.arc", avs_open_mode_read(), 42); handle_arc(file); EXPECT_FALSE(file.mod_path.has_value()); - exercise_inner_ifs_demangle(file.path); + exercise_inner_ifs_demangle(file.norm_path); } TEST(ArcArchive, IfsWithExtraOverrides) { @@ -338,10 +415,8 @@ TEST(ArcArchive, IfsWithExtraOverrides) { std::string tmp_path = std::string(config.get_mod_folder()) + "/_cache/ifs_with_extras_orig.arc"; mkdir_p(std::string(config.get_mod_folder()) + "/_cache"); ASSERT_TRUE(orig.save(tmp_path.c_str())); - std::ifstream f(tmp_path, std::ios::binary); - std::vector orig_bytes(std::istreambuf_iterator(f), {}); - ArcTestHookFile file("ifs_with_extras.arc", "ifs_with_extras.arc", std::move(orig_bytes)); + AvsOpenHookFile file(tmp_path, "ifs_with_extras.arc", avs_open_mode_read(), 42); handle_arc(file); ASSERT_TRUE(file.mod_path.has_value()); @@ -350,7 +425,7 @@ TEST(ArcArchive, IfsWithExtraOverrides) { std::vector expected_override = {'o','v','e','r','r','i','d','d','e','n','_','o','t','h','e','r','_','d','a','t','a'}; EXPECT_EQ(read_arc_file(*file.mod_path, "other.bin"), expected_override); - exercise_inner_ifs_demangle(file.path); + exercise_inner_ifs_demangle(file.norm_path); } TEST(ArcArchive, OverlayWithTwoMods) { @@ -363,11 +438,7 @@ TEST(ArcArchive, OverlayWithTwoMods) { mkdir_p(std::string(config.get_mod_folder()) + "/_cache"); ASSERT_TRUE(orig.save(tmp_path.c_str())); - std::ifstream f(tmp_path, std::ios::binary); - ASSERT_TRUE(f.good()); - std::vector orig_bytes(std::istreambuf_iterator(f), {}); - - ArcTestHookFile file("overlay_test.arc", "overlay_test.arc", std::move(orig_bytes)); + AvsOpenHookFile file(tmp_path, "overlay_test.arc", avs_open_mode_read(), 42); handle_arc(file); ASSERT_TRUE(file.mod_path.has_value()); diff --git a/src/utils.cpp b/src/utils.cpp index 5441435..b49e9b7 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -266,7 +266,7 @@ CacheHasher::CacheHasher(std::string hash_file): hash_file(hash_file) { } } -void CacheHasher::add(std::string &path) { +void CacheHasher::add(const std::string &path) { digest.add(path.c_str(), path.length()); auto ts = file_time(path.c_str()); diff --git a/src/utils.hpp b/src/utils.hpp index 368e8e0..919beaf 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -43,7 +43,7 @@ class CacheHasher { public: CacheHasher(std::string hash_file); // add a path and its timestamp to the hash. Should not be called after `finish` - void add(std::string &path); + void add(const std::string &path); // complete the hashing op void finish(); // check if the hashfile matches diff --git a/testcases_data_mods/arc_mod1/scratch_arc/merged/base.merged.xml b/testcases_data_mods/arc_mod1/scratch_arc/merged/base.merged.xml new file mode 100644 index 0000000..043b85d --- /dev/null +++ b/testcases_data_mods/arc_mod1/scratch_arc/merged/base.merged.xml @@ -0,0 +1,6 @@ + + + + 20 25 30 + + diff --git a/testcases_data_mods/arc_mod1/scratch_arc/merged/base.xml b/testcases_data_mods/arc_mod1/scratch_arc/merged/base.xml new file mode 100644 index 0000000..8b18da7 --- /dev/null +++ b/testcases_data_mods/arc_mod1/scratch_arc/merged/base.xml @@ -0,0 +1,6 @@ + + + + 5 10 15 + + diff --git a/testcases_data_mods/arc_mod1/scratch_arc/merged_only/base.merged.xml b/testcases_data_mods/arc_mod1/scratch_arc/merged_only/base.merged.xml new file mode 100644 index 0000000..043b85d --- /dev/null +++ b/testcases_data_mods/arc_mod1/scratch_arc/merged_only/base.merged.xml @@ -0,0 +1,6 @@ + + + + 20 25 30 + + diff --git a/testcases_data_mods/arc_mod1/xml_merge.arc b/testcases_data_mods/arc_mod1/xml_merge.arc new file mode 100644 index 0000000..154fe0d Binary files /dev/null and b/testcases_data_mods/arc_mod1/xml_merge.arc differ diff --git a/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.merged.xml b/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.merged.xml new file mode 100644 index 0000000..043b85d --- /dev/null +++ b/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.merged.xml @@ -0,0 +1,6 @@ + + + + 20 25 30 + + diff --git a/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.xml b/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.xml new file mode 100644 index 0000000..8b18da7 --- /dev/null +++ b/testcases_data_mods/arc_mod1/xml_merge_arc/merged/base.xml @@ -0,0 +1,6 @@ + + + + 5 10 15 + + diff --git a/testcases_data_mods/arc_mod1/xml_merge_arc/merged_only/base.merged.xml b/testcases_data_mods/arc_mod1/xml_merge_arc/merged_only/base.merged.xml new file mode 100644 index 0000000..043b85d --- /dev/null +++ b/testcases_data_mods/arc_mod1/xml_merge_arc/merged_only/base.merged.xml @@ -0,0 +1,6 @@ + + + + 20 25 30 + +