diff --git a/.gitignore b/.gitignore index d9a3e61..be92174 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.nds *.out +range_randomizer \ No newline at end of file diff --git a/Makefile b/Makefile index f7b9377..fbc6293 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -all: test +CXX = g++ +CXXFLAGS = -Wall -Wextra -Wpedantic -Wshadow -test: clean - g++ util.cpp pokedatastructure.cpp \ - pokeid.cpp test.cpp -o test.out +all: main + +main: clean + $(CXX) ${CXXFLAGS} util.cpp pokedatastructure.cpp \ + romprocessor.cpp main.cpp -o range_randomizer clean: - rm -f *.out \ No newline at end of file + rm -f range_randomizer \ No newline at end of file diff --git a/PokeID.bin b/PokeID.bin deleted file mode 100644 index 3f2e188..0000000 Binary files a/PokeID.bin and /dev/null differ diff --git a/README.md b/README.md new file mode 100644 index 0000000..d622511 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Pokemon Ranger: Shadows of Almia Randomizer + +## Usage diff --git a/fieldwaza_name_us.mes b/fieldwaza_name_us.mes deleted file mode 100644 index e86fb84..0000000 Binary files a/fieldwaza_name_us.mes and /dev/null differ diff --git a/main.cpp b/main.cpp new file mode 100644 index 0000000..a31b890 --- /dev/null +++ b/main.cpp @@ -0,0 +1,34 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "romprocessor.h" + +using namespace std; + +// const char* rom_file_path = "rom.nds"; +// const char* out_file_path = "randomized.nds"; + +int main (int argc, char *argv[]) { + if (argc < 3) { + cout << + "Usage: ./ranger_randomizer [input file name] [output file name]" + << endl; + + return 0; + } + + srand(time(NULL)); + + RomProcessor rp(argv[1], argv[2]); + + rp.randomize_rom(); + + return 0; +} \ No newline at end of file diff --git a/lcs.cpp b/old/lcs.cpp similarity index 100% rename from lcs.cpp rename to old/lcs.cpp diff --git a/lcs.h b/old/lcs.h similarity index 100% rename from lcs.h rename to old/lcs.h diff --git a/pokeid.cpp b/old/pokeid.cpp similarity index 93% rename from pokeid.cpp rename to old/pokeid.cpp index 7c292d7..3d0e176 100644 --- a/pokeid.cpp +++ b/old/pokeid.cpp @@ -195,22 +195,4 @@ vector get_pokemon_names_rom (const char *path, uint32_t offset) { vector get_field_moves_rom (const char *path, uint32_t offset) { return read_mes_from_rom(path, offset); -} - -vector get_pokemon_names () { - return read_mes_file("pokemon_name_us.mes"); -} - -vector get_field_moves () { - return read_mes_file("fieldwaza_name_us.mes"); -} - -// int main () { -// vector pkmn_names = get_pokemon_names (); -// vector field_moves = get_field_moves (); - - -// read_pokeID_bin (pkmn_names, field_moves); - -// return 0; -// } \ No newline at end of file +} \ No newline at end of file diff --git a/pokeid.h b/old/pokeid.h similarity index 100% rename from pokeid.h rename to old/pokeid.h diff --git a/pokedatastructure.h b/pokedatastructure.h index 0297e37..903175f 100644 --- a/pokedatastructure.h +++ b/pokedatastructure.h @@ -11,54 +11,54 @@ #include "util.h" class PokeDataStructure { - public: - PokeDataStructure(); - ~PokeDataStructure(); +public: + PokeDataStructure(); + ~PokeDataStructure(); + + void add_pokemon(uint16_t poke_id, uint16_t name_id, uint8_t field_id, uint8_t field_level); + std::pair get_field_move(uint16_t poke_id); + + std::vector get_pokemon_with_geq_field_move (uint16_t poke_id); + + void print_level_order(); + void self_test(); +private: + std::map pokemon_field_moves; + // ugh, hate having two maps + std::map pokemon_names_to_reals; + std::map pokemon_reals_to_names; + + struct TreeNode { + // first is (field_id << 8) | field_level, second is poke_id + // for leaves, field signature + // for non-leaves, max value of left subtree + uint16_t data; + std::vector poke_ids; - void add_pokemon(uint16_t poke_id, uint16_t name_id, uint8_t field_id, uint8_t field_level); - std::pair get_field_move(uint16_t poke_id); + int height; + TreeNode *left; + TreeNode *right; + }; - std::vector get_pokemon_with_geq_field_move (uint16_t poke_id); + TreeNode *root; - void print_level_order(); - void self_test(); - private: - std::map pokemon_field_moves; - // ugh, hate having two maps - std::map pokemon_names_to_reals; - std::map pokemon_reals_to_names; + TreeNode* find_vsplit (TreeNode *node, + uint16_t min, uint16_t max); + void collect_subtree (TreeNode *node, std::vector *vec); + std::vector range_query (uint16_t min, uint16_t max); - struct TreeNode { - // first is (field_id << 8) | field_level, second is poke_id - // for leaves, field signature - // for non-leaves, max value of left subtree - uint16_t data; - std::vector poke_ids; - - int height; - TreeNode *left; - TreeNode *right; - }; + TreeNode* add_node (TreeNode *nod, uint16_t field_signature, + uint16_t poke_id); - TreeNode *root; + bool is_leaf(TreeNode *node); + int height(TreeNode* node); + int get_balance(TreeNode *node); + TreeNode* left_rotate(TreeNode *x); + TreeNode* right_rotate(TreeNode *y); - TreeNode* find_vsplit (TreeNode *node, - uint16_t min, uint16_t max); - void collect_subtree (TreeNode *node, std::vector *vec); - std::vector range_query (uint16_t min, uint16_t max); + void print_current_level (TreeNode* node, int level); - TreeNode* add_node (TreeNode *nod, uint16_t field_signature, - uint16_t poke_id); - - bool is_leaf(TreeNode *node); - int height(TreeNode* node); - int get_balance(TreeNode *node); - TreeNode* left_rotate(TreeNode *x); - TreeNode* right_rotate(TreeNode *y); - - void print_current_level (TreeNode* node, int level); - - void post_order_delete(TreeNode *node); + void post_order_delete(TreeNode *node); }; diff --git a/pokemon_name_us.mes b/pokemon_name_us.mes deleted file mode 100644 index a1e31de..0000000 Binary files a/pokemon_name_us.mes and /dev/null differ diff --git a/romprocessor.cpp b/romprocessor.cpp new file mode 100644 index 0000000..8477596 --- /dev/null +++ b/romprocessor.cpp @@ -0,0 +1,311 @@ +#include "romprocessor.h" + +using namespace std; + +RomProcessor::RomProcessor(const char *in_path, const char *out_path) { + rom_file.open(in_path); + out_file.open(out_path, fstream::in | fstream::out | fstream::trunc); + + out_file << rom_file.rdbuf(); out_file.flush(); // copy over file + + rom_file.seekg(0x48, ios::beg); + fat_offset = read_int(&rom_file); + + get_file_locations(); + + uint32_t pokeid_bin_in_fat = fat_offset + file_locations["data/param/PokeID.bin"]*8; + rom_file.seekg(pokeid_bin_in_fat, ios::beg); + uint32_t pokeid_bin_offset = read_int(&rom_file); + + read_pokeID_rom(pokeid_bin_offset); +} + +RomProcessor::~RomProcessor() { + rom_file.close(); out_file.close(); +} + +void RomProcessor::randomize_rom() { + process_map_files(); +} + +void RomProcessor::process_map_files () { + map inv_start_offsets; // sorted by key in increasing order by default + + rom_file.seekg(0x20, ios::beg); + + for (auto it = file_locations.begin(); it != file_locations.end(); ++it) { + uint32_t file_in_fat = fat_offset + it->second*8; + rom_file.seekg(file_in_fat, ios::beg); + uint32_t file_start = read_int(&rom_file); + inv_start_offsets[file_start] = it->first; + } + + uint32_t last_end_offset = 0; + + rom_file.seekg(0x80, ios::beg); + uint32_t total_size = read_int(&rom_file); + cout << "total size: " << dec << total_size << endl; + + for (auto it = inv_start_offsets.begin(); it != inv_start_offsets.end(); ++it) { + uint32_t file_fat_offset = fat_offset + file_locations[it->second]*8; + rom_file.seekg(file_fat_offset, ios::beg); + uint32_t file_start = read_int(&rom_file); + uint32_t file_end = read_int(&rom_file); + uint32_t file_size = file_end - file_start; + + uint32_t new_file_start = file_start; + uint32_t new_file_end = file_end; + + if (file_start < last_end_offset) { + new_file_start = round_to_multiple(last_end_offset, 512); + + // pad with FF between last_end offset and new start + out_file.seekp(last_end_offset, ios::beg); + for (uint32_t i = last_end_offset; i < new_file_start; i++) + out_file.put((uint8_t) 0xFF); + + } + + if (str_ends_with(it->second, ".map.dat.lz") and !str_ends_with(it->second, "m001_013.map.dat.lz")) { + char *file_bytes = new char[file_size]; + rom_file.seekg(file_start, ios::beg); + rom_file.read(file_bytes, file_size); + + // decompress map file + uint8_t* decompressed_file; + uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); + + delete[] file_bytes; + + // unpack NARC + vector> narc_offsets = decode_NARC(decompressed_file); + uint32_t lyr_start; + for (auto jt = narc_offsets.begin(); jt != narc_offsets.end(); ++jt) { + uint32_t stamp = byte_array_to_int((char*) decompressed_file + jt->first, true); + if (stamp == LYR_MAGIC) { + lyr_start = jt->first; + } + } + + // decode NARC of LYR file + vector> lyr_narc_offsets = decode_NARC(decompressed_file + lyr_start + 0x4); + + for (auto jt = lyr_narc_offsets.begin(); jt != lyr_narc_offsets.end(); ++jt) { + uint32_t narc_container_start = jt->first; + vector> layers = decode_NARC(decompressed_file + lyr_start + 0x4 + narc_container_start); + + // find LYR layer that contains pokemon data and randomize it + for (auto kt = layers.begin(); kt != layers.end(); ++kt) { + uint32_t start = lyr_start + 0x4 + narc_container_start + kt->first; + uint32_t identifier = byte_array_to_int((char*) decompressed_file + start); + if (identifier == 0x9) { // Pokemon data + uint32_t num_entries = byte_array_to_int((char*) decompressed_file + start + 0x4); + for (uint32_t i = 0; i < num_entries; i++) { + + uint32_t pokeid_offset = 0x6 + (i*0xA); + uint16_t poke_id = byte_array_to_short((char*) decompressed_file + start + 0x8 + pokeid_offset); + + vector replacement_mons = pds.get_pokemon_with_geq_field_move(poke_id); + uint16_t new_pokemon = replacement_mons[rand() % replacement_mons.size()]; + + decompressed_file[start+0x8+pokeid_offset] = new_pokemon & 0xFF; + decompressed_file[start+0x8+pokeid_offset+1] = (new_pokemon >> 8) & 0xFF; + } + } + } + } + + // compress file + vector compressed_file; + uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); + + delete[] decompressed_file; + + new_file_end = new_file_start + compressed_size; + + out_file.seekp(file_fat_offset, ios::beg); + write_int((ofstream*) &out_file, new_file_start); // write new start in FAT + + out_file.seekp(file_fat_offset + 4, ios::beg); + write_int((ofstream*) &out_file, new_file_end); // write new end in FAT + + out_file.seekp(new_file_start, ios::beg); + out_file.write((char*) compressed_file.data(), compressed_size); + + for (uint32_t i = compressed_size; i < file_size; i++) + out_file.put((uint8_t) 0xFF); + + } else if (new_file_start != file_start) { + // need to copy over file starting at new offset + char *file_bytes = new char[file_size]; + rom_file.seekg(file_start, ios::beg); + rom_file.read(file_bytes, file_size); + + out_file.seekp(new_file_start, ios::beg); + out_file.write(file_bytes, file_size); + + delete[] file_bytes; + + new_file_end = new_file_start + file_size; + + out_file.seekp(file_fat_offset, ios::beg); + write_int((ofstream*) &out_file, new_file_start); // write new start in FAT + + out_file.seekp(file_fat_offset + 4, ios::beg); + write_int((ofstream*) &out_file, new_file_end); // write new end in FAT + } + + last_end_offset = new_file_end; + } + + cout << "new rom size: " << last_end_offset << endl; + + // update total size in header + out_file.seekp(0x80, ios::beg); + write_int((ofstream*) &out_file, last_end_offset); + + // recompute header checksum + uint8_t header_data[0x15E]; + out_file.seekg(0, ios::beg); + out_file.read((char *) header_data, 0x15E); + + // https://github.com/Luca1991/NDSFactory/blob/master/ndsfactory/ndsfactory.cpp#L156 + uint16_t header_checksum = compute_crc16(header_data, 0x15E); + uint8_t checksum_LE[] = {(uint8_t) (header_checksum & 0xFF), + (uint8_t) ((header_checksum >> 8) & 0xFF)}; + + out_file.seekp(0x15E, ios::beg); + out_file.write((char *) checksum_LE, 2); + + cout << "new header checksum: " << hex << header_checksum << endl; +} + +void RomProcessor::get_file_locations () { + // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#dsmemorymaps + // mapping of filenames (strings) to FAT offsets + rom_file.seekg(0x40, ios::beg); + uint32_t fnt_offset = read_int(&rom_file); + + cout << "FNT offset: " << hex << fnt_offset << endl; + cout << "FAT offset: " << hex << fat_offset << endl; + + // Go to start of fnt + rom_file.seekg(fnt_offset, ios::beg); + + uint32_t subtable_offset = read_int(&rom_file); + uint16_t first_file_id = read_short(&rom_file); + + cout << "First subtable offset: " << hex << subtable_offset << endl; + cout << "First file id: " << hex << first_file_id << endl; + + explore_subtable(subtable_offset, fnt_offset, "", &first_file_id); +} + +void RomProcessor::explore_subtable (uint32_t subtable_offset, + uint32_t fnt_offset, string path, uint16_t *file_id) { + + rom_file.seekg(fnt_offset + subtable_offset, ios::beg); + + // get first byte + char length_type_byte = 0; + rom_file.read(&length_type_byte, 1); + + uint32_t length_type = (uint32_t) (unsigned char)length_type_byte; + + while (length_type != 0) { + if (length_type & 0x80) { + // this is a directory + // get name, appended to end of path + char *new_path_bytes = new char[path.length() + length_type - 0x80 + 2]; + strcpy(new_path_bytes, path.c_str()); + rom_file.read(new_path_bytes + path.length(), length_type - 0x80); + + new_path_bytes [path.length() + length_type - 0x80] = '/'; + new_path_bytes[path.length() + length_type - 0x80+1] = '\0'; + + string new_path(new_path_bytes); + + // after name, offset for subdirectory from start of fnt + uint32_t subdir_id = (uint32_t) read_short(&rom_file); + + int pos_in_subtable = rom_file.tellg(); + + rom_file.seekg(fnt_offset + (subdir_id & 0xFFF)*8, ios::beg); + + uint32_t new_subtable = read_int(&rom_file); + + uint16_t first_file_id = read_short(&rom_file); + + // now, explore this subtable + explore_subtable (new_subtable, fnt_offset, new_path, + &first_file_id); + + // move filestream back to continue while loop + rom_file.seekg(pos_in_subtable, ios::beg); + + delete [] new_path_bytes; + } else { + char *filename_bytes = new char[length_type + 1]; + rom_file.read(filename_bytes, length_type); + filename_bytes[length_type] = '\0'; + + string filename_str(filename_bytes); + string full_path_str(path); + + full_path_str += filename_str; + + file_locations.insert(pair(full_path_str, *file_id)); + *file_id += 1; + + delete[] filename_bytes; + } + + rom_file.read(&length_type_byte, 1); + length_type = (uint32_t) (unsigned char)length_type_byte; + } +} + +void RomProcessor::read_pokeID_rom (uint32_t offset) { + rom_file.seekg(offset, ios::beg); + + // size of file + uint32_t total_size = read_int(&rom_file); + + cout << "File size: " << dec << total_size << " bytes" << endl; + + // other header stuff + uint32_t unique_size = read_int(&rom_file); + uint32_t data_size = read_int(&rom_file); + + char unknown_bytes[4]; + rom_file.read(&unknown_bytes[0], 4); + + + cout << "Unique size: " << hex << unique_size << endl; + cout << "Data size: " << hex << data_size << endl; + + assert(unique_size == UNIQUE_SIZE); + + char unique[UNIQUE_SIZE]; + + rom_file.read(&unique[0], UNIQUE_SIZE); + + for (int i = 0; i < UNIQUE_SIZE; i ++) { + assert(unique[i] == UNIQUE[i]); + } + + uint32_t num_entries = data_size / ENTRY_SIZE; + + cout << "Number of entries: " << dec << num_entries << endl; + + for (uint32_t i = 0; i < num_entries; i ++) { + char entry_data[ENTRY_SIZE]; + rom_file.read(&entry_data[0], ENTRY_SIZE); + + uint16_t name_id = byte_array_to_short(&entry_data[0]); + uint8_t field_id = (uint8_t) entry_data[5]; + uint8_t field_level = (uint8_t) entry_data[6]; + + pds.add_pokemon(i, name_id, field_id, field_level); + } +} \ No newline at end of file diff --git a/romprocessor.h b/romprocessor.h new file mode 100644 index 0000000..07e6b53 --- /dev/null +++ b/romprocessor.h @@ -0,0 +1,48 @@ +#ifndef ROMPROCESSOR_H +#define ROMPROCESSOR_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pokedatastructure.h" +#include "util.h" + +// For PokeID.bin: each pokemon entry is 28 bytes long +// from: https://github.com/SunakazeKun/AlmiaE/blob/6af31527f579037b71fe3461309820f481c42be9/src/com/aurum/almia/game/param/PokeID.java +#define ENTRY_SIZE 0x1C +#define UNIQUE_SIZE 24 + +class RomProcessor { +public: + RomProcessor(const char *in_path, const char *out_path); + ~RomProcessor(); + + void randomize_rom(); + +private: + std::ifstream rom_file; std::fstream out_file; + std::map file_locations; + PokeDataStructure pds; + + uint32_t fat_offset; + char UNIQUE[24] = { + 0x07, 0x03, 0x01, 0x07, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x01, 0x02, + 0x02, 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x05 + }; + + void get_file_locations(); + void read_pokeID_rom (uint32_t offset); + void process_map_files (); + + void explore_subtable (uint32_t subtable_offset, uint32_t fnt_offset, + std::string path, uint16_t *file_id); + +}; + +#endif \ No newline at end of file diff --git a/test.cpp b/test.cpp deleted file mode 100644 index 77e1297..0000000 --- a/test.cpp +++ /dev/null @@ -1,532 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "util.h" -#include "lcs.h" -#include "pokeid.h" -#include "pokedatastructure.h" - -using namespace std; - -const char* rom_file_path = "rom.nds"; -const char* out_file_path = "randomized.nds"; - -// DFS through subtable, return a map from filepath to FAT offset -void explore_subtable (uint32_t subtable_offset, uint32_t fnt_offset, string path, uint16_t *file_id, map *file_locations) { - ifstream rom_file; - rom_file.open(rom_file_path); - - rom_file.seekg(fnt_offset + subtable_offset, ios::beg); - - // get first byte - char length_type_byte = 0; - rom_file.read(&length_type_byte, 1); - - uint32_t length_type = (uint32_t) (unsigned char)length_type_byte; - - while (length_type != 0) { - if (length_type & 0x80) { - // this is a directory - // get name, appended to end of path - char *new_path_bytes = new char[path.length() + length_type - 0x80 + 2]; - strcpy(new_path_bytes, path.c_str()); - rom_file.read(new_path_bytes + path.length(), length_type - 0x80); - - new_path_bytes [path.length() + length_type - 0x80] = '/'; - new_path_bytes[path.length() + length_type - 0x80+1] = '\0'; - - string new_path(new_path_bytes); - - // after name, offset for subdirectory from start of fnt - uint32_t subdir_id = (uint32_t) read_short(&rom_file); - - int pos_in_subtable = rom_file.tellg(); - - rom_file.seekg(fnt_offset + (subdir_id & 0xFFF)*8, ios::beg); - - uint32_t new_subtable = read_int(&rom_file); - - uint16_t first_file_id = read_short(&rom_file); - - // now, explore this subtable - explore_subtable (new_subtable, fnt_offset, new_path, &first_file_id, file_locations); - - // move filestream back to continue while loop - rom_file.seekg(pos_in_subtable, ios::beg); - - delete [] new_path_bytes; - } else { - char *filename_bytes = new char[length_type + 1]; - rom_file.read(filename_bytes, length_type); - filename_bytes[length_type] = '\0'; - - string filename_str(filename_bytes); - string full_path_str(path); - - full_path_str += filename_str; - - // cout << full_path_str << endl; - - file_locations->insert(pair(full_path_str, *file_id)); - *file_id += 1; - - delete[] filename_bytes; - } - - rom_file.read(&length_type_byte, 1); - length_type = (uint32_t) (unsigned char)length_type_byte; - } - - rom_file.close(); -} - -map get_file_locations () { - // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#dsmemorymaps - // mapping of filenames (strings) to FAT offsets - map file_locations; - - ifstream rom_file; - rom_file.open(rom_file_path); - - rom_file.seekg(0x40, ios::beg); - uint32_t fnt_offset = read_int(&rom_file); - - rom_file.seekg(0x48, ios::beg); - uint32_t fat_offset = read_int(&rom_file); - - cout << "FNT offset: " << hex << fnt_offset << endl; - cout << "FAT offset: " << hex << fat_offset << endl; - - // Go to start of fnt - rom_file.seekg(fnt_offset, ios::beg); - - uint32_t subtable_offset = read_int(&rom_file); - uint16_t first_file_id = read_short(&rom_file); - - cout << "First subtable offset: " << hex << subtable_offset << endl; - cout << "First file id: " << hex << first_file_id << endl; - - explore_subtable(subtable_offset, fnt_offset, "", &first_file_id, &file_locations); - - rom_file.close(); - return file_locations; -} - -// https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski -// https://magikos.livejournal.com/7375.html? -// https://github.com/SciresM/FEAT/blob/master/FEAT/DSDecmp/Formats/Nitro/LZ10.cs#L83 -uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, uint8_t **uncompressed_data) { - assert(compressed_size >= 4); - - // First byte 0x10 means file is LZ10 compressed - assert(*compressed_bytes == 0x10); - - // next three bytes are size of uncompressed file - uint32_t uncompressed_size = (uint8_t) *(compressed_bytes + 3) << 16 | - (uint8_t) *(compressed_bytes + 2) << 8 | - (uint8_t) *(compressed_bytes + 1); - - uint8_t* uncompressed_file = new uint8_t[uncompressed_size]; - - uint32_t input_pos = 4; - uint32_t output_pos = 0; - - while (output_pos < uncompressed_size) { - assert(input_pos < compressed_size); - uint8_t flag_byte = (uint8_t) compressed_bytes[input_pos++]; - - for (int i = 1; i <= 8 and output_pos < uncompressed_size and input_pos < compressed_size; i ++) { - bool flag_bit = (flag_byte >> 8-i) & 1; - if (flag_bit) { - // Dictionary entry - uint8_t token1 = (uint8_t) compressed_bytes[input_pos++]; - uint8_t token2 = (uint8_t) compressed_bytes[input_pos++]; - - int disp = ((token1 & 0x0F) << 8) | token2; disp += 1; - uint8_t length = (token1 >> 4) + 3; // plus 3 for some reason? - - int read_start = output_pos - disp; - - assert (disp <= output_pos); - - for (int j = 0; j < length; j++) { - uncompressed_file[output_pos++] = uncompressed_file[read_start + (j % disp)]; - } - } else { - // Raw byte - uncompressed_file[output_pos++] = compressed_bytes[input_pos++]; - } - } - } - - *uncompressed_data = uncompressed_file; - return uncompressed_size; -} - -uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, vector *compressed_data) { - assert(uncompressed_size <= 0xFFFFFF); - - vector buffer; - - buffer.push_back(0x10); - // load uncompressed size into buffer - buffer.push_back((uint8_t) (uncompressed_size & 0xFF)); - buffer.push_back((uint8_t) ((uncompressed_size >> 8) & 0xFF)); - buffer.push_back((uint8_t) ((uncompressed_size >> 16) & 0xFF)); - - int max_disp = 0xFFF; - int max_length = 0xF + 3; - - uint32_t cur_position = 0; - - while (cur_position < uncompressed_size) { - // https://en.wikipedia.org/wiki/Longest_common_substring - // get longest common substring (up to max length of dictionary entry) - // if passes threshold (length >= 3), make it a dictionary entry - // otherwise just copy raw bytes - - buffer.push_back(0); // flag byte - int flag_index = buffer.size() - 1; - - uint8_t flag_byte = 0; - - for (int i = 1; i <= 8 and cur_position < uncompressed_size; i++) { - - // get longest common substring - int start_idx = get_max(0, cur_position - max_disp - 1); - int length_from_start = cur_position - start_idx; - int length_from_current = uncompressed_size - cur_position; - - int best_run_length = 0; - int best_run_start = start_idx; - int cur_run_start = start_idx; - - for (int j = start_idx; j < cur_position and best_run_length < max_length; j++) { - int length = 0, k = 0; - while (cur_position + k < uncompressed_size && uncompressed_data[j+k] == uncompressed_data[cur_position+k] && length < max_length) { - length ++; k++; - } - - if (length > best_run_length) { - best_run_length = get_min(length, max_length); - best_run_start = j; - } - } - - if (best_run_length < 3) // store as raw byte - buffer.push_back(uncompressed_data[cur_position++]); - else { // store as dictionary entry - uint16_t disp = cur_position - best_run_start -1; - - uint8_t length = best_run_length -3; - uint8_t token1 = ((disp >> 8) & 0x0F) | ((length << 4) & 0xF0); - uint8_t token2 = disp & 0xFF; - - buffer.push_back(token1); buffer.push_back(token2); - - flag_byte |= 1 << (8-i); // set corresponding flag bit - cur_position += best_run_length; - } - - } - - buffer[flag_index] = flag_byte; - } - - uint32_t compressed_size = buffer.size(); - - *compressed_data = buffer; - - return compressed_size; -} - -#define NARC_MAGIC 0x4E415243 -#define NARC_FATB 0x46415442 -#define NARC_FNTB 0x464E5442 -#define NARC_FIMG 0x46494D47 -#define LYR_MAGIC 0x4C595200 - -// http://llref.emutalk.net/docs/?file=xml/narc.xml#xml-doc -// http://www.pipian.com/ierukana/hacking/ds_narc.html -// https://www.romhacking.net/documents/%5B469%5Dnds_formats.htm#NARC -// returns vector of start/end offsets for files -vector> decode_NARC (uint8_t *data) { - uint32_t magic = byte_array_to_int((char*) data, true); // big endian due to byte order of magic (bom) - assert(magic == NARC_MAGIC); // magic is "NARC" - - uint32_t file_size = byte_array_to_int((char*) data +0x8); - - uint32_t fatb_offset = 0x10; - uint32_t fatb_stamp = byte_array_to_int((char*) data + fatb_offset); - assert(fatb_stamp == NARC_FATB); - - uint32_t num_files_offset = fatb_offset + 0x8; - uint32_t num_files = byte_array_to_int((char*) data + num_files_offset); - - vector> file_positions; - - for (int i = 0; i < num_files; i++) { - uint32_t start_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + i*8); - uint32_t end_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + (i*8) + 0x4); - file_positions.push_back(pair(start_offset, end_offset)); - } - - uint32_t fntb_offset = num_files_offset + 0x4 + num_files * 8; - uint32_t fntb_stamp = byte_array_to_int((char*) data + fntb_offset); - assert(fntb_stamp == NARC_FNTB); - - uint32_t fntb_size = byte_array_to_int((char *) data + fntb_offset + 0x4); - - uint32_t fimg_offset = fntb_offset + fntb_size; - - uint32_t fimg_stamp = byte_array_to_int((char *) data + fimg_offset); - - assert(fimg_stamp == NARC_FIMG); - - // update vector so it contains absolute offsets (not just relative to start of FIMG) - for (auto it = file_positions.begin(); it != file_positions.end(); ++it) { - it->first = fimg_offset + 0x8 + it->first; // +8 b/c relative offsets start after FIMG header - it->second = fimg_offset + 0x8 + it->second; - } - - return file_positions; -} - -void process_map_files (PokeDataStructure* pds, map file_locations, uint32_t fat_offset) { - map inv_start_offsets; // sorted by key in increasing order by default - - ifstream rom_file(rom_file_path); - fstream out_file(out_file_path); - - rom_file.seekg(0x20, ios::beg); - uint32_t arm9_offset = read_int(&rom_file); - uint32_t arm9_size = read_int(&rom_file); - - for (auto it = file_locations.begin(); it != file_locations.end(); ++it) { - uint32_t file_in_fat = fat_offset + it->second*8; - rom_file.seekg(file_in_fat, ios::beg); - uint32_t file_start = read_int(&rom_file); - inv_start_offsets[file_start] = it->first; - } - - uint32_t last_end_offset = 0; - - for (auto it = inv_start_offsets.begin(); it != inv_start_offsets.end(); ++it) { - uint32_t file_fat_offset = fat_offset + file_locations[it->second]*8; - rom_file.seekg(file_fat_offset, ios::beg); - uint32_t file_start = read_int(&rom_file); - uint32_t file_end = read_int(&rom_file); - uint32_t file_size = file_end - file_start; - - uint32_t new_file_start = file_start; - uint32_t new_file_end = file_end; - - if (file_start < last_end_offset) { - new_file_start = round_to_multiple(last_end_offset, 512); - - // pad with FF between last_end offset and new start - out_file.seekp(last_end_offset, ios::beg); - for (uint32_t i = last_end_offset; i < new_file_start; i++) - out_file.put(0xFF); - - } - - if (str_ends_with(it->second, ".map.dat.lz") and !str_ends_with(it->second, "m001_013.map.dat.lz")) { - char file_bytes[file_size]; - rom_file.seekg(file_start, ios::beg); - rom_file.read(file_bytes, file_size); - - // decompress map file - uint8_t* decompressed_file; - uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); - - // unpack NARC - vector> narc_offsets = decode_NARC(decompressed_file); - uint32_t lyr_start, lyr_end; - for (auto it = narc_offsets.begin(); it != narc_offsets.end(); ++it) { - uint32_t stamp = byte_array_to_int((char*) decompressed_file + it->first, true); - if (stamp == LYR_MAGIC) { - lyr_start = it->first; lyr_end = it->second; - } - } - - // decode NARC of LYR file - vector> lyr_narc_offsets = decode_NARC(decompressed_file + lyr_start + 0x4); - - for (auto jt = lyr_narc_offsets.begin(); jt != lyr_narc_offsets.end(); ++jt) { - uint32_t narc_container_start = jt->first; - vector> layers = decode_NARC(decompressed_file + lyr_start + 0x4 + narc_container_start); - - // find LYR layer that contains pokemon data and randomize it - for (auto it = layers.begin(); it != layers.end(); ++it) { - uint32_t start = lyr_start + 0x4 + narc_container_start + it->first; - uint32_t identifier = byte_array_to_int((char*) decompressed_file + start); - if (identifier == 0x9) { // Pokemon data - uint32_t num_entries = byte_array_to_int((char*) decompressed_file + start + 0x4); - for (int i = 0; i < num_entries; i++) { - - uint32_t pokeid_offset = 0x6 + (i*0xA); - uint16_t poke_id = byte_array_to_short((char*) decompressed_file + start + 0x8 + pokeid_offset); - - vector replacement_mons = pds->get_pokemon_with_geq_field_move(poke_id); - uint16_t new_pokemon = replacement_mons[rand() % replacement_mons.size()]; - - decompressed_file[start+0x8+pokeid_offset] = new_pokemon & 0xFF; - decompressed_file[start+0x8+pokeid_offset+1] = (new_pokemon >> 8) & 0xFF; - } - } - } - } - - // compress file - vector compressed_file; - uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); - - new_file_end = new_file_start + compressed_size; - - if (round_to_multiple(new_file_end, 512) > round_to_multiple(file_end,512)) continue; - - out_file.seekp(file_fat_offset, ios::beg); - write_int((ofstream*) &out_file, new_file_start); // write new start in FAT - - out_file.seekp(file_fat_offset + 4, ios::beg); - write_int((ofstream*) &out_file, new_file_end); // write new end in FAT - - out_file.seekp(new_file_start, ios::beg); - out_file.write((char*) compressed_file.data(), compressed_size); - - for (uint32_t i = compressed_size; i < file_size; i++) - out_file.put(0xFF); - - delete[] decompressed_file; - - } else if (new_file_start != file_start) { - // need to copy over file starting at new offset - char file_bytes[file_size]; - rom_file.seekg(file_start, ios::beg); - rom_file.read(file_bytes, file_size); - - out_file.seekp(new_file_start, ios::beg); - out_file.write(file_bytes, file_size); - - new_file_end = new_file_start + file_size; - - out_file.seekp(file_fat_offset, ios::beg); - write_int((ofstream*) &out_file, new_file_start); // write new start in FAT - - out_file.seekp(file_fat_offset + 4, ios::beg); - write_int((ofstream*) &out_file, new_file_end); // write new end in FAT - } - - last_end_offset = new_file_end; - } - - cout << "new rom size: " << last_end_offset << endl; - - // update total size in header - out_file.seekp(0x80, ios::beg); - write_int((ofstream*) &out_file, last_end_offset); - - // recompute header checksum - uint8_t header_data[0x15E]; - out_file.seekg(0, ios::beg); - out_file.read((char *) header_data, 0x15E); - - // https://github.com/Luca1991/NDSFactory/blob/master/ndsfactory/ndsfactory.cpp#L156 - uint16_t header_checksum = compute_crc16(header_data, 0x15E); - uint8_t checksum_LE[] = {(uint8_t) (header_checksum & 0xFF), - (uint8_t) ((header_checksum >> 8) & 0xFF)}; - - out_file.seekp(0x15E, ios::beg); - out_file.write((char *) checksum_LE, 2); - - cout << "new header checksum: " << hex << header_checksum << endl; - - // TODO decompress .map.dat.lz files (LZ10 compression) - // https://github.com/SunakazeKun/AlmiaE/blob/master/src/com/aurum/almia/game/Compression.java - // https://ndspy.readthedocs.io/en/latest/_modules/ndspy/lz10.html#decompress - - // m001_004 - // uint32_t file_fat_offset = fat_offset + file_locations["data/field/map/m001_004.map.dat.lz"]*8; - // rom_file.seekg(file_fat_offset, ios::beg); - // uint32_t file_start = read_int(&rom_file); - // uint32_t file_end = read_int(&rom_file); - // uint32_t file_size = file_end - file_start; - - // char file_bytes[file_size]; - // rom_file.seekg(file_start, ios::beg); - // rom_file.read(file_bytes, file_size); - - // uint8_t* decompressed_file; - // uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); - - // vector compressed_file; - // uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); - - // ofstream decomp_file("recompressed_001_004.map.dat"); - // decomp_file.write((char *) compressed_file.data(), compressed_size); - - // delete[] decompressed_file; - - rom_file.close(); out_file.close(); -} - -int main () { - srand(time(NULL)); - - map file_locations = get_file_locations (); - - ifstream rom_file(rom_file_path); - - ofstream out_file(out_file_path); // copy over file - out_file << rom_file.rdbuf(); out_file.close(); - - rom_file.seekg(0x48, ios::beg); - uint32_t fat_offset = read_int(&rom_file); - - uint32_t pkmn_name_in_fat = fat_offset + file_locations["data/message/etc/pokemon_name_us.mes"]*8; - uint32_t field_move_in_fat = fat_offset + file_locations["data/message/etc/fieldwaza_name_us.mes"]*8; - uint32_t pokeid_bin_in_fat = fat_offset + file_locations["data/param/PokeID.bin"]*8; - - rom_file.seekg(pkmn_name_in_fat, ios::beg); - uint32_t pkmn_name_offset = read_int(&rom_file); - - rom_file.seekg(field_move_in_fat, ios::beg); - uint32_t field_move_offset = read_int(&rom_file); - - rom_file.seekg(pokeid_bin_in_fat, ios::beg); - uint32_t pokeid_bin_offset = read_int(&rom_file); - - cout << "Pokemon name offset: " << pkmn_name_offset << endl; - cout << "Field name offset: " << field_move_offset << endl; - cout << "PokeID bin offset: " << pokeid_bin_offset << endl; - - vector pkmn_names = get_pokemon_names_rom (rom_file_path, - pkmn_name_offset); - vector field_moves = get_field_moves_rom (rom_file_path, - field_move_offset); - - // cout << pkmn_names.size() << endl; - - PokeDataStructure pds; - - // TODO: make this a PDS member function - read_pokeID_rom (&pds, rom_file_path, pokeid_bin_offset); - - rom_file.seekg(0x80, ios::beg); - uint32_t total_size = read_int(&rom_file); - cout << "total size: " << dec << total_size << endl; - - process_map_files(&pds, file_locations, fat_offset); - - rom_file.close(); - - return 0; -} \ No newline at end of file diff --git a/util.cpp b/util.cpp index a6ee1d1..7d17302 100644 --- a/util.cpp +++ b/util.cpp @@ -77,7 +77,7 @@ uint32_t round_to_multiple (uint32_t n, uint32_t multiple) { // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#biosmiscfunctions // https://github.com/Zetten/hachoir/blob/master/hachoir-parser/hachoir_parser/program/nds.py uint16_t compute_crc16 (uint8_t *bytes, int size) { - uint16_t val[8] = {0xC0C1,0xC181,0xC301,0xC601,0xCC01,0xD801,0xF001,0xA001}; + // uint16_t val[8] = {0xC0C1,0xC181,0xC301,0xC601,0xCC01,0xD801,0xF001,0xA001}; uint16_t crc = 0xFFFF; for (int i = 0; i < size; i++) { @@ -91,4 +91,172 @@ uint16_t compute_crc16 (uint8_t *bytes, int size) { } return crc; +} + +// https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski +// https://magikos.livejournal.com/7375.html? +// https://github.com/SciresM/FEAT/blob/master/FEAT/DSDecmp/Formats/Nitro/LZ10.cs#L83 +uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, uint8_t **uncompressed_data) { + assert(compressed_size >= 4); + + // First byte 0x10 means file is LZ10 compressed + assert(*compressed_bytes == 0x10); + + // next three bytes are size of uncompressed file + uint32_t uncompressed_size = (uint8_t) *(compressed_bytes + 3) << 16 | + (uint8_t) *(compressed_bytes + 2) << 8 | + (uint8_t) *(compressed_bytes + 1); + + uint8_t* uncompressed_file = new uint8_t[uncompressed_size]; + + uint32_t input_pos = 4; + uint32_t output_pos = 0; + + while (output_pos < uncompressed_size) { + assert(input_pos < compressed_size); + uint8_t flag_byte = (uint8_t) compressed_bytes[input_pos++]; + + for (int i = 1; i <= 8 and output_pos < uncompressed_size and input_pos < compressed_size; i ++) { + bool flag_bit = (flag_byte >> (8-i)) & 1; + if (flag_bit) { + // Dictionary entry + uint8_t token1 = (uint8_t) compressed_bytes[input_pos++]; + uint8_t token2 = (uint8_t) compressed_bytes[input_pos++]; + + int disp = ((token1 & 0x0F) << 8) | token2; disp += 1; + uint8_t length = (token1 >> 4) + 3; // plus 3 for some reason? + + int read_start = output_pos - disp; + + assert ((uint32_t) disp <= output_pos); + + for (int j = 0; j < length; j++) { + uncompressed_file[output_pos++] = uncompressed_file[read_start + (j % disp)]; + } + } else { + // Raw byte + uncompressed_file[output_pos++] = compressed_bytes[input_pos++]; + } + } + } + + *uncompressed_data = uncompressed_file; + return uncompressed_size; +} + +uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, vector *compressed_data) { + assert(uncompressed_size <= 0xFFFFFF); + + vector buffer; + + buffer.push_back(0x10); + // load uncompressed size into buffer + buffer.push_back((uint8_t) (uncompressed_size & 0xFF)); + buffer.push_back((uint8_t) ((uncompressed_size >> 8) & 0xFF)); + buffer.push_back((uint8_t) ((uncompressed_size >> 16) & 0xFF)); + + int max_disp = 0xFFF; + int max_length = 0xF + 3; + + uint32_t cur_position = 0; + + while (cur_position < uncompressed_size) { + // https://en.wikipedia.org/wiki/Longest_common_substring + // get longest common substring (up to max length of dictionary entry) + // if passes threshold (length >= 3), make it a dictionary entry + // otherwise just copy raw bytes + + buffer.push_back(0); // flag byte + int flag_index = buffer.size() - 1; + + uint8_t flag_byte = 0; + + for (int i = 1; i <= 8 and cur_position < uncompressed_size; i++) { + + // get longest common substring + int start_idx = get_max(0, cur_position - max_disp - 1); + int best_run_length = 0; + int best_run_start = start_idx; + + for (uint32_t j = start_idx; j < cur_position and best_run_length < max_length; j++) { + int length = 0, k = 0; + while (cur_position + k < uncompressed_size && uncompressed_data[j+k] == uncompressed_data[cur_position+k] && length < max_length) { + length ++; k++; + } + + if (length > best_run_length) { + best_run_length = get_min(length, max_length); + best_run_start = j; + } + } + + if (best_run_length < 3) // store as raw byte + buffer.push_back(uncompressed_data[cur_position++]); + else { // store as dictionary entry + uint16_t disp = cur_position - best_run_start -1; + + uint8_t length = best_run_length -3; + uint8_t token1 = ((disp >> 8) & 0x0F) | ((length << 4) & 0xF0); + uint8_t token2 = disp & 0xFF; + + buffer.push_back(token1); buffer.push_back(token2); + + flag_byte |= 1 << (8-i); // set corresponding flag bit + cur_position += best_run_length; + } + + } + + buffer[flag_index] = flag_byte; + } + + uint32_t compressed_size = buffer.size(); + + *compressed_data = buffer; + + return compressed_size; +} + +// http://llref.emutalk.net/docs/?file=xml/narc.xml#xml-doc +// http://www.pipian.com/ierukana/hacking/ds_narc.html +// https://www.romhacking.net/documents/%5B469%5Dnds_formats.htm#NARC +// returns vector of start/end offsets for files +vector> decode_NARC (uint8_t *data) { + uint32_t magic = byte_array_to_int((char*) data, true); // big endian due to byte order of magic (bom) + assert(magic == NARC_MAGIC); // magic is "NARC" + + uint32_t fatb_offset = 0x10; + uint32_t fatb_stamp = byte_array_to_int((char*) data + fatb_offset); + assert(fatb_stamp == NARC_FATB); + + uint32_t num_files_offset = fatb_offset + 0x8; + uint32_t num_files = byte_array_to_int((char*) data + num_files_offset); + + vector> file_positions; + + for (uint32_t i = 0; i < num_files; i++) { + uint32_t start_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + i*8); + uint32_t end_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + (i*8) + 0x4); + file_positions.push_back(pair(start_offset, end_offset)); + } + + uint32_t fntb_offset = num_files_offset + 0x4 + num_files * 8; + uint32_t fntb_stamp = byte_array_to_int((char*) data + fntb_offset); + assert(fntb_stamp == NARC_FNTB); + + uint32_t fntb_size = byte_array_to_int((char *) data + fntb_offset + 0x4); + + uint32_t fimg_offset = fntb_offset + fntb_size; + + uint32_t fimg_stamp = byte_array_to_int((char *) data + fimg_offset); + + assert(fimg_stamp == NARC_FIMG); + + // update vector so it contains absolute offsets (not just relative to start of FIMG) + for (auto it = file_positions.begin(); it != file_positions.end(); ++it) { + it->first = fimg_offset + 0x8 + it->first; // +8 b/c relative offsets start after FIMG header + it->second = fimg_offset + 0x8 + it->second; + } + + return file_positions; } \ No newline at end of file diff --git a/util.h b/util.h index 0ff7f8f..f2bdeaf 100644 --- a/util.h +++ b/util.h @@ -2,8 +2,16 @@ #define UTIL_H #include +#include #include #include +#include + +#define NARC_MAGIC 0x4E415243 +#define NARC_FATB 0x46415442 +#define NARC_FNTB 0x464E5442 +#define NARC_FIMG 0x46494D47 +#define LYR_MAGIC 0x4C595200 uint32_t byte_array_to_int (char* bytes, bool big_endian=false); uint16_t byte_array_to_short (char* bytes, bool big_endian=false); @@ -22,4 +30,13 @@ uint32_t round_to_multiple (uint32_t n, uint32_t multiple); uint16_t compute_crc16 (uint8_t *bytes, int size); +uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, + uint8_t **uncompressed_data); + +uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, + std::vector *compressed_data); + +std::vector> decode_NARC (uint8_t *data); + + #endif \ No newline at end of file