From 47cd143de68ac758e631f34f82905f129c3558cf Mon Sep 17 00:00:00 2001 From: Philippe Symons Date: Fri, 18 Jul 2025 16:19:34 +0200 Subject: [PATCH] Replace ZX0 by the builtin LZ10 compression. LZ10 decompression is builtin to the GBA's bios, so we don't need ZX0. It's also significantly faster (618 usec instead of 2311 usec in my personal benchmark code for decompression of the same data) And it seems like by doing so, we saved 1 KB as well! So, seems like replacing ZX0 is the right move. The reason I didn't initially is because I misunderstood the documentation. I assumed LZ77UnCompWram could only uncompress into EWRAM, not IWRAM. But it turns out it can do both. And using standardized tools is usually better than using a custom implementation. The only downside of this right now, is that we can no longer stream text tables through a smaller buffer than the entire decompressed size. Anyway, things seem to work fine, so bye bye ZX0. It's been fun. --- Makefile | 7 +- compress_lz10.sh | 4 + include/text_data_table.h | 53 +--- include/zx0_decompressor.h | 49 ---- source/box_menu.cpp | 1 - source/main.cpp | 59 ++++ source/pokedex.cpp | 12 +- source/pokemon.cpp | 8 +- source/pokemon_data.cpp | 47 ++-- source/pokemon_party.cpp | 35 +-- source/rom_data.cpp | 32 +-- source/text_data_table.cpp | 123 +-------- source/text_engine.cpp | 5 +- source/zx0_decompressor.cpp | 326 ---------------------- text_helper/main.py | 24 +- tools/compressZX0/Makefile | 43 --- tools/compressZX0/main.cpp | 436 ------------------------------ tools/data-generator/src/main.cpp | 2 +- 18 files changed, 151 insertions(+), 1115 deletions(-) create mode 100755 compress_lz10.sh delete mode 100644 include/zx0_decompressor.h delete mode 100644 source/zx0_decompressor.cpp delete mode 100644 tools/compressZX0/Makefile delete mode 100644 tools/compressZX0/main.cpp diff --git a/Makefile b/Makefile index bf04eec..62c391c 100644 --- a/Makefile +++ b/Makefile @@ -146,11 +146,10 @@ all: $(BUILD) generate_data: mkdir -p data mkdir -p to_compress - @env -i "PATH=$(PATH)" $(MAKE) -C tools/compressZX0 @env -i "PATH=$(PATH)" $(MAKE) -C tools/data-generator @tools/data-generator/data-generator to_compress @python3 text_helper/main.py - @find to_compress -name "*.bin" | xargs -i tools/compressZX0/compressZX0 {} data/ + @find to_compress -name "*.bin" -print0 | xargs -0 -n1 ./compress_lz10.sh #--------------------------------------------------------------------------------- $(BUILD): generate_data @@ -163,7 +162,6 @@ $(BUILD): generate_data #--------------------------------------------------------------------------------- clean: @echo clean ... - @$(MAKE) -C tools/compressZX0 clean @$(MAKE) -C tools/data-generator clean @$(MAKE) -C loader clean @rm -fr $(BUILD) $(TARGET).elf $(TARGET).gba data/ to_compress/ @@ -176,9 +174,6 @@ BINFILES := $(foreach dir,../$(DATA),$(notdir $(wildcard $(dir)/*.*))) export OFILES_BIN := $(addsuffix .o,$(BINFILES)) OFILES := $(OFILES_BIN) $(OFILES) -# Optimize zx0_decompressor for speed -zx0_decompressor.o: CXXFLAGS += -O2 - #--------------------------------------------------------------------------------- # main targets #--------------------------------------------------------------------------------- diff --git a/compress_lz10.sh b/compress_lz10.sh new file mode 100755 index 0000000..223ce80 --- /dev/null +++ b/compress_lz10.sh @@ -0,0 +1,4 @@ +#!/bin/sh +infile="$1" +outfile="data/$(basename "$infile" .bin)_lz10.bin" +gbalzss e "$infile" "$outfile" diff --git a/include/text_data_table.h b/include/text_data_table.h index 92fd6ee..3b942e3 100644 --- a/include/text_data_table.h +++ b/include/text_data_table.h @@ -8,10 +8,6 @@ * and then gives you utility functions to retrieve the text entries * * But it requires a buffer large enough to contain the entire decompressed table. - * It will also completely decompress the table, which may not what you want. - * - * If you want to use a text table in a streamed manner (use smaller decompression buffer and only decompress what's needed) - * consider using streamed_text_data_table instead. */ class text_data_table { @@ -40,54 +36,7 @@ public: uint16_t get_text_entry_size(uint16_t index) const; private: uint8_t *decompression_buffer_; -}; - -/** - * This class is an alternative to translated_text table. - * It provides the same functionality, yet in a streamed manner. - * - * This allows you to use a decompression_buffer that is smaller than the fully decoded text table and only decompress until - * you have what you need. - * - * To make sure we have access to the table index at all times, you need to specify a buffer to hold the table index as well. - * - * REQUIREMENT: decompression_buffer needs to be larger than the zx0 window size!! - */ -class streamed_text_data_table -{ -public: - streamed_text_data_table(uint8_t *decompression_buffer, uint32_t decompression_buffer_size, uint8_t *index_buffer); - - /** - * This function sets up the zx0 decompressor and decompresses the index into the index_buffer - */ - void decompress(const uint8_t *compressed_table); - - /** - * Returns the number of text entries in the decompression_buffer_ - */ - uint16_t get_number_of_text_entries() const; - - /** - * This function returns a pointer to a text entry in the decompression_buffer - */ - const uint8_t* get_text_entry(uint16_t index); - - /** - * This function returns the text entry size in bytes at the given index - */ - uint16_t get_text_entry_size(uint16_t index) const; -private: - uint8_t* get_window_start() const; - uint8_t* get_window_end() const; - uint16_t get_current_zx0_window_size() const; - - const uint8_t *compressed_table_; - uint8_t *decompression_buffer_; - uint32_t decompression_buffer_size_; - uint8_t *index_buffer_; - mutable uint16_t bytes_decompressed_; - uint16_t last_chunk_size_; + uint32_t decompressed_size_; }; #endif \ No newline at end of file diff --git a/include/zx0_decompressor.h b/include/zx0_decompressor.h deleted file mode 100644 index 7a812b1..0000000 --- a/include/zx0_decompressor.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef _ZX0_DECODE_H -#define _ZX0_DECODE_H - -#include - -#define ZX0_DEFAULT_WINDOW_SIZE 2048 - -// The ZX0 decompressor offers functionality to decompress data -// compressed with the ZX0 algorithm (see tools/compressZX0) -// This algorithm was invented by Einar Saukas -// Original implementation can be found here: https://github.com/einar-saukas/ZX0 -// However, we've implemented a custom variant of this algorithm. -// (for instance: we're storing the uncompressed size in the first 2 bytes in little endian) -extern "C" -{ - /** - * @brief This function slots the specified input_data buffer into the zx0 decompressor. - * Calling this function effectively resets the ZX0 decompressors' internal state. - */ - void zx0_decompressor_start(uint8_t *output_buffer, const uint8_t *input_data); - - /** - * @brief This function returns the uncompressed size of the current input_data buffer. - * It reads this from the first 2 bytes of input_data - */ - uint32_t zx0_decompressor_get_decompressed_size(); - - /** - * @brief This function copies of decompressed data into the specified - * It will append to the existing output_buffer you set earlier with zx0_decompressor_start() - */ - void zx0_decompressor_read(uint32_t num_bytes); - - /** - * @brief this function does a partial decompress into output_buffer. - * HOWEVER: zx0 decompression requires you to use previously decompressed bytes to decompress the current ones. - * In order to accomplish this, output_buffer MUST NOT point to the start of the buffer!! - * Instead it should refer to a point within the buffer with previously decoded bytes available before it, - * with up to the bytes available before that point. - * (So if you decoded >= window size, bytes should be available before output_buffer. - * If not, before output_buffer should have ) - * - * This function is intended as a way to read data in a "streamed" way into a smaller buffer that is smaller than the actual decompressed file size. - * NOTE: when used in a loop, you should manually move the decompressed data from the previous iteration to the front of the buffer (output_buffer + window_size - num_bytes) - */ - void zx0_decompressor_read_partial(uint8_t *output_buffer, uint16_t num_bytes); -} - -#endif \ No newline at end of file diff --git a/source/box_menu.cpp b/source/box_menu.cpp index 78d9776..4a91fd9 100644 --- a/source/box_menu.cpp +++ b/source/box_menu.cpp @@ -10,7 +10,6 @@ #include "text_engine.h" #include "translated_text.h" #include "text_data_table.h" -#include "zx0_decompressor.h" Box_Menu::Box_Menu() {}; diff --git a/source/main.cpp b/source/main.cpp index e144658..c549807 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -213,6 +213,59 @@ void first_load_message(void) tte_erase_rect(0, 0, H_MAX, V_MAX); } +#include "translated_text.h" + +#define TIMER_ENABLE 0x80 +#define TIMER_CASCADE 0x4 +#define TIMER_FREQ_1 0x0 // 16.78 MHz +#define TIMER_FREQ_64 0x1 // 262,144 Hz +#define TIMER_FREQ_256 0x2 // 65,536 Hz +#define TIMER_FREQ_1024 0x3 // 16,384 Hz + +int test_decompress() +{ + uint16_t charset[256]; + + // Reset both timers + REG_TM0CNT = 0; + REG_TM1CNT = 0; + REG_TM0D = 0; + REG_TM1D = 0; + + // Set up TIMER0: count with no prescaler + REG_TM0CNT = TIMER_ENABLE | TIMER_FREQ_1; + // Set up TIMER1: cascade mode (increment when TIMER0 overflows) + REG_TM1CNT = TIMER_ENABLE | TIMER_CASCADE; + + load_localized_charset(charset, 3, ENG_ID); + + // Read combined 32-bit timer value + const u32 ticks = ((u32)REG_TM1D << 16) | REG_TM0D; + + // Stop timers + REG_TM0CNT = 0; + REG_TM1CNT = 0; + + create_textbox(4, 1, 160, 80, true); + + ptgb_write_debug(charset, "Test results:\n\nDecompress: ", true); + ptgb_write_debug(charset, ptgb::to_string(ticks * 1000 / 16777), true); + ptgb_write_debug(charset, " usec\n", true); + + + while (true) + { + if (key_hit(KEY_B)) + { + hide_text_box(); + reset_textbox(); + return 0; + } + global_next_frame(); + } + return 0; +} + int credits() { u8 text_decompression_buffer[2048]; @@ -249,6 +302,11 @@ int credits() curr_credits_num++; update = true; } + if(key_hit(KEY_SELECT)) + { + return test_decompress(); + } + #if 0 if (ENABLE_DEBUG_SCREEN && key_hit(KEY_SELECT)) { char hexBuffer[16]; @@ -325,6 +383,7 @@ int credits() global_next_frame(); } } +#endif global_next_frame(); } diff --git a/source/pokedex.cpp b/source/pokedex.cpp index 88738af..7aa2c6a 100644 --- a/source/pokedex.cpp +++ b/source/pokedex.cpp @@ -10,9 +10,8 @@ #include "button_handler.h" #include "translated_text.h" #include "text_engine.h" -#include "zx0_decompressor.h" #include "text_data_table.h" -#include "TYPES_zx0_bin.h" +#include "TYPES_lz10_bin.h" Dex dex_array[DEX_MAX]; int dex_shift = 0; @@ -89,7 +88,7 @@ void pokedex_init() obj_hide(down_arrow); } -#include "gen_3_charsets_zx0_bin.h" +#include "gen_3_charsets_lz10_bin.h" #include "libstd_replacements.h" int pokedex_loop() @@ -100,11 +99,8 @@ int pokedex_loop() u8 decompression_buffer[3072]; u16 charset[256]; - zx0_decompressor_start((u8*)TYPES, TYPES_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); - - zx0_decompressor_start((u8*)charset, gen_3_charsets_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(TYPES_lz10_bin, (u8*)TYPES); + LZ77UnCompWram(gen_3_charsets_lz10_bin, (u8*)charset); load_general_table_text_entries(decompression_buffer, kanto_name, johto_name); diff --git a/source/pokemon.cpp b/source/pokemon.cpp index 86e7c2b..31ae81e 100644 --- a/source/pokemon.cpp +++ b/source/pokemon.cpp @@ -6,8 +6,7 @@ #include "save_data_manager.h" #include "debug_mode.h" #include "text_engine.h" -#include "zx0_decompressor.h" -#include "JPN_NAMES_zx0_bin.h" +#include "JPN_NAMES_lz10_bin.h" Pokemon::Pokemon() {}; @@ -215,9 +214,8 @@ void Pokemon::convert_to_gen_three(PokemonTables& data_tables, Conversion_Types u16 cur_char; data_tables.load_gen3_charset(language); - // setup the zx0 decompressor to decompress the JPN_NAMES table - zx0_decompressor_start((u8*)JPN_NAMES, JPN_NAMES_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + + LZ77UnCompWram(JPN_NAMES_lz10_bin, (u8*)JPN_NAMES); for (int i = 0; i < 6; i++) { // Read the JPN name and convert it diff --git a/source/pokemon_data.cpp b/source/pokemon_data.cpp index c082f0a..994f2bd 100644 --- a/source/pokemon_data.cpp +++ b/source/pokemon_data.cpp @@ -1,15 +1,14 @@ #include "pokemon_data.h" -#include "zx0_decompressor.h" -#include "EXP_GROUPS_zx0_bin.h" -#include "GENDER_RATIO_zx0_bin.h" -#include "NUM_ABILITIES_zx0_bin.h" -#include "FIRST_MOVES_zx0_bin.h" -#include "POWER_POINTS_zx0_bin.h" -#include "EVENT_PKMN_zx0_bin.h" -#include "TYPES_zx0_bin.h" -#include "gen_1_charsets_zx0_bin.h" -#include "gen_2_charsets_zx0_bin.h" -#include "gen_3_charsets_zx0_bin.h" +#include "EXP_GROUPS_lz10_bin.h" +#include "GENDER_RATIO_lz10_bin.h" +#include "NUM_ABILITIES_lz10_bin.h" +#include "FIRST_MOVES_lz10_bin.h" +#include "POWER_POINTS_lz10_bin.h" +#include "EVENT_PKMN_lz10_bin.h" +#include "TYPES_lz10_bin.h" +#include "gen_1_charsets_lz10_bin.h" +#include "gen_2_charsets_lz10_bin.h" +#include "gen_3_charsets_lz10_bin.h" #include #include @@ -744,8 +743,7 @@ static void load_table(u8 *table, const u8* source, bool &loadedBool) { return; } - zx0_decompressor_start(table, source); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(source, table); loadedBool = true; } @@ -764,37 +762,37 @@ PokemonTables::PokemonTables() void PokemonTables::load_exp_groups() { - load_table(EXP_GROUPS, EXP_GROUPS_zx0_bin, exp_groups_loaded); + load_table(EXP_GROUPS, EXP_GROUPS_lz10_bin, exp_groups_loaded); } void PokemonTables::load_gender_ratios() { - load_table(GENDER_RATIO, GENDER_RATIO_zx0_bin, gender_ratios_loaded); + load_table(GENDER_RATIO, GENDER_RATIO_lz10_bin, gender_ratios_loaded); } void PokemonTables::load_num_abilities() { - load_table((uint8_t*)NUM_ABILITIES, NUM_ABILITIES_zx0_bin, num_abilities_loaded); + load_table((uint8_t*)NUM_ABILITIES, NUM_ABILITIES_lz10_bin, num_abilities_loaded); } void PokemonTables::load_first_moves() { - load_table(FIRST_MOVES, FIRST_MOVES_zx0_bin, first_moves_loaded); + load_table(FIRST_MOVES, FIRST_MOVES_lz10_bin, first_moves_loaded); } void PokemonTables::load_power_points() { - load_table(POWER_POINTS, POWER_POINTS_zx0_bin, power_points_loaded); + load_table(POWER_POINTS, POWER_POINTS_lz10_bin, power_points_loaded); } void PokemonTables::load_event_pkmn() { - load_table((uint8_t*)EVENT_PKMN, EVENT_PKMN_zx0_bin, event_pkmn_loaded); + load_table((uint8_t*)EVENT_PKMN, EVENT_PKMN_lz10_bin, event_pkmn_loaded); } void PokemonTables::load_types() { - load_table((uint8_t*)TYPES, TYPES_zx0_bin, types_loaded); + load_table((uint8_t*)TYPES, TYPES_lz10_bin, types_loaded); } void PokemonTables::load_input_charset(byte gen, byte lang) @@ -909,21 +907,20 @@ void load_localized_charset(u16 *output_char_array, byte gen, byte lang) switch(gen) { case 1: - input_data = gen_1_charsets_zx0_bin; + input_data = gen_1_charsets_lz10_bin; break; case 2: - input_data = gen_2_charsets_zx0_bin; + input_data = gen_2_charsets_lz10_bin; break; case 3: - input_data = gen_3_charsets_zx0_bin; + input_data = gen_3_charsets_lz10_bin; break; default: // Invalid generation, return without doing anything return; } - zx0_decompressor_start(generation_charsets, input_data); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(input_data, generation_charsets); memcpy(output_char_array, generation_charsets + (lang_index * 256 * sizeof(u16)), 256 * sizeof(u16)); // copy the charset into the output array } diff --git a/source/pokemon_party.cpp b/source/pokemon_party.cpp index ea80b37..e9fc8a0 100644 --- a/source/pokemon_party.cpp +++ b/source/pokemon_party.cpp @@ -7,13 +7,12 @@ #include "gb_rom_values/gb_rom_values.h" #include "sprite_data.h" #include "box_menu.h" -#include "zx0_decompressor.h" #include "payload_file_reader.h" -#include "gb_rom_values_eng_zx0_bin.h" -#include "gb_rom_values_fre_zx0_bin.h" -#include "gb_gen1_payloads_RB_zx0_bin.h" -#include "gb_gen1_payloads_Y_zx0_bin.h" -#include "gb_gen2_payloads_zx0_bin.h" +#include "gb_rom_values_eng_lz10_bin.h" +#include "gb_rom_values_fre_lz10_bin.h" +#include "gb_gen1_payloads_RB_lz10_bin.h" +#include "gb_gen1_payloads_Y_lz10_bin.h" +#include "gb_gen2_payloads_lz10_bin.h" static const byte gen1_rb_debug_box_data[0x462] = { // Num of Pokemon @@ -273,19 +272,19 @@ bool Pokemon_Party::load_gb_rom() switch(lang) { case ENG_ID: - compressed_rom_table = gb_rom_values_eng_zx0_bin; + compressed_rom_table = gb_rom_values_eng_lz10_bin; break; case FRE_ID: - compressed_rom_table = gb_rom_values_fre_zx0_bin; + compressed_rom_table = gb_rom_values_fre_lz10_bin; break; default: // no rom table for this language return false; } - zx0_decompressor_start(gb_rom_table_buffer, compressed_rom_table); - rom_table_size = zx0_decompressor_get_decompressed_size(); - zx0_decompressor_read(rom_table_size); + // byte 2-4 of the compressed data store the decompressed size + rom_table_size = compressed_rom_table[1] | (compressed_rom_table[2] << 8) | (compressed_rom_table[3] << 16); + LZ77UnCompWram(compressed_rom_table, gb_rom_table_buffer); cur = gb_rom_table_buffer; while(cur < gb_rom_table_buffer + rom_table_size) @@ -347,6 +346,7 @@ void Pokemon_Party::init_payload() { u8 decompression_buffer[1512]; const u8 *payload_src; + u32 payload_file_size; //WARNING: Ensure sure decompression_buffer is large enough! @@ -354,21 +354,22 @@ void Pokemon_Party::init_payload() { if(curr_gb_rom.version == YELLOW_ID) { - payload_src = gb_gen1_payloads_Y_zx0_bin; + payload_src = gb_gen1_payloads_Y_lz10_bin; } else { - payload_src = gb_gen1_payloads_RB_zx0_bin; + payload_src = gb_gen1_payloads_RB_lz10_bin; } } else // if(curr_gb_rom.generation == 2) { - payload_src = gb_gen2_payloads_zx0_bin; + payload_src = gb_gen2_payloads_lz10_bin; } - zx0_decompressor_start(decompression_buffer, payload_src); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + // byte 2-4 of the compressed data store the decompressed size + payload_file_size = payload_src[1] | (payload_src[2] << 8) | (payload_src[3] << 16); + LZ77UnCompWram(payload_src, decompression_buffer); - payload_file_reader payload_reader(decompression_buffer, zx0_decompressor_get_decompressed_size()); + payload_file_reader payload_reader(decompression_buffer, payload_file_size); payload_reader.read_payload(current_payload, curr_gb_rom.language, curr_gb_rom.version); } \ No newline at end of file diff --git a/source/rom_data.cpp b/source/rom_data.cpp index d39eb11..a4b7e67 100644 --- a/source/rom_data.cpp +++ b/source/rom_data.cpp @@ -5,13 +5,12 @@ #include "text_engine.h" #include "gba_rom_values/gba_rom_values.h" #include "libraries/nanoprintf/nanoprintf.h" -#include "zx0_decompressor.h" -#include "gba_rom_values_eng_zx0_bin.h" -#include "gba_rom_values_fre_zx0_bin.h" -#include "gba_rom_values_ger_zx0_bin.h" -#include "gba_rom_values_ita_zx0_bin.h" -#include "gba_rom_values_jpn_zx0_bin.h" -#include "gba_rom_values_spa_zx0_bin.h" +#include "gba_rom_values_eng_lz10_bin.h" +#include "gba_rom_values_fre_lz10_bin.h" +#include "gba_rom_values_ger_lz10_bin.h" +#include "gba_rom_values_ita_lz10_bin.h" +#include "gba_rom_values_jpn_lz10_bin.h" +#include "gba_rom_values_spa_lz10_bin.h" extern rom_data curr_rom; @@ -42,30 +41,31 @@ bool rom_data::load_rom() switch(language) { case LANG_JPN: - compressed_rom_list = gba_rom_values_jpn_zx0_bin; + compressed_rom_list = gba_rom_values_jpn_lz10_bin; break; case LANG_ENG: - compressed_rom_list = gba_rom_values_eng_zx0_bin; + compressed_rom_list = gba_rom_values_eng_lz10_bin; break; case LANG_FRE: - compressed_rom_list = gba_rom_values_fre_zx0_bin; + compressed_rom_list = gba_rom_values_fre_lz10_bin; break; case LANG_GER: - compressed_rom_list = gba_rom_values_ger_zx0_bin; + compressed_rom_list = gba_rom_values_ger_lz10_bin; break; case LANG_ITA: - compressed_rom_list = gba_rom_values_ita_zx0_bin; + compressed_rom_list = gba_rom_values_ita_lz10_bin; break; case LANG_SPA: - compressed_rom_list = gba_rom_values_spa_zx0_bin; + compressed_rom_list = gba_rom_values_spa_lz10_bin; break; default: return false; // Unsupported language } - zx0_decompressor_start(rom_list_buffer, compressed_rom_list); - rom_list_size = zx0_decompressor_get_decompressed_size(); - zx0_decompressor_read(rom_list_size); + // byte 2-4 of the compressed data store the decompressed size + rom_list_size = compressed_rom_list[1] | (compressed_rom_list[2] << 8) | (compressed_rom_list[3] << 16); + LZ77UnCompWram(compressed_rom_list, rom_list_buffer); + cur = rom_list_buffer; while(cur < rom_list_buffer + rom_list_size) diff --git a/source/text_data_table.cpp b/source/text_data_table.cpp index 01e3559..6ed82a7 100644 --- a/source/text_data_table.cpp +++ b/source/text_data_table.cpp @@ -1,6 +1,6 @@ #include "text_data_table.h" -#include "zx0_decompressor.h" #include +#include static uint16_t get_entry_offset_by_index(const uint8_t *text_table, uint16_t index) { @@ -18,7 +18,7 @@ static uint16_t get_num_text_entries(const uint8_t *index_buffer) return *((uint16_t*)index_buffer); } -static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t index) +static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint32_t decompressed_size, uint16_t index) { const uint16_t entry_offset = get_entry_offset_by_index(index_buffer, index); const uint16_t num_text_entries = get_num_text_entries(index_buffer); @@ -33,7 +33,7 @@ static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t in { const uint16_t entry_byte_offset = get_entries_start_offset_of(num_text_entries) + entry_offset; // we don't have a next entry. So we need to consider the end of the file - const uint16_t decompressed_size = static_cast(zx0_decompressor_get_decompressed_size()); + const uint16_t decompressed_size = static_cast(decompressed_size); entry_size_in_bytes = decompressed_size - entry_byte_offset; } return entry_size_in_bytes; @@ -41,13 +41,15 @@ static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t in text_data_table::text_data_table(uint8_t *decompression_buffer) : decompression_buffer_(decompression_buffer) + , decompressed_size_(0) { } void text_data_table::decompress(const uint8_t *compressed_table) { - zx0_decompressor_start(decompression_buffer_, compressed_table); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + // byte 2-4 of the compressed data store the decompressed size + decompressed_size_ = compressed_table[1] | (compressed_table[2] << 8) | (compressed_table[3] << 16); + LZ77UnCompWram(compressed_table, decompression_buffer_); } uint16_t text_data_table::get_number_of_text_entries() const @@ -63,114 +65,5 @@ const uint8_t* text_data_table::get_text_entry(uint16_t index) const uint16_t text_data_table::get_text_entry_size(uint16_t index) const { - return get_entry_size_in_bytes(decompression_buffer_, index); + return get_entry_size_in_bytes(decompression_buffer_, decompressed_size_, index); } - -streamed_text_data_table::streamed_text_data_table(uint8_t *decompression_buffer, uint32_t decompression_buffer_size, uint8_t *index_buffer) - : compressed_table_(nullptr) - , decompression_buffer_(decompression_buffer) - , decompression_buffer_size_(decompression_buffer_size) - , index_buffer_(index_buffer) - , bytes_decompressed_(0) - , last_chunk_size_(0) -{ -} - -void streamed_text_data_table::decompress(const uint8_t *compressed_table) -{ - zx0_decompressor_start(index_buffer_, compressed_table); - zx0_decompressor_read(2); - zx0_decompressor_read(get_number_of_text_entries() * 2); - compressed_table_ = compressed_table; - bytes_decompressed_ = 2 + get_number_of_text_entries() * 2; - - // for further decompressing, we need this data to be available in the decompression buffer too. - // ZX0 looks back to already decompressed data after all. - memcpy(decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE, index_buffer_, bytes_decompressed_); - last_chunk_size_ = bytes_decompressed_; -} - -uint16_t streamed_text_data_table::get_number_of_text_entries() const -{ - return *((uint16_t*)index_buffer_); -} - -const uint8_t* streamed_text_data_table::get_text_entry(uint16_t index) -{ - const uint16_t num_text_entries = get_number_of_text_entries(); - const uint16_t entry_byte_offset = get_entries_start_offset_of(num_text_entries) + get_entry_offset_by_index(index_buffer_, index); - const uint16_t entry_size_in_bytes = get_text_entry_size(index); - const uint16_t space_remaining_outside_lookback_window = decompression_buffer_size_ - ZX0_DEFAULT_WINDOW_SIZE; - const uint16_t window_start_offset = bytes_decompressed_ - get_current_zx0_window_size(); - uint16_t bytes_to_decompress; - uint16_t chunk_size; - uint16_t entry_end_byte_offset; - - // figure out how many bytes we need to read to have the entire text entry - // unfortunately ZX0 doesn't have random access, so we need to linearly decompress - // until we have reached the bytes we actually want. - entry_end_byte_offset = entry_byte_offset + entry_size_in_bytes; - - if(entry_end_byte_offset < bytes_decompressed_) - { - // already decoded, let's check if we have it completely in our current decompressed window - if(entry_byte_offset >= window_start_offset) - { - // one thing to realize is that when we have less than our ZX0 window size, the decoded data doesn't start - // at the start of the buffer. But instead it ends at decompression_buffer + ZX0_DEFAULT_WINDOW_SIZE - return get_window_start() + (entry_byte_offset - window_start_offset); - } - else - { - // unfortunately it's in front of our current decompression window. - // Since ZX0 doesn't actually have random access, it means we have to start - // decompression from scratch - decompress(compressed_table_); - // now that we decompressed JUST the index table again, - // we should be able to reach desired_byte_offset. - } - } - - bytes_to_decompress = entry_end_byte_offset - bytes_decompressed_; - // keep decompressing until we have decompressed what we need. - while(bytes_to_decompress > 0) - { - // move the last decompressed chunk backwards - memmove(decompression_buffer_, decompression_buffer_ + last_chunk_size_, ZX0_DEFAULT_WINDOW_SIZE); - chunk_size = (bytes_to_decompress > space_remaining_outside_lookback_window) ? space_remaining_outside_lookback_window : bytes_to_decompress; - - zx0_decompressor_read_partial(decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE, chunk_size); - last_chunk_size_ = chunk_size; - bytes_to_decompress -= chunk_size; - bytes_decompressed_ += chunk_size; - } - - // we know the last byte we decompressed should be the last byte of the entry - // so we need to count backwards to get to the beginning - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE + last_chunk_size_ - entry_size_in_bytes; -} - -uint16_t streamed_text_data_table::get_text_entry_size(uint16_t index) const -{ - return get_entry_size_in_bytes(index_buffer_, index); -} - -uint8_t* streamed_text_data_table::get_window_start() const -{ - uint16_t without_last_chunk_size = (bytes_decompressed_ - last_chunk_size_); - if(without_last_chunk_size > ZX0_DEFAULT_WINDOW_SIZE) - { - without_last_chunk_size = ZX0_DEFAULT_WINDOW_SIZE; - } - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE - without_last_chunk_size; -} - -uint8_t* streamed_text_data_table::get_window_end() const -{ - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE + last_chunk_size_; -} - -uint16_t streamed_text_data_table::get_current_zx0_window_size() const -{ - return static_cast(get_window_end() - get_window_start()); -} \ No newline at end of file diff --git a/source/text_engine.cpp b/source/text_engine.cpp index 9fb9b3f..d5f06b0 100644 --- a/source/text_engine.cpp +++ b/source/text_engine.cpp @@ -30,11 +30,10 @@ bool text_exit; // attribute noinline was used to make sure the compiler doesn't inline this code back into text_loop() static __attribute__((noinline)) const u8* read_dialogue_text_entry(uint8_t index, u8 *output_buffer) { - u8 text_decompression_buffer[3072]; - u8 index_buffer[100]; + u8 text_decompression_buffer[6144]; const u8 *text_entry; - streamed_text_data_table dialogue_table(text_decompression_buffer, sizeof(text_decompression_buffer), index_buffer); + text_data_table dialogue_table(text_decompression_buffer); dialogue_table.decompress(get_compressed_PTGB_table()); diff --git a/source/zx0_decompressor.cpp b/source/zx0_decompressor.cpp deleted file mode 100644 index 8aac8a3..0000000 --- a/source/zx0_decompressor.cpp +++ /dev/null @@ -1,326 +0,0 @@ -#include "zx0_decompressor.h" -#include -#include - -// The following code is a custom implementation of the ZX0 decompression algorithm invented by Einar Saukas -// Original implementation can be found here: https://github.com/einar-saukas/ZX0 -// The header provides a C facade to access the relevant methods, but the rest of Poke Transporter GB -// doesn't need to be aware of all the datatypes/classes defined here. -/** -* This class makes reading on a per-bit basis much easier. -*/ -class BitReader -{ -public: - BitReader(const uint8_t* buffer); - - IWRAM_CODE uint32_t read(uint32_t numBits); -protected: -private: - const uint8_t* cur_buffer_; - uint32_t cur_dword_; - uint32_t bits_left_; -}; - -enum class ZX0OperationType -{ - NONE, - LITERAL_BLOCK, - COPY_LAST_OFFSET, - COPY_NEW_OFFSET -}; - -typedef struct ZX0Command -{ - ZX0OperationType cmd_type; - uint32_t length; - uint32_t offset; - uint32_t byte_pos; -} ZX0Command; - -/** - * @brief This class implements the actual ZX0 decompression. - */ -class ZX0Decompressor -{ -public: - ZX0Decompressor(); - - /** - * @brief This function prepares the ZX0Decompressor instance - * for decompressing the specified inputData - * into the specified output_buffer - * @param compressedData - */ - void start(uint8_t *output_buffer, const uint8_t *input_data); - - /** - * @brief Retrieves the size of the data when it is fully decompressed - * This is read from the first 2 bytes of the inputData - */ - IWRAM_CODE uint32_t get_decompressed_size() const; - - /** - * @brief This function reads of data into - */ - IWRAM_CODE void read(uint32_t num_bytes); - - /** - * @brief This function swaps out the current output buffer for the given one - */ - IWRAM_CODE void swap_output_buffer(uint8_t *new_output_buffer); -protected: -private: - IWRAM_CODE void read_next_command(); - IWRAM_CODE uint32_t copy_block(uint32_t num_bytes); - - BitReader reader_; - ZX0Command cur_command_; - const uint8_t *input_data_; - uint8_t *back_pos_; - uint8_t *cur_out_; - uint32_t last_offset_; -}; - -IWRAM_CODE static inline uint32_t read_elias_gamma(BitReader& reader) -{ - uint32_t num_non_leading_bits = 0; - uint32_t value; - while (!reader.read(1)) - { - ++num_non_leading_bits; // Count leading zeros - } - // reconstruct the most significant bit of value - value = (1 << num_non_leading_bits) | reader.read(num_non_leading_bits); // Start with MSB - - // Adjust back to zero-based - return value - 1; -} - -IWRAM_CODE static inline void read_new_offset(BitReader& reader, uint32_t& offset) -{ - const uint32_t has_msb = reader.read(1); - - const uint32_t lsb = reader.read(7); - const uint32_t msb = (has_msb) ? read_elias_gamma(reader) : 0; - - offset = ((msb << 7) | lsb) + 1; -} - -BitReader::BitReader(const uint8_t* buffer) - : cur_buffer_(buffer) - , cur_dword_(0) - , bits_left_(0) -{ -} - -IWRAM_CODE inline uint32_t BitReader::read(uint32_t num_bits) -{ - uint32_t result; - - // Fast path: Read all bits from cached data - if (num_bits <= bits_left_) - { - result = (cur_dword_ >> (bits_left_ - num_bits)) & ((1 << num_bits) - 1); - bits_left_ -= num_bits; - return result; - } - - // Slow path: Refill cache and combine bits - result = cur_dword_ & ((1 << bits_left_) - 1); - num_bits -= bits_left_; - - // Refill cache (32-bit aligned read) - // but the GBA (or x86 processor on pc) would read the value as little endian. - // and we need it as big endian. Therefore we do a byte swap - cur_dword_ = __builtin_bswap32(*(uint32_t*)cur_buffer_); - - cur_buffer_ += sizeof(uint32_t); - bits_left_ = 32; - - // Combine remaining bits - result = (result << num_bits) | (cur_dword_ >> (32 - num_bits)); - bits_left_ -= num_bits; - - return result; -} - -ZX0Decompressor::ZX0Decompressor() - : reader_(nullptr) - , cur_command_({ZX0OperationType::NONE, 0, 0, 0}) - , input_data_(nullptr) - , back_pos_(nullptr) - , cur_out_(nullptr) - , last_offset_(UINT32_MAX) -{ -} - -void ZX0Decompressor::start(uint8_t *output_buffer, const uint8_t *input_data) -{ - reader_ = BitReader(input_data + 4); - cur_command_ = {ZX0OperationType::NONE, 0, 0, 0}; - input_data_ = input_data; - back_pos_ = nullptr; - cur_out_ = output_buffer; - last_offset_ = UINT32_MAX; -} - -IWRAM_CODE uint32_t ZX0Decompressor::get_decompressed_size() const -{ - if(!input_data_) - { - return 0; - } - return *((uint32_t*)input_data_); -} - -IWRAM_CODE void ZX0Decompressor::read(uint32_t num_bytes) -{ - while(num_bytes) - { - // Check if we have finished processing the previous pending command - // if we have, we need to read a new operation - if(cur_command_.byte_pos >= cur_command_.length) - { - read_next_command(); - } - - const uint32_t bytes_read = copy_block(num_bytes); - num_bytes -= bytes_read; - } -} - -IWRAM_CODE void ZX0Decompressor::swap_output_buffer(uint8_t *new_output_buffer) -{ - const uint32_t current_offset = cur_out_ - back_pos_; - cur_out_ = new_output_buffer; - back_pos_ = new_output_buffer - current_offset; -} - -IWRAM_CODE inline void ZX0Decompressor::read_next_command() -{ - const uint32_t cmd_bit = reader_.read(1); - - // the "COPY_NEW_OFFSET" command adds + 1 to the length, but the other commands don't. - // given that read_elias_gamma() function is marked "inline", the way I set the length - // is to avoid having multiple calls to it here. (for code size) - if(cmd_bit) - { - read_new_offset(reader_, last_offset_); - cur_command_.cmd_type = ZX0OperationType::COPY_NEW_OFFSET; - cur_command_.length = 1; - cur_command_.offset = last_offset_; - } - else if(cur_command_.cmd_type == ZX0OperationType::LITERAL_BLOCK) - { - cur_command_.cmd_type = ZX0OperationType::COPY_LAST_OFFSET; - // copy from new offset and last offset differs in the sense that with the new offset the encoded length is reduced by one - // and for last offset it isn't. This is likely because you still need to be able to insert a dummy "copy-from-last-offset" operation. - cur_command_.length = 0; - cur_command_.offset = last_offset_; - } - else - { - cur_command_.cmd_type = ZX0OperationType::LITERAL_BLOCK; - cur_command_.length = 0; - } - cur_command_.length += read_elias_gamma(reader_); - cur_command_.byte_pos = 0; -} - -IWRAM_CODE uint32_t ZX0Decompressor::copy_block(uint32_t num_bytes) -{ - const uint32_t available = cur_command_.length - cur_command_.byte_pos; - const uint32_t bytes_to_read = (num_bytes > available) ? available : num_bytes; - uint32_t bytes_remaining = bytes_to_read; - - if(cur_command_.cmd_type == ZX0OperationType::LITERAL_BLOCK) - { - // Literal copy - - // Align cur_out_ first - while (bytes_remaining && ((uintptr_t)cur_out_ & 3)) - { - (*cur_out_++) = reader_.read(8); - bytes_remaining--; - } - - // Use bulk 32-bit writes when aligned - while (bytes_remaining >= 4) - { - // we need to swap again, because the data was originally stored in big endian format - // BitReader converted it to little endian format to make reading easier. - // and now we need to convert it back to big endian format. - *(uint32_t*)cur_out_ = __builtin_bswap32(reader_.read(32)); - cur_out_ += 4; - bytes_remaining -= 4; - } - // Handle remaining bytes - while (bytes_remaining--) - { - (*cur_out_++) = reader_.read(8); - } - } - else - { - if(!cur_command_.byte_pos) - { - back_pos_ = cur_out_ - cur_command_.offset; - } - - // try to get cur_out_ and back_pos aligned to 32 bit accesses first - while (bytes_remaining && (((uintptr_t)cur_out_ & 3) || ((uintptr_t)back_pos_ & 3))) - { - (*cur_out_++) = (*back_pos_++); - bytes_remaining--; - } - - // now try bulk 32 bit writes - while(bytes_remaining >= 4) - { - // these don't need to be byteswapped, because the data is being read with the same endianness as it is being written. - // this is different when reading from BitReader. - *(uint32_t*)cur_out_ = *((uint32_t*)back_pos_); - cur_out_ += 4; - back_pos_ += 4; - bytes_remaining -= 4; - } - - while(bytes_remaining--) - { - (*cur_out_++) = (*back_pos_++); - } - } - - cur_command_.byte_pos += bytes_to_read; - - return bytes_to_read; -} - -// gets stored in .bss, and therefore will end up in IWRAM by default -static ZX0Decompressor decompressor; - -extern "C" -{ -void zx0_decompressor_start(uint8_t *output_buffer, const uint8_t *input_data) -{ - decompressor.start(output_buffer, input_data); -} - -uint32_t zx0_decompressor_get_decompressed_size() -{ - return decompressor.get_decompressed_size(); -} - -void zx0_decompressor_read(uint32_t num_bytes) -{ - decompressor.read(num_bytes); -} - -void zx0_decompressor_read_partial(uint8_t *output_buffer, uint16_t num_bytes) -{ - decompressor.swap_output_buffer(output_buffer); - decompressor.read(num_bytes); -} - -} \ No newline at end of file diff --git a/text_helper/main.py b/text_helper/main.py index 723d0ed..6876350 100755 --- a/text_helper/main.py +++ b/text_helper/main.py @@ -449,15 +449,15 @@ with open (os.curdir + '/include/translated_text.h', 'w') as hFile: # PKMN_NAMES write_enum_to_header_file(hFile, "PKMN_NAMES_", mainDict[lang.name]["PKMN_NAMES"]) - hFile.write("/** Returns the ZX0 compressed PTGB text table.*/\n") + hFile.write("/** Returns the LZ10 compressed PTGB text table.*/\n") hFile.write("const u8* get_compressed_PTGB_table();\n\n") - hFile.write("/** Returns the ZX0 compressed RSEFRLG text table.*/\n") + hFile.write("/** Returns the LZ10 compressed RSEFRLG text table.*/\n") hFile.write("const u8* get_compressed_rsefrlg_table();\n\n") - hFile.write("/** Returns the ZX0 compressed GENERAL text table.*/\n") + hFile.write("/** Returns the LZ10 compressed GENERAL text table.*/\n") hFile.write("const u8* get_compressed_general_table();\n\n") - hFile.write("/** Returns the ZX0 compressed CREDITS text table.*/\n") + hFile.write("/** Returns the LZ10 compressed CREDITS text table.*/\n") hFile.write("const u8* get_compressed_credits_table();\n\n") - hFile.write("/** Returns the ZX0 compressed PKMN_NAMES text table.*/\n") + hFile.write("/** Returns the LZ10 compressed PKMN_NAMES text table.*/\n") hFile.write("const u8* get_compressed_pkmn_names_table();\n\n") hFile.write("\n#endif") @@ -487,39 +487,39 @@ for lang in Languages: # now generate the cpp file. with open(os.curdir + '/source/translated_text.cpp', 'w') as cppFile: - cppFile.write("#include \"translated_text.h\"\n#include \"debug_mode.h\"\n#include \"pokemon_data.h\"\n#include \"zx0_decompressor.h\"\n") + cppFile.write("#include \"translated_text.h\"\n#include \"debug_mode.h\"\n#include \"pokemon_data.h\"\n") # generate includes for each language for lang in Languages: for cat in mainDict[lang.name]: if cat in {"PTGB", "RSEFRLG", "GENERAL", "CREDITS", "PKMN_NAMES"}: - cppFile.write("#include \"" + cat.upper() + "_" + lang.name.lower() + "_zx0_bin.h\"\n") + cppFile.write("#include \"" + cat.upper() + "_" + lang.name.lower() + "_lz10_bin.h\"\n") for lang in Languages: cppFile.write(f"\n#if PTGB_BUILD_LANGUAGE == {lang.value + 1}\n") # PTGB cppFile.write("const u8* get_compressed_PTGB_table()\n") cppFile.write("{\n") - cppFile.write("\treturn PTGB_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn PTGB_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # RSEFRLG cppFile.write("const u8* get_compressed_rsefrlg_table()\n") cppFile.write("{\n") - cppFile.write("\treturn RSEFRLG_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn RSEFRLG_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # GENERAL cppFile.write("const u8* get_compressed_general_table()\n") cppFile.write("{\n") - cppFile.write("\treturn GENERAL_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn GENERAL_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # CREDITS cppFile.write("const u8* get_compressed_credits_table()\n") cppFile.write("{\n") - cppFile.write("\treturn CREDITS_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn CREDITS_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # PKMN_NAMES cppFile.write("const u8* get_compressed_pkmn_names_table()\n") cppFile.write("{\n") - cppFile.write("\treturn PKMN_NAMES_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn PKMN_NAMES_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") cppFile.write(f"#endif\n\n\n") diff --git a/tools/compressZX0/Makefile b/tools/compressZX0/Makefile deleted file mode 100644 index 5582b00..0000000 --- a/tools/compressZX0/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# # Compiler flags -CXXFLAGS := -std=c++11 -fno-rtti -fno-exceptions -fno-unwind-tables -Wall -Wextra -I $(CURDIR) -g - -# Source files directory -SRC_DIR := . -# Build directory -BUILD_DIR := build - -# Source files (add more as needed) -SRCS := $(shell find $(SRC_DIR) -type f -name '*.cpp') -# Object files -OBJS := $(patsubst $(SRC_DIR)/%.cpp,$(BUILD_DIR)/%.o,$(SRCS)) - -# Ensure necessary directories exist -# This function ensures the directory for the target exists -define make_directory - @mkdir -p $(dir $@) -endef - -# Target executable -TARGET := compressZX0 - -# Phony targets -.PHONY: all clean - -# Default target -all: $(TARGET) - -$(TARGET): $(OBJS) - $(CXX) $(CXXFLAGS) $^ -o $@ - -# Rule to compile source files -$(BUILD_DIR)/%.o: $(SRC_DIR)/%.cpp | $(BUILD_DIR) - $(make_directory) - $(CXX) $(CXXFLAGS) -c $< -o $@ - -# Create the build directory if it doesn't exist -$(BUILD_DIR): - mkdir -p $(BUILD_DIR) - -# Clean rule -clean: - rm -rf $(BUILD_DIR) $(TARGET) diff --git a/tools/compressZX0/main.cpp b/tools/compressZX0/main.cpp deleted file mode 100644 index 9bbce2f..0000000 --- a/tools/compressZX0/main.cpp +++ /dev/null @@ -1,436 +0,0 @@ -#include -#include -#include -#include - -// @author risingPhil -// This file implements the zx0 compression algorithm. -// It serves as a test to see if this algorithm could suit us for the compression of several static arrays/buffers in Poke Transporter GB. - -//#define LOG_OPERATIONS 1 - -#define MAX_OFFSET 2048 // Maximum backward offset (ZX0 limit), tuned for Poke Transporter GB to be able to use a 2 KB decompression buffer -#define MAX_LEN 255 // Maximum match length (ZX0 limit) -#define OUTPUT_BUFFER_SIZE 256 * 1024 // Maximum output buffer size (artificial limitation, I suppose) - -#ifdef LOG_OPERATIONS - #define LOG_OP(...) printf(__VA_ARGS__) -#else - #define LOG_OP(...) -#endif - -// anonymous namespace for internal linkage -namespace -{ - -/** -* This class makes writing on a per-bit basis much easier -*/ -class BitWriter -{ -public: - BitWriter(uint8_t* buffer); - - void write(uint8_t value, uint8_t numBits); - void write_bit(uint8_t value); - void write_byte(uint8_t value); - - size_t get_bits_written() const; -protected: -private: - uint8_t* buffer_; - uint8_t* cur_buffer_; - uint8_t bit_index_; -}; - -BitWriter::BitWriter(uint8_t* buffer) - : buffer_(buffer) - , cur_buffer_(buffer) - , bit_index_(0) -{ -} - -void BitWriter::write_bit(uint8_t value) -{ - value &= 0x1; - *cur_buffer_ |= (value << (7 - bit_index_)); - - if(bit_index_ == 7) - { - ++cur_buffer_; - bit_index_ = 0; - } - else - { - ++bit_index_; - } -} - -void BitWriter::write_byte(uint8_t value) -{ - if(bit_index_) - { - write(value, 8); - return; - } - *cur_buffer_ = value; - ++cur_buffer_; -} - -void BitWriter::write(uint8_t value, uint8_t numBits) -{ - for(int i=numBits - 1; i >= 0; --i) - { - write_bit(value >> i); - } -} - -size_t BitWriter::get_bits_written() const -{ - return (cur_buffer_ - buffer_) * 8 + bit_index_; -} -} - -/** - * @brief Find the best match for the current position (LZ77-style) - * We simply try to find the longest matching bytes backwards in the buffer. - */ -static void find_backwards_match(const unsigned char *buffer, size_t buffer_size, int pos, int *best_offset, int *best_len) -{ - *best_offset = 0; - *best_len = 0; - const size_t max_offset = (pos > MAX_OFFSET) ? MAX_OFFSET : pos; - const int max_len = (buffer_size - pos > MAX_LEN) ? MAX_LEN : buffer_size - pos; - int len; - - for (size_t offset = 1; offset <= max_offset; offset++) - { - len = 0; - while (len < max_len && buffer[pos - offset + len] == buffer[pos + len]) - { - ++len; - } - if (len > *best_len) - { - *best_len = len; - *best_offset = offset; - } - } -} - -/** - * @brief This function encodes the specified value with gamma encoding. - * - * The way it works is that we first determine of how many bits the value consists, except for the leading bits. (=num_non_leading_bits) - * Then we write zeros. - * We also write the original value in bits - * - * For decoding, we can determine the number of zeros and that will indicate how many bits we need to read for the actual value. - * - */ -static void write_elias_gamma(BitWriter& writer, int value) -{ - value++; // Adjust because Gamma only encodes n ≥ 1 - int num_non_leading_bits = 0; - int i; - - // Calculate floor(log2(value)) - int tmp = value >> 1; - while(tmp) - { - ++num_non_leading_bits; - tmp >>= 1; - } - - // Write unary part (k zeros) - for (i = 0; i < num_non_leading_bits; i++) - { - writer.write_bit(0); - } - // Write binary part (num_non_leading_bits+1 bits of value) - for (int i = num_non_leading_bits; i >= 0; i--) - { - writer.write_bit(value >> i); - } -} - -/** - * This struct represents a buffer to hold a number of pending "literal" bytes - * before they actually get written to the output - */ -typedef struct LiteralBuffer -{ - uint8_t buffer[1024]; - uint16_t size; -} LiteralBuffer; - -/** - * @brief This function writes a command for the decompressor to start copying bytes - * from the last offset specified with the write_copy_from_new_offset_block() function - */ -static void write_copy_from_last_offset_block(BitWriter& writer, int length) -{ - LOG_OP("copy_last: %d\n", length); - writer.write_bit(0); - write_elias_gamma(writer, length); -} - -/** - * @brief Writes a command to copy the bytes in LiteralBuffer to the decompressed buffer. - */ -static void write_literal_block(BitWriter& writer, LiteralBuffer& literal_buffer) -{ - uint16_t i = 0; - - if(!literal_buffer.size) - { - return; - } - - LOG_OP("copy_literal: %hu\n", literal_buffer.size); - - // flag that this is a literal block - writer.write_bit(0); - write_elias_gamma(writer, literal_buffer.size); - - while(i < literal_buffer.size) - { - writer.write_byte(literal_buffer.buffer[i]); - - ++i; - } - literal_buffer.size = 0; -} - -/** - * @brief Writes a command to indicate that the decompressor must copy bytes from the given backwards offset. - */ -static void write_copy_from_new_offset_block(BitWriter& writer, int offset, int length) -{ - LOG_OP("copy_new: offset: %d, length: %d\n", offset, length); - writer.write_bit(1); // Match flag - - // Encode offset (Elias Gamma + 7-bit LSB) - const int msb = ((offset - 1) >> 7) & 0xFF; - const int lsb = (offset - 1) & 0x7F; - - // first bit of LSB indicates whether the MSB follows. - writer.write_bit((msb > 0)); - - // write 7 bit LSB raw bits - writer.write(lsb, 7); - - if (msb > 0) - { - write_elias_gamma(writer, msb); - } - - // Encode length (Elias Gamma) - write_elias_gamma(writer, length - 1); -} - -static void literal_buffer_push(BitWriter& writer, LiteralBuffer& literal_buffer, uint8_t byte) -{ - if(literal_buffer.size == 1024) - { - // EDGE case: buffer is full. - // back-to-back literal blocks are forbidden, - // so we must insert a dummy "use last offset" block - write_literal_block(writer, literal_buffer); - write_copy_from_last_offset_block(writer, 0); - } - - literal_buffer.buffer[literal_buffer.size] = byte; - ++literal_buffer.size; -} - -/** - * This function encodes the specified buffer with the ZX0 compression algorithm - * and stores the result into output_buffer. - * - * Please make sure the output_buffer is sufficiently large enough before calling this function. - */ -static size_t encodeZX0(uint8_t* output_buffer, const uint8_t* buffer, size_t buffer_size) -{ - BitWriter writer(output_buffer); - LiteralBuffer literal_buffer = { - .buffer = {0}, - .size = 0 - }; - - int pos = 0; - int last_offset = 0x7FFFFFFF; - int offset; - int length; - int numBytes = buffer_size; - - // first write the size of the input in little endian format in the output buffer - writer.write_byte(static_cast(buffer_size)); - writer.write_byte(static_cast(buffer_size >> 8)); - writer.write_byte(static_cast(buffer_size >> 16)); - writer.write_byte(static_cast(buffer_size >> 24)); - - while(pos < numBytes) - { - find_backwards_match(buffer, numBytes, pos, &offset, &length); - - // important rules: You cannot have 2 consecutive literal blocks. - // reusing the last offset can only happen after a literal block! - - if(length < 2) - { - // we must buffer the literals because we can only start writing them when we know the "length" - literal_buffer_push(writer, literal_buffer, buffer[pos]); - ++pos; - } - else if(offset == last_offset) - { - // write any pending literal bytes - write_literal_block(writer, literal_buffer); - write_copy_from_last_offset_block(writer, length); - pos += length; - } - else - { - // write any pending literal bytes - write_literal_block(writer, literal_buffer); - write_copy_from_new_offset_block(writer, offset, length); - last_offset = offset; - pos += length; - } - } - - return writer.get_bits_written(); -} - -/** - * @brief Reads the given file completely into the specified buffer. - * The buffer is allocated by this function, but should be delete[]'d by the caller. - */ -static bool read_file(const char* filename, uint8_t*& out_buffer, size_t& out_size) -{ - FILE* file; - long size; - size_t read; - uint8_t* buffer; - - file = fopen(filename, "rb"); - if (!file) return false; - - // Seek to end to determine size - if (fseek(file, 0, SEEK_END) != 0) - { - fclose(file); - return false; - } - - size = ftell(file); - if (size < 0) - { - fclose(file); - return false; - } - - rewind(file); - - buffer = new uint8_t[size]; - - read = fread(buffer, 1, size, file); - fclose(file); - - if (read != (size_t)size) { - delete[] buffer; - return false; - } - - out_buffer = buffer; - out_size = size; - - return true; -} - -static void print_usage() -{ - const char* usageString = R"delim( -Usage: compressZX0 - -This program will compress the given file with the ZX0 compression algorithm and store the output in -/_zx0.bin -)delim"; - printf(usageString); -} - -int main(int argc, char** argv) -{ - // Reserve 256KB buffer, which is already much larger than the maximum file size we'd allow for PTGB. - // (the reason why I'm using a buffer instead of writing directly to a file is simply because I'm lazy. - // I wrote a test of the algorithm using buffers first. And I know that for Poke Transporter GB specifically - // we'll never exceed the 256KB filesize. So I'm not going to rework this code, because there's currently no need) - uint8_t output_buffer[OUTPUT_BUFFER_SIZE] = {0}; - uint8_t *input_buffer = nullptr; - char *filename; - char *extension_dot; - size_t input_buffer_size; - size_t bits_written; - size_t num_bytes; - double compress_ratio; - char output_path[4096]; - FILE* f; - - if(argc < 3) - { - print_usage(); - return 1; - } - - if(!read_file(argv[1], input_buffer, input_buffer_size)) - { - perror("Could not open file: "); - return 1; - } - - // make sure the input_buffer_size is not larger than our output_buffer we statically allocated - // This is a bit of an artificial limitation though. - if(input_buffer_size > sizeof(output_buffer)) - { - fprintf(stderr, "ERROR: The input file should not be larger than %zu KB!\n", sizeof(output_buffer)); - return 1; - } - - // get the filename part of the given file - // and remove the extension. - // basename uses statically allocated memory that gets overwritten by each call. - // but it returns a modifiable char* - // so we might as well just edit that buffer directly because no-one will depend on this value later. - filename = basename(argv[1]); - - printf("Compressing %s...", filename); - - bits_written = encodeZX0(output_buffer, input_buffer, input_buffer_size); - delete[] input_buffer; - input_buffer = nullptr; - num_bytes = (bits_written + 7) / 8; - - printf("done\n"); - - // if we have an extension in the filename, just end the string at the '.' position. - extension_dot = strchr(filename, '.'); - if(extension_dot) - { - *extension_dot = '\0'; - } - - // argv[2] should be the output directory - snprintf(output_path, sizeof(output_path), "%s/%s_zx0.bin", argv[2], filename); - - f = fopen(output_path, "wb+"); - fwrite(output_buffer, 1, num_bytes, f); - fclose(f); - - compress_ratio = static_cast(num_bytes) / input_buffer_size; - printf("Compressed size: %zu bytes/%zu bytes, Compression ratio: %f%%\n", num_bytes, input_buffer_size, compress_ratio * 100.f); - - return 0; -} - - diff --git a/tools/data-generator/src/main.cpp b/tools/data-generator/src/main.cpp index da5cfed..44890cc 100644 --- a/tools/data-generator/src/main.cpp +++ b/tools/data-generator/src/main.cpp @@ -11,7 +11,7 @@ #include // This application holds the various long static data arrays that Poke Transporter GB uses -// and it writes them to .bin files that can be compressed with compressZX0 later. +// and it writes them to .bin files that can be compressed with gbalzss later. // it's useful to do it this way because it keeps this data easy to view, edit and document // This function generates a binary file containing the specified list of ROM_DATA structs