diff --git a/Makefile b/Makefile index bf04eec..62c391c 100644 --- a/Makefile +++ b/Makefile @@ -146,11 +146,10 @@ all: $(BUILD) generate_data: mkdir -p data mkdir -p to_compress - @env -i "PATH=$(PATH)" $(MAKE) -C tools/compressZX0 @env -i "PATH=$(PATH)" $(MAKE) -C tools/data-generator @tools/data-generator/data-generator to_compress @python3 text_helper/main.py - @find to_compress -name "*.bin" | xargs -i tools/compressZX0/compressZX0 {} data/ + @find to_compress -name "*.bin" -print0 | xargs -0 -n1 ./compress_lz10.sh #--------------------------------------------------------------------------------- $(BUILD): generate_data @@ -163,7 +162,6 @@ $(BUILD): generate_data #--------------------------------------------------------------------------------- clean: @echo clean ... - @$(MAKE) -C tools/compressZX0 clean @$(MAKE) -C tools/data-generator clean @$(MAKE) -C loader clean @rm -fr $(BUILD) $(TARGET).elf $(TARGET).gba data/ to_compress/ @@ -176,9 +174,6 @@ BINFILES := $(foreach dir,../$(DATA),$(notdir $(wildcard $(dir)/*.*))) export OFILES_BIN := $(addsuffix .o,$(BINFILES)) OFILES := $(OFILES_BIN) $(OFILES) -# Optimize zx0_decompressor for speed -zx0_decompressor.o: CXXFLAGS += -O2 - #--------------------------------------------------------------------------------- # main targets #--------------------------------------------------------------------------------- diff --git a/compress_lz10.sh b/compress_lz10.sh new file mode 100755 index 0000000..223ce80 --- /dev/null +++ b/compress_lz10.sh @@ -0,0 +1,4 @@ +#!/bin/sh +infile="$1" +outfile="data/$(basename "$infile" .bin)_lz10.bin" +gbalzss e "$infile" "$outfile" diff --git a/include/text_data_table.h b/include/text_data_table.h index 92fd6ee..3b942e3 100644 --- a/include/text_data_table.h +++ b/include/text_data_table.h @@ -8,10 +8,6 @@ * and then gives you utility functions to retrieve the text entries * * But it requires a buffer large enough to contain the entire decompressed table. - * It will also completely decompress the table, which may not what you want. - * - * If you want to use a text table in a streamed manner (use smaller decompression buffer and only decompress what's needed) - * consider using streamed_text_data_table instead. */ class text_data_table { @@ -40,54 +36,7 @@ public: uint16_t get_text_entry_size(uint16_t index) const; private: uint8_t *decompression_buffer_; -}; - -/** - * This class is an alternative to translated_text table. - * It provides the same functionality, yet in a streamed manner. - * - * This allows you to use a decompression_buffer that is smaller than the fully decoded text table and only decompress until - * you have what you need. - * - * To make sure we have access to the table index at all times, you need to specify a buffer to hold the table index as well. - * - * REQUIREMENT: decompression_buffer needs to be larger than the zx0 window size!! - */ -class streamed_text_data_table -{ -public: - streamed_text_data_table(uint8_t *decompression_buffer, uint32_t decompression_buffer_size, uint8_t *index_buffer); - - /** - * This function sets up the zx0 decompressor and decompresses the index into the index_buffer - */ - void decompress(const uint8_t *compressed_table); - - /** - * Returns the number of text entries in the decompression_buffer_ - */ - uint16_t get_number_of_text_entries() const; - - /** - * This function returns a pointer to a text entry in the decompression_buffer - */ - const uint8_t* get_text_entry(uint16_t index); - - /** - * This function returns the text entry size in bytes at the given index - */ - uint16_t get_text_entry_size(uint16_t index) const; -private: - uint8_t* get_window_start() const; - uint8_t* get_window_end() const; - uint16_t get_current_zx0_window_size() const; - - const uint8_t *compressed_table_; - uint8_t *decompression_buffer_; - uint32_t decompression_buffer_size_; - uint8_t *index_buffer_; - mutable uint16_t bytes_decompressed_; - uint16_t last_chunk_size_; + uint32_t decompressed_size_; }; #endif \ No newline at end of file diff --git a/include/zx0_decompressor.h b/include/zx0_decompressor.h deleted file mode 100644 index 7a812b1..0000000 --- a/include/zx0_decompressor.h +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef _ZX0_DECODE_H -#define _ZX0_DECODE_H - -#include - -#define ZX0_DEFAULT_WINDOW_SIZE 2048 - -// The ZX0 decompressor offers functionality to decompress data -// compressed with the ZX0 algorithm (see tools/compressZX0) -// This algorithm was invented by Einar Saukas -// Original implementation can be found here: https://github.com/einar-saukas/ZX0 -// However, we've implemented a custom variant of this algorithm. -// (for instance: we're storing the uncompressed size in the first 2 bytes in little endian) -extern "C" -{ - /** - * @brief This function slots the specified input_data buffer into the zx0 decompressor. - * Calling this function effectively resets the ZX0 decompressors' internal state. - */ - void zx0_decompressor_start(uint8_t *output_buffer, const uint8_t *input_data); - - /** - * @brief This function returns the uncompressed size of the current input_data buffer. - * It reads this from the first 2 bytes of input_data - */ - uint32_t zx0_decompressor_get_decompressed_size(); - - /** - * @brief This function copies of decompressed data into the specified - * It will append to the existing output_buffer you set earlier with zx0_decompressor_start() - */ - void zx0_decompressor_read(uint32_t num_bytes); - - /** - * @brief this function does a partial decompress into output_buffer. - * HOWEVER: zx0 decompression requires you to use previously decompressed bytes to decompress the current ones. - * In order to accomplish this, output_buffer MUST NOT point to the start of the buffer!! - * Instead it should refer to a point within the buffer with previously decoded bytes available before it, - * with up to the bytes available before that point. - * (So if you decoded >= window size, bytes should be available before output_buffer. - * If not, before output_buffer should have ) - * - * This function is intended as a way to read data in a "streamed" way into a smaller buffer that is smaller than the actual decompressed file size. - * NOTE: when used in a loop, you should manually move the decompressed data from the previous iteration to the front of the buffer (output_buffer + window_size - num_bytes) - */ - void zx0_decompressor_read_partial(uint8_t *output_buffer, uint16_t num_bytes); -} - -#endif \ No newline at end of file diff --git a/source/box_menu.cpp b/source/box_menu.cpp index 78d9776..4a91fd9 100644 --- a/source/box_menu.cpp +++ b/source/box_menu.cpp @@ -10,7 +10,6 @@ #include "text_engine.h" #include "translated_text.h" #include "text_data_table.h" -#include "zx0_decompressor.h" Box_Menu::Box_Menu() {}; diff --git a/source/main.cpp b/source/main.cpp index e144658..c549807 100644 --- a/source/main.cpp +++ b/source/main.cpp @@ -213,6 +213,59 @@ void first_load_message(void) tte_erase_rect(0, 0, H_MAX, V_MAX); } +#include "translated_text.h" + +#define TIMER_ENABLE 0x80 +#define TIMER_CASCADE 0x4 +#define TIMER_FREQ_1 0x0 // 16.78 MHz +#define TIMER_FREQ_64 0x1 // 262,144 Hz +#define TIMER_FREQ_256 0x2 // 65,536 Hz +#define TIMER_FREQ_1024 0x3 // 16,384 Hz + +int test_decompress() +{ + uint16_t charset[256]; + + // Reset both timers + REG_TM0CNT = 0; + REG_TM1CNT = 0; + REG_TM0D = 0; + REG_TM1D = 0; + + // Set up TIMER0: count with no prescaler + REG_TM0CNT = TIMER_ENABLE | TIMER_FREQ_1; + // Set up TIMER1: cascade mode (increment when TIMER0 overflows) + REG_TM1CNT = TIMER_ENABLE | TIMER_CASCADE; + + load_localized_charset(charset, 3, ENG_ID); + + // Read combined 32-bit timer value + const u32 ticks = ((u32)REG_TM1D << 16) | REG_TM0D; + + // Stop timers + REG_TM0CNT = 0; + REG_TM1CNT = 0; + + create_textbox(4, 1, 160, 80, true); + + ptgb_write_debug(charset, "Test results:\n\nDecompress: ", true); + ptgb_write_debug(charset, ptgb::to_string(ticks * 1000 / 16777), true); + ptgb_write_debug(charset, " usec\n", true); + + + while (true) + { + if (key_hit(KEY_B)) + { + hide_text_box(); + reset_textbox(); + return 0; + } + global_next_frame(); + } + return 0; +} + int credits() { u8 text_decompression_buffer[2048]; @@ -249,6 +302,11 @@ int credits() curr_credits_num++; update = true; } + if(key_hit(KEY_SELECT)) + { + return test_decompress(); + } + #if 0 if (ENABLE_DEBUG_SCREEN && key_hit(KEY_SELECT)) { char hexBuffer[16]; @@ -325,6 +383,7 @@ int credits() global_next_frame(); } } +#endif global_next_frame(); } diff --git a/source/pokedex.cpp b/source/pokedex.cpp index 88738af..7aa2c6a 100644 --- a/source/pokedex.cpp +++ b/source/pokedex.cpp @@ -10,9 +10,8 @@ #include "button_handler.h" #include "translated_text.h" #include "text_engine.h" -#include "zx0_decompressor.h" #include "text_data_table.h" -#include "TYPES_zx0_bin.h" +#include "TYPES_lz10_bin.h" Dex dex_array[DEX_MAX]; int dex_shift = 0; @@ -89,7 +88,7 @@ void pokedex_init() obj_hide(down_arrow); } -#include "gen_3_charsets_zx0_bin.h" +#include "gen_3_charsets_lz10_bin.h" #include "libstd_replacements.h" int pokedex_loop() @@ -100,11 +99,8 @@ int pokedex_loop() u8 decompression_buffer[3072]; u16 charset[256]; - zx0_decompressor_start((u8*)TYPES, TYPES_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); - - zx0_decompressor_start((u8*)charset, gen_3_charsets_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(TYPES_lz10_bin, (u8*)TYPES); + LZ77UnCompWram(gen_3_charsets_lz10_bin, (u8*)charset); load_general_table_text_entries(decompression_buffer, kanto_name, johto_name); diff --git a/source/pokemon.cpp b/source/pokemon.cpp index 86e7c2b..31ae81e 100644 --- a/source/pokemon.cpp +++ b/source/pokemon.cpp @@ -6,8 +6,7 @@ #include "save_data_manager.h" #include "debug_mode.h" #include "text_engine.h" -#include "zx0_decompressor.h" -#include "JPN_NAMES_zx0_bin.h" +#include "JPN_NAMES_lz10_bin.h" Pokemon::Pokemon() {}; @@ -215,9 +214,8 @@ void Pokemon::convert_to_gen_three(PokemonTables& data_tables, Conversion_Types u16 cur_char; data_tables.load_gen3_charset(language); - // setup the zx0 decompressor to decompress the JPN_NAMES table - zx0_decompressor_start((u8*)JPN_NAMES, JPN_NAMES_zx0_bin); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + + LZ77UnCompWram(JPN_NAMES_lz10_bin, (u8*)JPN_NAMES); for (int i = 0; i < 6; i++) { // Read the JPN name and convert it diff --git a/source/pokemon_data.cpp b/source/pokemon_data.cpp index c082f0a..994f2bd 100644 --- a/source/pokemon_data.cpp +++ b/source/pokemon_data.cpp @@ -1,15 +1,14 @@ #include "pokemon_data.h" -#include "zx0_decompressor.h" -#include "EXP_GROUPS_zx0_bin.h" -#include "GENDER_RATIO_zx0_bin.h" -#include "NUM_ABILITIES_zx0_bin.h" -#include "FIRST_MOVES_zx0_bin.h" -#include "POWER_POINTS_zx0_bin.h" -#include "EVENT_PKMN_zx0_bin.h" -#include "TYPES_zx0_bin.h" -#include "gen_1_charsets_zx0_bin.h" -#include "gen_2_charsets_zx0_bin.h" -#include "gen_3_charsets_zx0_bin.h" +#include "EXP_GROUPS_lz10_bin.h" +#include "GENDER_RATIO_lz10_bin.h" +#include "NUM_ABILITIES_lz10_bin.h" +#include "FIRST_MOVES_lz10_bin.h" +#include "POWER_POINTS_lz10_bin.h" +#include "EVENT_PKMN_lz10_bin.h" +#include "TYPES_lz10_bin.h" +#include "gen_1_charsets_lz10_bin.h" +#include "gen_2_charsets_lz10_bin.h" +#include "gen_3_charsets_lz10_bin.h" #include #include @@ -744,8 +743,7 @@ static void load_table(u8 *table, const u8* source, bool &loadedBool) { return; } - zx0_decompressor_start(table, source); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(source, table); loadedBool = true; } @@ -764,37 +762,37 @@ PokemonTables::PokemonTables() void PokemonTables::load_exp_groups() { - load_table(EXP_GROUPS, EXP_GROUPS_zx0_bin, exp_groups_loaded); + load_table(EXP_GROUPS, EXP_GROUPS_lz10_bin, exp_groups_loaded); } void PokemonTables::load_gender_ratios() { - load_table(GENDER_RATIO, GENDER_RATIO_zx0_bin, gender_ratios_loaded); + load_table(GENDER_RATIO, GENDER_RATIO_lz10_bin, gender_ratios_loaded); } void PokemonTables::load_num_abilities() { - load_table((uint8_t*)NUM_ABILITIES, NUM_ABILITIES_zx0_bin, num_abilities_loaded); + load_table((uint8_t*)NUM_ABILITIES, NUM_ABILITIES_lz10_bin, num_abilities_loaded); } void PokemonTables::load_first_moves() { - load_table(FIRST_MOVES, FIRST_MOVES_zx0_bin, first_moves_loaded); + load_table(FIRST_MOVES, FIRST_MOVES_lz10_bin, first_moves_loaded); } void PokemonTables::load_power_points() { - load_table(POWER_POINTS, POWER_POINTS_zx0_bin, power_points_loaded); + load_table(POWER_POINTS, POWER_POINTS_lz10_bin, power_points_loaded); } void PokemonTables::load_event_pkmn() { - load_table((uint8_t*)EVENT_PKMN, EVENT_PKMN_zx0_bin, event_pkmn_loaded); + load_table((uint8_t*)EVENT_PKMN, EVENT_PKMN_lz10_bin, event_pkmn_loaded); } void PokemonTables::load_types() { - load_table((uint8_t*)TYPES, TYPES_zx0_bin, types_loaded); + load_table((uint8_t*)TYPES, TYPES_lz10_bin, types_loaded); } void PokemonTables::load_input_charset(byte gen, byte lang) @@ -909,21 +907,20 @@ void load_localized_charset(u16 *output_char_array, byte gen, byte lang) switch(gen) { case 1: - input_data = gen_1_charsets_zx0_bin; + input_data = gen_1_charsets_lz10_bin; break; case 2: - input_data = gen_2_charsets_zx0_bin; + input_data = gen_2_charsets_lz10_bin; break; case 3: - input_data = gen_3_charsets_zx0_bin; + input_data = gen_3_charsets_lz10_bin; break; default: // Invalid generation, return without doing anything return; } - zx0_decompressor_start(generation_charsets, input_data); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + LZ77UnCompWram(input_data, generation_charsets); memcpy(output_char_array, generation_charsets + (lang_index * 256 * sizeof(u16)), 256 * sizeof(u16)); // copy the charset into the output array } diff --git a/source/pokemon_party.cpp b/source/pokemon_party.cpp index ea80b37..e9fc8a0 100644 --- a/source/pokemon_party.cpp +++ b/source/pokemon_party.cpp @@ -7,13 +7,12 @@ #include "gb_rom_values/gb_rom_values.h" #include "sprite_data.h" #include "box_menu.h" -#include "zx0_decompressor.h" #include "payload_file_reader.h" -#include "gb_rom_values_eng_zx0_bin.h" -#include "gb_rom_values_fre_zx0_bin.h" -#include "gb_gen1_payloads_RB_zx0_bin.h" -#include "gb_gen1_payloads_Y_zx0_bin.h" -#include "gb_gen2_payloads_zx0_bin.h" +#include "gb_rom_values_eng_lz10_bin.h" +#include "gb_rom_values_fre_lz10_bin.h" +#include "gb_gen1_payloads_RB_lz10_bin.h" +#include "gb_gen1_payloads_Y_lz10_bin.h" +#include "gb_gen2_payloads_lz10_bin.h" static const byte gen1_rb_debug_box_data[0x462] = { // Num of Pokemon @@ -273,19 +272,19 @@ bool Pokemon_Party::load_gb_rom() switch(lang) { case ENG_ID: - compressed_rom_table = gb_rom_values_eng_zx0_bin; + compressed_rom_table = gb_rom_values_eng_lz10_bin; break; case FRE_ID: - compressed_rom_table = gb_rom_values_fre_zx0_bin; + compressed_rom_table = gb_rom_values_fre_lz10_bin; break; default: // no rom table for this language return false; } - zx0_decompressor_start(gb_rom_table_buffer, compressed_rom_table); - rom_table_size = zx0_decompressor_get_decompressed_size(); - zx0_decompressor_read(rom_table_size); + // byte 2-4 of the compressed data store the decompressed size + rom_table_size = compressed_rom_table[1] | (compressed_rom_table[2] << 8) | (compressed_rom_table[3] << 16); + LZ77UnCompWram(compressed_rom_table, gb_rom_table_buffer); cur = gb_rom_table_buffer; while(cur < gb_rom_table_buffer + rom_table_size) @@ -347,6 +346,7 @@ void Pokemon_Party::init_payload() { u8 decompression_buffer[1512]; const u8 *payload_src; + u32 payload_file_size; //WARNING: Ensure sure decompression_buffer is large enough! @@ -354,21 +354,22 @@ void Pokemon_Party::init_payload() { if(curr_gb_rom.version == YELLOW_ID) { - payload_src = gb_gen1_payloads_Y_zx0_bin; + payload_src = gb_gen1_payloads_Y_lz10_bin; } else { - payload_src = gb_gen1_payloads_RB_zx0_bin; + payload_src = gb_gen1_payloads_RB_lz10_bin; } } else // if(curr_gb_rom.generation == 2) { - payload_src = gb_gen2_payloads_zx0_bin; + payload_src = gb_gen2_payloads_lz10_bin; } - zx0_decompressor_start(decompression_buffer, payload_src); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + // byte 2-4 of the compressed data store the decompressed size + payload_file_size = payload_src[1] | (payload_src[2] << 8) | (payload_src[3] << 16); + LZ77UnCompWram(payload_src, decompression_buffer); - payload_file_reader payload_reader(decompression_buffer, zx0_decompressor_get_decompressed_size()); + payload_file_reader payload_reader(decompression_buffer, payload_file_size); payload_reader.read_payload(current_payload, curr_gb_rom.language, curr_gb_rom.version); } \ No newline at end of file diff --git a/source/rom_data.cpp b/source/rom_data.cpp index d39eb11..a4b7e67 100644 --- a/source/rom_data.cpp +++ b/source/rom_data.cpp @@ -5,13 +5,12 @@ #include "text_engine.h" #include "gba_rom_values/gba_rom_values.h" #include "libraries/nanoprintf/nanoprintf.h" -#include "zx0_decompressor.h" -#include "gba_rom_values_eng_zx0_bin.h" -#include "gba_rom_values_fre_zx0_bin.h" -#include "gba_rom_values_ger_zx0_bin.h" -#include "gba_rom_values_ita_zx0_bin.h" -#include "gba_rom_values_jpn_zx0_bin.h" -#include "gba_rom_values_spa_zx0_bin.h" +#include "gba_rom_values_eng_lz10_bin.h" +#include "gba_rom_values_fre_lz10_bin.h" +#include "gba_rom_values_ger_lz10_bin.h" +#include "gba_rom_values_ita_lz10_bin.h" +#include "gba_rom_values_jpn_lz10_bin.h" +#include "gba_rom_values_spa_lz10_bin.h" extern rom_data curr_rom; @@ -42,30 +41,31 @@ bool rom_data::load_rom() switch(language) { case LANG_JPN: - compressed_rom_list = gba_rom_values_jpn_zx0_bin; + compressed_rom_list = gba_rom_values_jpn_lz10_bin; break; case LANG_ENG: - compressed_rom_list = gba_rom_values_eng_zx0_bin; + compressed_rom_list = gba_rom_values_eng_lz10_bin; break; case LANG_FRE: - compressed_rom_list = gba_rom_values_fre_zx0_bin; + compressed_rom_list = gba_rom_values_fre_lz10_bin; break; case LANG_GER: - compressed_rom_list = gba_rom_values_ger_zx0_bin; + compressed_rom_list = gba_rom_values_ger_lz10_bin; break; case LANG_ITA: - compressed_rom_list = gba_rom_values_ita_zx0_bin; + compressed_rom_list = gba_rom_values_ita_lz10_bin; break; case LANG_SPA: - compressed_rom_list = gba_rom_values_spa_zx0_bin; + compressed_rom_list = gba_rom_values_spa_lz10_bin; break; default: return false; // Unsupported language } - zx0_decompressor_start(rom_list_buffer, compressed_rom_list); - rom_list_size = zx0_decompressor_get_decompressed_size(); - zx0_decompressor_read(rom_list_size); + // byte 2-4 of the compressed data store the decompressed size + rom_list_size = compressed_rom_list[1] | (compressed_rom_list[2] << 8) | (compressed_rom_list[3] << 16); + LZ77UnCompWram(compressed_rom_list, rom_list_buffer); + cur = rom_list_buffer; while(cur < rom_list_buffer + rom_list_size) diff --git a/source/text_data_table.cpp b/source/text_data_table.cpp index 01e3559..6ed82a7 100644 --- a/source/text_data_table.cpp +++ b/source/text_data_table.cpp @@ -1,6 +1,6 @@ #include "text_data_table.h" -#include "zx0_decompressor.h" #include +#include static uint16_t get_entry_offset_by_index(const uint8_t *text_table, uint16_t index) { @@ -18,7 +18,7 @@ static uint16_t get_num_text_entries(const uint8_t *index_buffer) return *((uint16_t*)index_buffer); } -static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t index) +static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint32_t decompressed_size, uint16_t index) { const uint16_t entry_offset = get_entry_offset_by_index(index_buffer, index); const uint16_t num_text_entries = get_num_text_entries(index_buffer); @@ -33,7 +33,7 @@ static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t in { const uint16_t entry_byte_offset = get_entries_start_offset_of(num_text_entries) + entry_offset; // we don't have a next entry. So we need to consider the end of the file - const uint16_t decompressed_size = static_cast(zx0_decompressor_get_decompressed_size()); + const uint16_t decompressed_size = static_cast(decompressed_size); entry_size_in_bytes = decompressed_size - entry_byte_offset; } return entry_size_in_bytes; @@ -41,13 +41,15 @@ static uint16_t get_entry_size_in_bytes(const uint8_t *index_buffer, uint16_t in text_data_table::text_data_table(uint8_t *decompression_buffer) : decompression_buffer_(decompression_buffer) + , decompressed_size_(0) { } void text_data_table::decompress(const uint8_t *compressed_table) { - zx0_decompressor_start(decompression_buffer_, compressed_table); - zx0_decompressor_read(zx0_decompressor_get_decompressed_size()); + // byte 2-4 of the compressed data store the decompressed size + decompressed_size_ = compressed_table[1] | (compressed_table[2] << 8) | (compressed_table[3] << 16); + LZ77UnCompWram(compressed_table, decompression_buffer_); } uint16_t text_data_table::get_number_of_text_entries() const @@ -63,114 +65,5 @@ const uint8_t* text_data_table::get_text_entry(uint16_t index) const uint16_t text_data_table::get_text_entry_size(uint16_t index) const { - return get_entry_size_in_bytes(decompression_buffer_, index); + return get_entry_size_in_bytes(decompression_buffer_, decompressed_size_, index); } - -streamed_text_data_table::streamed_text_data_table(uint8_t *decompression_buffer, uint32_t decompression_buffer_size, uint8_t *index_buffer) - : compressed_table_(nullptr) - , decompression_buffer_(decompression_buffer) - , decompression_buffer_size_(decompression_buffer_size) - , index_buffer_(index_buffer) - , bytes_decompressed_(0) - , last_chunk_size_(0) -{ -} - -void streamed_text_data_table::decompress(const uint8_t *compressed_table) -{ - zx0_decompressor_start(index_buffer_, compressed_table); - zx0_decompressor_read(2); - zx0_decompressor_read(get_number_of_text_entries() * 2); - compressed_table_ = compressed_table; - bytes_decompressed_ = 2 + get_number_of_text_entries() * 2; - - // for further decompressing, we need this data to be available in the decompression buffer too. - // ZX0 looks back to already decompressed data after all. - memcpy(decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE, index_buffer_, bytes_decompressed_); - last_chunk_size_ = bytes_decompressed_; -} - -uint16_t streamed_text_data_table::get_number_of_text_entries() const -{ - return *((uint16_t*)index_buffer_); -} - -const uint8_t* streamed_text_data_table::get_text_entry(uint16_t index) -{ - const uint16_t num_text_entries = get_number_of_text_entries(); - const uint16_t entry_byte_offset = get_entries_start_offset_of(num_text_entries) + get_entry_offset_by_index(index_buffer_, index); - const uint16_t entry_size_in_bytes = get_text_entry_size(index); - const uint16_t space_remaining_outside_lookback_window = decompression_buffer_size_ - ZX0_DEFAULT_WINDOW_SIZE; - const uint16_t window_start_offset = bytes_decompressed_ - get_current_zx0_window_size(); - uint16_t bytes_to_decompress; - uint16_t chunk_size; - uint16_t entry_end_byte_offset; - - // figure out how many bytes we need to read to have the entire text entry - // unfortunately ZX0 doesn't have random access, so we need to linearly decompress - // until we have reached the bytes we actually want. - entry_end_byte_offset = entry_byte_offset + entry_size_in_bytes; - - if(entry_end_byte_offset < bytes_decompressed_) - { - // already decoded, let's check if we have it completely in our current decompressed window - if(entry_byte_offset >= window_start_offset) - { - // one thing to realize is that when we have less than our ZX0 window size, the decoded data doesn't start - // at the start of the buffer. But instead it ends at decompression_buffer + ZX0_DEFAULT_WINDOW_SIZE - return get_window_start() + (entry_byte_offset - window_start_offset); - } - else - { - // unfortunately it's in front of our current decompression window. - // Since ZX0 doesn't actually have random access, it means we have to start - // decompression from scratch - decompress(compressed_table_); - // now that we decompressed JUST the index table again, - // we should be able to reach desired_byte_offset. - } - } - - bytes_to_decompress = entry_end_byte_offset - bytes_decompressed_; - // keep decompressing until we have decompressed what we need. - while(bytes_to_decompress > 0) - { - // move the last decompressed chunk backwards - memmove(decompression_buffer_, decompression_buffer_ + last_chunk_size_, ZX0_DEFAULT_WINDOW_SIZE); - chunk_size = (bytes_to_decompress > space_remaining_outside_lookback_window) ? space_remaining_outside_lookback_window : bytes_to_decompress; - - zx0_decompressor_read_partial(decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE, chunk_size); - last_chunk_size_ = chunk_size; - bytes_to_decompress -= chunk_size; - bytes_decompressed_ += chunk_size; - } - - // we know the last byte we decompressed should be the last byte of the entry - // so we need to count backwards to get to the beginning - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE + last_chunk_size_ - entry_size_in_bytes; -} - -uint16_t streamed_text_data_table::get_text_entry_size(uint16_t index) const -{ - return get_entry_size_in_bytes(index_buffer_, index); -} - -uint8_t* streamed_text_data_table::get_window_start() const -{ - uint16_t without_last_chunk_size = (bytes_decompressed_ - last_chunk_size_); - if(without_last_chunk_size > ZX0_DEFAULT_WINDOW_SIZE) - { - without_last_chunk_size = ZX0_DEFAULT_WINDOW_SIZE; - } - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE - without_last_chunk_size; -} - -uint8_t* streamed_text_data_table::get_window_end() const -{ - return decompression_buffer_ + ZX0_DEFAULT_WINDOW_SIZE + last_chunk_size_; -} - -uint16_t streamed_text_data_table::get_current_zx0_window_size() const -{ - return static_cast(get_window_end() - get_window_start()); -} \ No newline at end of file diff --git a/source/text_engine.cpp b/source/text_engine.cpp index 9fb9b3f..d5f06b0 100644 --- a/source/text_engine.cpp +++ b/source/text_engine.cpp @@ -30,11 +30,10 @@ bool text_exit; // attribute noinline was used to make sure the compiler doesn't inline this code back into text_loop() static __attribute__((noinline)) const u8* read_dialogue_text_entry(uint8_t index, u8 *output_buffer) { - u8 text_decompression_buffer[3072]; - u8 index_buffer[100]; + u8 text_decompression_buffer[6144]; const u8 *text_entry; - streamed_text_data_table dialogue_table(text_decompression_buffer, sizeof(text_decompression_buffer), index_buffer); + text_data_table dialogue_table(text_decompression_buffer); dialogue_table.decompress(get_compressed_PTGB_table()); diff --git a/source/zx0_decompressor.cpp b/source/zx0_decompressor.cpp deleted file mode 100644 index 8aac8a3..0000000 --- a/source/zx0_decompressor.cpp +++ /dev/null @@ -1,326 +0,0 @@ -#include "zx0_decompressor.h" -#include -#include - -// The following code is a custom implementation of the ZX0 decompression algorithm invented by Einar Saukas -// Original implementation can be found here: https://github.com/einar-saukas/ZX0 -// The header provides a C facade to access the relevant methods, but the rest of Poke Transporter GB -// doesn't need to be aware of all the datatypes/classes defined here. -/** -* This class makes reading on a per-bit basis much easier. -*/ -class BitReader -{ -public: - BitReader(const uint8_t* buffer); - - IWRAM_CODE uint32_t read(uint32_t numBits); -protected: -private: - const uint8_t* cur_buffer_; - uint32_t cur_dword_; - uint32_t bits_left_; -}; - -enum class ZX0OperationType -{ - NONE, - LITERAL_BLOCK, - COPY_LAST_OFFSET, - COPY_NEW_OFFSET -}; - -typedef struct ZX0Command -{ - ZX0OperationType cmd_type; - uint32_t length; - uint32_t offset; - uint32_t byte_pos; -} ZX0Command; - -/** - * @brief This class implements the actual ZX0 decompression. - */ -class ZX0Decompressor -{ -public: - ZX0Decompressor(); - - /** - * @brief This function prepares the ZX0Decompressor instance - * for decompressing the specified inputData - * into the specified output_buffer - * @param compressedData - */ - void start(uint8_t *output_buffer, const uint8_t *input_data); - - /** - * @brief Retrieves the size of the data when it is fully decompressed - * This is read from the first 2 bytes of the inputData - */ - IWRAM_CODE uint32_t get_decompressed_size() const; - - /** - * @brief This function reads of data into - */ - IWRAM_CODE void read(uint32_t num_bytes); - - /** - * @brief This function swaps out the current output buffer for the given one - */ - IWRAM_CODE void swap_output_buffer(uint8_t *new_output_buffer); -protected: -private: - IWRAM_CODE void read_next_command(); - IWRAM_CODE uint32_t copy_block(uint32_t num_bytes); - - BitReader reader_; - ZX0Command cur_command_; - const uint8_t *input_data_; - uint8_t *back_pos_; - uint8_t *cur_out_; - uint32_t last_offset_; -}; - -IWRAM_CODE static inline uint32_t read_elias_gamma(BitReader& reader) -{ - uint32_t num_non_leading_bits = 0; - uint32_t value; - while (!reader.read(1)) - { - ++num_non_leading_bits; // Count leading zeros - } - // reconstruct the most significant bit of value - value = (1 << num_non_leading_bits) | reader.read(num_non_leading_bits); // Start with MSB - - // Adjust back to zero-based - return value - 1; -} - -IWRAM_CODE static inline void read_new_offset(BitReader& reader, uint32_t& offset) -{ - const uint32_t has_msb = reader.read(1); - - const uint32_t lsb = reader.read(7); - const uint32_t msb = (has_msb) ? read_elias_gamma(reader) : 0; - - offset = ((msb << 7) | lsb) + 1; -} - -BitReader::BitReader(const uint8_t* buffer) - : cur_buffer_(buffer) - , cur_dword_(0) - , bits_left_(0) -{ -} - -IWRAM_CODE inline uint32_t BitReader::read(uint32_t num_bits) -{ - uint32_t result; - - // Fast path: Read all bits from cached data - if (num_bits <= bits_left_) - { - result = (cur_dword_ >> (bits_left_ - num_bits)) & ((1 << num_bits) - 1); - bits_left_ -= num_bits; - return result; - } - - // Slow path: Refill cache and combine bits - result = cur_dword_ & ((1 << bits_left_) - 1); - num_bits -= bits_left_; - - // Refill cache (32-bit aligned read) - // but the GBA (or x86 processor on pc) would read the value as little endian. - // and we need it as big endian. Therefore we do a byte swap - cur_dword_ = __builtin_bswap32(*(uint32_t*)cur_buffer_); - - cur_buffer_ += sizeof(uint32_t); - bits_left_ = 32; - - // Combine remaining bits - result = (result << num_bits) | (cur_dword_ >> (32 - num_bits)); - bits_left_ -= num_bits; - - return result; -} - -ZX0Decompressor::ZX0Decompressor() - : reader_(nullptr) - , cur_command_({ZX0OperationType::NONE, 0, 0, 0}) - , input_data_(nullptr) - , back_pos_(nullptr) - , cur_out_(nullptr) - , last_offset_(UINT32_MAX) -{ -} - -void ZX0Decompressor::start(uint8_t *output_buffer, const uint8_t *input_data) -{ - reader_ = BitReader(input_data + 4); - cur_command_ = {ZX0OperationType::NONE, 0, 0, 0}; - input_data_ = input_data; - back_pos_ = nullptr; - cur_out_ = output_buffer; - last_offset_ = UINT32_MAX; -} - -IWRAM_CODE uint32_t ZX0Decompressor::get_decompressed_size() const -{ - if(!input_data_) - { - return 0; - } - return *((uint32_t*)input_data_); -} - -IWRAM_CODE void ZX0Decompressor::read(uint32_t num_bytes) -{ - while(num_bytes) - { - // Check if we have finished processing the previous pending command - // if we have, we need to read a new operation - if(cur_command_.byte_pos >= cur_command_.length) - { - read_next_command(); - } - - const uint32_t bytes_read = copy_block(num_bytes); - num_bytes -= bytes_read; - } -} - -IWRAM_CODE void ZX0Decompressor::swap_output_buffer(uint8_t *new_output_buffer) -{ - const uint32_t current_offset = cur_out_ - back_pos_; - cur_out_ = new_output_buffer; - back_pos_ = new_output_buffer - current_offset; -} - -IWRAM_CODE inline void ZX0Decompressor::read_next_command() -{ - const uint32_t cmd_bit = reader_.read(1); - - // the "COPY_NEW_OFFSET" command adds + 1 to the length, but the other commands don't. - // given that read_elias_gamma() function is marked "inline", the way I set the length - // is to avoid having multiple calls to it here. (for code size) - if(cmd_bit) - { - read_new_offset(reader_, last_offset_); - cur_command_.cmd_type = ZX0OperationType::COPY_NEW_OFFSET; - cur_command_.length = 1; - cur_command_.offset = last_offset_; - } - else if(cur_command_.cmd_type == ZX0OperationType::LITERAL_BLOCK) - { - cur_command_.cmd_type = ZX0OperationType::COPY_LAST_OFFSET; - // copy from new offset and last offset differs in the sense that with the new offset the encoded length is reduced by one - // and for last offset it isn't. This is likely because you still need to be able to insert a dummy "copy-from-last-offset" operation. - cur_command_.length = 0; - cur_command_.offset = last_offset_; - } - else - { - cur_command_.cmd_type = ZX0OperationType::LITERAL_BLOCK; - cur_command_.length = 0; - } - cur_command_.length += read_elias_gamma(reader_); - cur_command_.byte_pos = 0; -} - -IWRAM_CODE uint32_t ZX0Decompressor::copy_block(uint32_t num_bytes) -{ - const uint32_t available = cur_command_.length - cur_command_.byte_pos; - const uint32_t bytes_to_read = (num_bytes > available) ? available : num_bytes; - uint32_t bytes_remaining = bytes_to_read; - - if(cur_command_.cmd_type == ZX0OperationType::LITERAL_BLOCK) - { - // Literal copy - - // Align cur_out_ first - while (bytes_remaining && ((uintptr_t)cur_out_ & 3)) - { - (*cur_out_++) = reader_.read(8); - bytes_remaining--; - } - - // Use bulk 32-bit writes when aligned - while (bytes_remaining >= 4) - { - // we need to swap again, because the data was originally stored in big endian format - // BitReader converted it to little endian format to make reading easier. - // and now we need to convert it back to big endian format. - *(uint32_t*)cur_out_ = __builtin_bswap32(reader_.read(32)); - cur_out_ += 4; - bytes_remaining -= 4; - } - // Handle remaining bytes - while (bytes_remaining--) - { - (*cur_out_++) = reader_.read(8); - } - } - else - { - if(!cur_command_.byte_pos) - { - back_pos_ = cur_out_ - cur_command_.offset; - } - - // try to get cur_out_ and back_pos aligned to 32 bit accesses first - while (bytes_remaining && (((uintptr_t)cur_out_ & 3) || ((uintptr_t)back_pos_ & 3))) - { - (*cur_out_++) = (*back_pos_++); - bytes_remaining--; - } - - // now try bulk 32 bit writes - while(bytes_remaining >= 4) - { - // these don't need to be byteswapped, because the data is being read with the same endianness as it is being written. - // this is different when reading from BitReader. - *(uint32_t*)cur_out_ = *((uint32_t*)back_pos_); - cur_out_ += 4; - back_pos_ += 4; - bytes_remaining -= 4; - } - - while(bytes_remaining--) - { - (*cur_out_++) = (*back_pos_++); - } - } - - cur_command_.byte_pos += bytes_to_read; - - return bytes_to_read; -} - -// gets stored in .bss, and therefore will end up in IWRAM by default -static ZX0Decompressor decompressor; - -extern "C" -{ -void zx0_decompressor_start(uint8_t *output_buffer, const uint8_t *input_data) -{ - decompressor.start(output_buffer, input_data); -} - -uint32_t zx0_decompressor_get_decompressed_size() -{ - return decompressor.get_decompressed_size(); -} - -void zx0_decompressor_read(uint32_t num_bytes) -{ - decompressor.read(num_bytes); -} - -void zx0_decompressor_read_partial(uint8_t *output_buffer, uint16_t num_bytes) -{ - decompressor.swap_output_buffer(output_buffer); - decompressor.read(num_bytes); -} - -} \ No newline at end of file diff --git a/text_helper/main.py b/text_helper/main.py index 723d0ed..6876350 100755 --- a/text_helper/main.py +++ b/text_helper/main.py @@ -449,15 +449,15 @@ with open (os.curdir + '/include/translated_text.h', 'w') as hFile: # PKMN_NAMES write_enum_to_header_file(hFile, "PKMN_NAMES_", mainDict[lang.name]["PKMN_NAMES"]) - hFile.write("/** Returns the ZX0 compressed PTGB text table.*/\n") + hFile.write("/** Returns the LZ10 compressed PTGB text table.*/\n") hFile.write("const u8* get_compressed_PTGB_table();\n\n") - hFile.write("/** Returns the ZX0 compressed RSEFRLG text table.*/\n") + hFile.write("/** Returns the LZ10 compressed RSEFRLG text table.*/\n") hFile.write("const u8* get_compressed_rsefrlg_table();\n\n") - hFile.write("/** Returns the ZX0 compressed GENERAL text table.*/\n") + hFile.write("/** Returns the LZ10 compressed GENERAL text table.*/\n") hFile.write("const u8* get_compressed_general_table();\n\n") - hFile.write("/** Returns the ZX0 compressed CREDITS text table.*/\n") + hFile.write("/** Returns the LZ10 compressed CREDITS text table.*/\n") hFile.write("const u8* get_compressed_credits_table();\n\n") - hFile.write("/** Returns the ZX0 compressed PKMN_NAMES text table.*/\n") + hFile.write("/** Returns the LZ10 compressed PKMN_NAMES text table.*/\n") hFile.write("const u8* get_compressed_pkmn_names_table();\n\n") hFile.write("\n#endif") @@ -487,39 +487,39 @@ for lang in Languages: # now generate the cpp file. with open(os.curdir + '/source/translated_text.cpp', 'w') as cppFile: - cppFile.write("#include \"translated_text.h\"\n#include \"debug_mode.h\"\n#include \"pokemon_data.h\"\n#include \"zx0_decompressor.h\"\n") + cppFile.write("#include \"translated_text.h\"\n#include \"debug_mode.h\"\n#include \"pokemon_data.h\"\n") # generate includes for each language for lang in Languages: for cat in mainDict[lang.name]: if cat in {"PTGB", "RSEFRLG", "GENERAL", "CREDITS", "PKMN_NAMES"}: - cppFile.write("#include \"" + cat.upper() + "_" + lang.name.lower() + "_zx0_bin.h\"\n") + cppFile.write("#include \"" + cat.upper() + "_" + lang.name.lower() + "_lz10_bin.h\"\n") for lang in Languages: cppFile.write(f"\n#if PTGB_BUILD_LANGUAGE == {lang.value + 1}\n") # PTGB cppFile.write("const u8* get_compressed_PTGB_table()\n") cppFile.write("{\n") - cppFile.write("\treturn PTGB_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn PTGB_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # RSEFRLG cppFile.write("const u8* get_compressed_rsefrlg_table()\n") cppFile.write("{\n") - cppFile.write("\treturn RSEFRLG_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn RSEFRLG_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # GENERAL cppFile.write("const u8* get_compressed_general_table()\n") cppFile.write("{\n") - cppFile.write("\treturn GENERAL_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn GENERAL_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # CREDITS cppFile.write("const u8* get_compressed_credits_table()\n") cppFile.write("{\n") - cppFile.write("\treturn CREDITS_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn CREDITS_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") # PKMN_NAMES cppFile.write("const u8* get_compressed_pkmn_names_table()\n") cppFile.write("{\n") - cppFile.write("\treturn PKMN_NAMES_" + lang.name.lower() + "_zx0_bin;\n") + cppFile.write("\treturn PKMN_NAMES_" + lang.name.lower() + "_lz10_bin;\n") cppFile.write("}\n\n") cppFile.write(f"#endif\n\n\n") diff --git a/tools/compressZX0/Makefile b/tools/compressZX0/Makefile deleted file mode 100644 index 5582b00..0000000 --- a/tools/compressZX0/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -# # Compiler flags -CXXFLAGS := -std=c++11 -fno-rtti -fno-exceptions -fno-unwind-tables -Wall -Wextra -I $(CURDIR) -g - -# Source files directory -SRC_DIR := . -# Build directory -BUILD_DIR := build - -# Source files (add more as needed) -SRCS := $(shell find $(SRC_DIR) -type f -name '*.cpp') -# Object files -OBJS := $(patsubst $(SRC_DIR)/%.cpp,$(BUILD_DIR)/%.o,$(SRCS)) - -# Ensure necessary directories exist -# This function ensures the directory for the target exists -define make_directory - @mkdir -p $(dir $@) -endef - -# Target executable -TARGET := compressZX0 - -# Phony targets -.PHONY: all clean - -# Default target -all: $(TARGET) - -$(TARGET): $(OBJS) - $(CXX) $(CXXFLAGS) $^ -o $@ - -# Rule to compile source files -$(BUILD_DIR)/%.o: $(SRC_DIR)/%.cpp | $(BUILD_DIR) - $(make_directory) - $(CXX) $(CXXFLAGS) -c $< -o $@ - -# Create the build directory if it doesn't exist -$(BUILD_DIR): - mkdir -p $(BUILD_DIR) - -# Clean rule -clean: - rm -rf $(BUILD_DIR) $(TARGET) diff --git a/tools/compressZX0/main.cpp b/tools/compressZX0/main.cpp deleted file mode 100644 index 9bbce2f..0000000 --- a/tools/compressZX0/main.cpp +++ /dev/null @@ -1,436 +0,0 @@ -#include -#include -#include -#include - -// @author risingPhil -// This file implements the zx0 compression algorithm. -// It serves as a test to see if this algorithm could suit us for the compression of several static arrays/buffers in Poke Transporter GB. - -//#define LOG_OPERATIONS 1 - -#define MAX_OFFSET 2048 // Maximum backward offset (ZX0 limit), tuned for Poke Transporter GB to be able to use a 2 KB decompression buffer -#define MAX_LEN 255 // Maximum match length (ZX0 limit) -#define OUTPUT_BUFFER_SIZE 256 * 1024 // Maximum output buffer size (artificial limitation, I suppose) - -#ifdef LOG_OPERATIONS - #define LOG_OP(...) printf(__VA_ARGS__) -#else - #define LOG_OP(...) -#endif - -// anonymous namespace for internal linkage -namespace -{ - -/** -* This class makes writing on a per-bit basis much easier -*/ -class BitWriter -{ -public: - BitWriter(uint8_t* buffer); - - void write(uint8_t value, uint8_t numBits); - void write_bit(uint8_t value); - void write_byte(uint8_t value); - - size_t get_bits_written() const; -protected: -private: - uint8_t* buffer_; - uint8_t* cur_buffer_; - uint8_t bit_index_; -}; - -BitWriter::BitWriter(uint8_t* buffer) - : buffer_(buffer) - , cur_buffer_(buffer) - , bit_index_(0) -{ -} - -void BitWriter::write_bit(uint8_t value) -{ - value &= 0x1; - *cur_buffer_ |= (value << (7 - bit_index_)); - - if(bit_index_ == 7) - { - ++cur_buffer_; - bit_index_ = 0; - } - else - { - ++bit_index_; - } -} - -void BitWriter::write_byte(uint8_t value) -{ - if(bit_index_) - { - write(value, 8); - return; - } - *cur_buffer_ = value; - ++cur_buffer_; -} - -void BitWriter::write(uint8_t value, uint8_t numBits) -{ - for(int i=numBits - 1; i >= 0; --i) - { - write_bit(value >> i); - } -} - -size_t BitWriter::get_bits_written() const -{ - return (cur_buffer_ - buffer_) * 8 + bit_index_; -} -} - -/** - * @brief Find the best match for the current position (LZ77-style) - * We simply try to find the longest matching bytes backwards in the buffer. - */ -static void find_backwards_match(const unsigned char *buffer, size_t buffer_size, int pos, int *best_offset, int *best_len) -{ - *best_offset = 0; - *best_len = 0; - const size_t max_offset = (pos > MAX_OFFSET) ? MAX_OFFSET : pos; - const int max_len = (buffer_size - pos > MAX_LEN) ? MAX_LEN : buffer_size - pos; - int len; - - for (size_t offset = 1; offset <= max_offset; offset++) - { - len = 0; - while (len < max_len && buffer[pos - offset + len] == buffer[pos + len]) - { - ++len; - } - if (len > *best_len) - { - *best_len = len; - *best_offset = offset; - } - } -} - -/** - * @brief This function encodes the specified value with gamma encoding. - * - * The way it works is that we first determine of how many bits the value consists, except for the leading bits. (=num_non_leading_bits) - * Then we write zeros. - * We also write the original value in bits - * - * For decoding, we can determine the number of zeros and that will indicate how many bits we need to read for the actual value. - * - */ -static void write_elias_gamma(BitWriter& writer, int value) -{ - value++; // Adjust because Gamma only encodes n ≥ 1 - int num_non_leading_bits = 0; - int i; - - // Calculate floor(log2(value)) - int tmp = value >> 1; - while(tmp) - { - ++num_non_leading_bits; - tmp >>= 1; - } - - // Write unary part (k zeros) - for (i = 0; i < num_non_leading_bits; i++) - { - writer.write_bit(0); - } - // Write binary part (num_non_leading_bits+1 bits of value) - for (int i = num_non_leading_bits; i >= 0; i--) - { - writer.write_bit(value >> i); - } -} - -/** - * This struct represents a buffer to hold a number of pending "literal" bytes - * before they actually get written to the output - */ -typedef struct LiteralBuffer -{ - uint8_t buffer[1024]; - uint16_t size; -} LiteralBuffer; - -/** - * @brief This function writes a command for the decompressor to start copying bytes - * from the last offset specified with the write_copy_from_new_offset_block() function - */ -static void write_copy_from_last_offset_block(BitWriter& writer, int length) -{ - LOG_OP("copy_last: %d\n", length); - writer.write_bit(0); - write_elias_gamma(writer, length); -} - -/** - * @brief Writes a command to copy the bytes in LiteralBuffer to the decompressed buffer. - */ -static void write_literal_block(BitWriter& writer, LiteralBuffer& literal_buffer) -{ - uint16_t i = 0; - - if(!literal_buffer.size) - { - return; - } - - LOG_OP("copy_literal: %hu\n", literal_buffer.size); - - // flag that this is a literal block - writer.write_bit(0); - write_elias_gamma(writer, literal_buffer.size); - - while(i < literal_buffer.size) - { - writer.write_byte(literal_buffer.buffer[i]); - - ++i; - } - literal_buffer.size = 0; -} - -/** - * @brief Writes a command to indicate that the decompressor must copy bytes from the given backwards offset. - */ -static void write_copy_from_new_offset_block(BitWriter& writer, int offset, int length) -{ - LOG_OP("copy_new: offset: %d, length: %d\n", offset, length); - writer.write_bit(1); // Match flag - - // Encode offset (Elias Gamma + 7-bit LSB) - const int msb = ((offset - 1) >> 7) & 0xFF; - const int lsb = (offset - 1) & 0x7F; - - // first bit of LSB indicates whether the MSB follows. - writer.write_bit((msb > 0)); - - // write 7 bit LSB raw bits - writer.write(lsb, 7); - - if (msb > 0) - { - write_elias_gamma(writer, msb); - } - - // Encode length (Elias Gamma) - write_elias_gamma(writer, length - 1); -} - -static void literal_buffer_push(BitWriter& writer, LiteralBuffer& literal_buffer, uint8_t byte) -{ - if(literal_buffer.size == 1024) - { - // EDGE case: buffer is full. - // back-to-back literal blocks are forbidden, - // so we must insert a dummy "use last offset" block - write_literal_block(writer, literal_buffer); - write_copy_from_last_offset_block(writer, 0); - } - - literal_buffer.buffer[literal_buffer.size] = byte; - ++literal_buffer.size; -} - -/** - * This function encodes the specified buffer with the ZX0 compression algorithm - * and stores the result into output_buffer. - * - * Please make sure the output_buffer is sufficiently large enough before calling this function. - */ -static size_t encodeZX0(uint8_t* output_buffer, const uint8_t* buffer, size_t buffer_size) -{ - BitWriter writer(output_buffer); - LiteralBuffer literal_buffer = { - .buffer = {0}, - .size = 0 - }; - - int pos = 0; - int last_offset = 0x7FFFFFFF; - int offset; - int length; - int numBytes = buffer_size; - - // first write the size of the input in little endian format in the output buffer - writer.write_byte(static_cast(buffer_size)); - writer.write_byte(static_cast(buffer_size >> 8)); - writer.write_byte(static_cast(buffer_size >> 16)); - writer.write_byte(static_cast(buffer_size >> 24)); - - while(pos < numBytes) - { - find_backwards_match(buffer, numBytes, pos, &offset, &length); - - // important rules: You cannot have 2 consecutive literal blocks. - // reusing the last offset can only happen after a literal block! - - if(length < 2) - { - // we must buffer the literals because we can only start writing them when we know the "length" - literal_buffer_push(writer, literal_buffer, buffer[pos]); - ++pos; - } - else if(offset == last_offset) - { - // write any pending literal bytes - write_literal_block(writer, literal_buffer); - write_copy_from_last_offset_block(writer, length); - pos += length; - } - else - { - // write any pending literal bytes - write_literal_block(writer, literal_buffer); - write_copy_from_new_offset_block(writer, offset, length); - last_offset = offset; - pos += length; - } - } - - return writer.get_bits_written(); -} - -/** - * @brief Reads the given file completely into the specified buffer. - * The buffer is allocated by this function, but should be delete[]'d by the caller. - */ -static bool read_file(const char* filename, uint8_t*& out_buffer, size_t& out_size) -{ - FILE* file; - long size; - size_t read; - uint8_t* buffer; - - file = fopen(filename, "rb"); - if (!file) return false; - - // Seek to end to determine size - if (fseek(file, 0, SEEK_END) != 0) - { - fclose(file); - return false; - } - - size = ftell(file); - if (size < 0) - { - fclose(file); - return false; - } - - rewind(file); - - buffer = new uint8_t[size]; - - read = fread(buffer, 1, size, file); - fclose(file); - - if (read != (size_t)size) { - delete[] buffer; - return false; - } - - out_buffer = buffer; - out_size = size; - - return true; -} - -static void print_usage() -{ - const char* usageString = R"delim( -Usage: compressZX0 - -This program will compress the given file with the ZX0 compression algorithm and store the output in -/_zx0.bin -)delim"; - printf(usageString); -} - -int main(int argc, char** argv) -{ - // Reserve 256KB buffer, which is already much larger than the maximum file size we'd allow for PTGB. - // (the reason why I'm using a buffer instead of writing directly to a file is simply because I'm lazy. - // I wrote a test of the algorithm using buffers first. And I know that for Poke Transporter GB specifically - // we'll never exceed the 256KB filesize. So I'm not going to rework this code, because there's currently no need) - uint8_t output_buffer[OUTPUT_BUFFER_SIZE] = {0}; - uint8_t *input_buffer = nullptr; - char *filename; - char *extension_dot; - size_t input_buffer_size; - size_t bits_written; - size_t num_bytes; - double compress_ratio; - char output_path[4096]; - FILE* f; - - if(argc < 3) - { - print_usage(); - return 1; - } - - if(!read_file(argv[1], input_buffer, input_buffer_size)) - { - perror("Could not open file: "); - return 1; - } - - // make sure the input_buffer_size is not larger than our output_buffer we statically allocated - // This is a bit of an artificial limitation though. - if(input_buffer_size > sizeof(output_buffer)) - { - fprintf(stderr, "ERROR: The input file should not be larger than %zu KB!\n", sizeof(output_buffer)); - return 1; - } - - // get the filename part of the given file - // and remove the extension. - // basename uses statically allocated memory that gets overwritten by each call. - // but it returns a modifiable char* - // so we might as well just edit that buffer directly because no-one will depend on this value later. - filename = basename(argv[1]); - - printf("Compressing %s...", filename); - - bits_written = encodeZX0(output_buffer, input_buffer, input_buffer_size); - delete[] input_buffer; - input_buffer = nullptr; - num_bytes = (bits_written + 7) / 8; - - printf("done\n"); - - // if we have an extension in the filename, just end the string at the '.' position. - extension_dot = strchr(filename, '.'); - if(extension_dot) - { - *extension_dot = '\0'; - } - - // argv[2] should be the output directory - snprintf(output_path, sizeof(output_path), "%s/%s_zx0.bin", argv[2], filename); - - f = fopen(output_path, "wb+"); - fwrite(output_buffer, 1, num_bytes, f); - fclose(f); - - compress_ratio = static_cast(num_bytes) / input_buffer_size; - printf("Compressed size: %zu bytes/%zu bytes, Compression ratio: %f%%\n", num_bytes, input_buffer_size, compress_ratio * 100.f); - - return 0; -} - - diff --git a/tools/data-generator/src/main.cpp b/tools/data-generator/src/main.cpp index da5cfed..44890cc 100644 --- a/tools/data-generator/src/main.cpp +++ b/tools/data-generator/src/main.cpp @@ -11,7 +11,7 @@ #include // This application holds the various long static data arrays that Poke Transporter GB uses -// and it writes them to .bin files that can be compressed with compressZX0 later. +// and it writes them to .bin files that can be compressed with gbalzss later. // it's useful to do it this way because it keeps this data easy to view, edit and document // This function generates a binary file containing the specified list of ROM_DATA structs