From ddf7820eacd4eee0102dfd77cd49429915d52365 Mon Sep 17 00:00:00 2001 From: Rangi <35663410+Rangi42@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:42:20 -0400 Subject: [PATCH] Replace lzcomp with lzcompress (#148) --- Makefile | 2 +- gfx/lz.mk | 58 +-- tools/.gitignore | 2 +- tools/Makefile | 6 +- tools/lz/global.c | 35 -- tools/lz/main.c | 54 --- tools/lz/merging.c | 102 ------ tools/lz/mpcomp.c | 112 ------ tools/lz/nullcomp.c | 20 -- tools/lz/options.c | 141 -------- tools/lz/output.c | 146 -------- tools/lz/packing.c | 56 --- tools/lz/proto.h | 107 ------ tools/lz/repcomp.c | 63 ---- tools/lz/spcomp.c | 141 -------- tools/lz/uncomp.c | 92 ----- tools/lz/util.c | 54 --- tools/lzcompress.c | 841 ++++++++++++++++++++++++++++++++++++++++++++ 18 files changed, 873 insertions(+), 1159 deletions(-) delete mode 100644 tools/lz/global.c delete mode 100644 tools/lz/main.c delete mode 100644 tools/lz/merging.c delete mode 100644 tools/lz/mpcomp.c delete mode 100644 tools/lz/nullcomp.c delete mode 100644 tools/lz/options.c delete mode 100644 tools/lz/output.c delete mode 100644 tools/lz/packing.c delete mode 100644 tools/lz/proto.h delete mode 100644 tools/lz/repcomp.c delete mode 100644 tools/lz/spcomp.c delete mode 100644 tools/lz/uncomp.c delete mode 100644 tools/lz/util.c create mode 100644 tools/lzcompress.c diff --git a/Makefile b/Makefile index 8ccbe174..552f6228 100644 --- a/Makefile +++ b/Makefile @@ -207,7 +207,7 @@ pokesilver_vc.gbc: RGBFIXFLAGS += -t POKEMON_SLV -i AAXE include gfx/lz.mk %.lz: % - tools/lzcomp $(LZFLAGS) -- $< $@ + tools/lzcompress $(LZFLAGS) -- $< $@ ### Pokemon and trainer sprite rules diff --git a/gfx/lz.mk b/gfx/lz.mk index f8187948..305c9499 100644 --- a/gfx/lz.mk +++ b/gfx/lz.mk @@ -1,17 +1,17 @@ # Delete these rules if you don't care about matching. -%.lz: LZFLAGS = --compressor multipass +%.lz: LZFLAGS = --matching -gfx/battle_anims/%.lz: LZFLAGS += --method 2 --align 4 +gfx/battle_anims/%.lz: LZFLAGS += --odd-alternate --align 4 gfx/battle/dude.2bpp.lz: LZFLAGS += --align 4 -gfx/diploma/diploma.2bpp.lz: LZFLAGS += --method 8 --align 1 +gfx/diploma/diploma.2bpp.lz: LZFLAGS += --prefer-alternate --align 1 -gfx/memory_game/memory_game.2bpp.lz: LZFLAGS += --method 2 --align 4 +gfx/memory_game/memory_game.2bpp.lz: LZFLAGS += --odd-alternate --align 4 gfx/intro/%.lz: LZFLAGS += --align 4 -gfx/intro/water1.2bpp.lz: LZFLAGS += --method 2 --align 4 +gfx/intro/water1.2bpp.lz: LZFLAGS += --odd-alternate --align 4 gfx/new_game/shrink1.2bpp.lz: LZFLAGS += --align 4 gfx/new_game/shrink2.2bpp.lz: LZFLAGS += --align 4 @@ -20,41 +20,41 @@ gfx/pc/pc.2bpp.lz: LZFLAGS += --align 4 gfx/player/chris_back.2bpp.lz: LZFLAGS += --align 4 -gfx/pokedex/pokedex.2bpp.lz: LZFLAGS += --method 8 --align 1 -gfx/pokedex/pokedex_sgb.2bpp.lz: LZFLAGS += --method 8 --align 1 +gfx/pokedex/pokedex.2bpp.lz: LZFLAGS += --prefer-alternate --align 1 +gfx/pokedex/pokedex_sgb.2bpp.lz: LZFLAGS += --prefer-alternate --align 1 gfx/pokedex/question_mark.2bpp.lz: LZFLAGS += --align 4 -gfx/pokedex/slowpoke.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 +gfx/pokedex/slowpoke.2bpp.lz: LZFLAGS += --literal-only --align 1 -gfx/pokegear/pokegear.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 +gfx/pokegear/pokegear.2bpp.lz: LZFLAGS += --literal-only --align 1 gfx/pokegear/pokegear_sprites.2bpp.lz: LZFLAGS += --align 4 -gfx/pokegear/town_map.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 +gfx/pokegear/town_map.2bpp.lz: LZFLAGS += --literal-only --align 1 -gfx/pokemon/hitmontop/back_gold.2bpp.lz: LZFLAGS += --method 4 -gfx/pokemon/miltank/back_gold.2bpp.lz: LZFLAGS += --method 4 +gfx/pokemon/hitmontop/back_gold.2bpp.lz: LZFLAGS += --no-lookback-3 +gfx/pokemon/miltank/back_gold.2bpp.lz: LZFLAGS += --no-lookback-3 -gfx/slots/slots_1.2bpp.lz: LZFLAGS += --method 2 --align 4 +gfx/slots/slots_1.2bpp.lz: LZFLAGS += --odd-alternate --align 4 gfx/slots/slots_2.2bpp.lz: LZFLAGS += --align 4 gfx/slots/slots_3.2bpp.lz: LZFLAGS += --align 4 gfx/tilesets/%.lz: LZFLAGS += --align 4 -gfx/tilesets/game_corner.2bpp.lz: LZFLAGS += --method 2 --align 4 -gfx/tilesets/house.2bpp.lz: LZFLAGS += --method 2 --align 4 -gfx/tilesets/johto.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 -gfx/tilesets/johto_modern.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 -gfx/tilesets/kanto.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1 -gfx/tilesets/mansion.2bpp.lz: LZFLAGS += --method 2 --align 4 -gfx/tilesets/mart.2bpp.lz: LZFLAGS += --method 2 --align 4 -gfx/tilesets/players_room.2bpp.lz: LZFLAGS += --method 4 --align 4 -gfx/tilesets/radio_tower.2bpp.lz: LZFLAGS += --method 2 --align 4 -gfx/tilesets/ruins_of_alph.2bpp.lz: LZFLAGS += --method 6 --align 4 -gfx/tilesets/tower.2bpp.lz: LZFLAGS += --method 2 --align 4 +gfx/tilesets/game_corner.2bpp.lz: LZFLAGS += --odd-alternate --align 4 +gfx/tilesets/house.2bpp.lz: LZFLAGS += --odd-alternate --align 4 +gfx/tilesets/johto.2bpp.lz: LZFLAGS += --literal-only --align 1 +gfx/tilesets/johto_modern.2bpp.lz: LZFLAGS += --literal-only --align 1 +gfx/tilesets/kanto.2bpp.lz: LZFLAGS += --literal-only --align 1 +gfx/tilesets/mansion.2bpp.lz: LZFLAGS += --odd-alternate --align 4 +gfx/tilesets/mart.2bpp.lz: LZFLAGS += --odd-alternate --align 4 +gfx/tilesets/players_room.2bpp.lz: LZFLAGS += --no-lookback-3 --align 4 +gfx/tilesets/radio_tower.2bpp.lz: LZFLAGS += --odd-alternate --align 4 +gfx/tilesets/ruins_of_alph.2bpp.lz: LZFLAGS += --odd-alternate --no-lookback-3 --align 4 +gfx/tilesets/tower.2bpp.lz: LZFLAGS += --odd-alternate --align 4 gfx/title/hooh_gold.2bpp.lz: LZFLAGS += --align 3 -gfx/title/logo_bottom_gold.2bpp.lz: LZFLAGS += --method 9 --align 1 -gfx/title/logo_bottom_silver.2bpp.lz: LZFLAGS += --method 9 --align 1 -gfx/title/logo_top_gold.2bpp.lz: LZFLAGS += --method 9 --align 1 -gfx/title/logo_top_silver.2bpp.lz: LZFLAGS += --method 9 --align 1 +gfx/title/logo_bottom_gold.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1 +gfx/title/logo_bottom_silver.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1 +gfx/title/logo_top_gold.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1 +gfx/title/logo_top_silver.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1 gfx/title/lugia_silver.2bpp.lz: LZFLAGS += --align 4 gfx/unown_puzzle/%.lz: LZFLAGS += --align 4 -gfx/unown_puzzle/start_cancel.2bpp.lz: LZFLAGS += --method 8 --align 1 +gfx/unown_puzzle/start_cancel.2bpp.lz: LZFLAGS += --prefer-alternate --align 1 diff --git a/tools/.gitignore b/tools/.gitignore index e69be250..6ddc35e6 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -1,6 +1,6 @@ gbcpal gfx -lzcomp +lzcompress make_patch png_dimensions scan_includes diff --git a/tools/Makefile b/tools/Makefile index 69a1af05..36bc73b8 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -6,7 +6,7 @@ CFLAGS := -O3 -std=c11 -Wall -Wextra -pedantic tools := \ gbcpal \ gfx \ - lzcomp \ + lzcompress \ make_patch \ png_dimensions \ scan_includes \ @@ -18,9 +18,5 @@ all: $(tools) clean: $(RM) $(tools) -lzcomp: CFLAGS += -Wno-strict-overflow -Wno-sign-compare -lzcomp: $(wildcard lz/*.c) $(wildcard lz/*.h) - $(CC) $(CFLAGS) -o $@ lz/*.c - %: %.c common.h $(CC) $(CFLAGS) -o $@ $< diff --git a/tools/lz/global.c b/tools/lz/global.c deleted file mode 100644 index 4d26acee..00000000 --- a/tools/lz/global.c +++ /dev/null @@ -1,35 +0,0 @@ -#include "proto.h" - -const struct compressor compressors[] = { - // NOTE: the "flags" field for each compressor will be set to the chosen/current method number minus the base - // number for that particular compressor. That means that each compressor will use a zero-based flags value. - {.methods = 72, .name = "singlepass", .function = &try_compress_single_pass}, // 0-71 - {.methods = 2, .name = "null", .function = &store_uncompressed}, // 72-73 - {.methods = 6, .name = "repetitions", .function = &try_compress_repetitions}, // 74-79 - {.methods = 16, .name = "multipass", .function = &try_compress_multi_pass}, // 80-95 - {0} // end of the list -}; - -const unsigned char bit_flipping_table[] = { - // For each byte, the table contains that same byte with its bits flipped around (for instance, - // 0x58 (01011000 binary) becomes 0x1a (00011010 binary)). This is faster than flipping bits - // manually at runtime. - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff -}; - -char option_name_buffer[] = "-?"; // used to extract the name of a short option (separated from its argument) diff --git a/tools/lz/main.c b/tools/lz/main.c deleted file mode 100644 index 62e6dc41..00000000 --- a/tools/lz/main.c +++ /dev/null @@ -1,54 +0,0 @@ -#include "proto.h" - -int main (int argc, char ** argv) { - struct options options = get_options(argc, argv); - unsigned short size; - unsigned char * file_buffer = read_file_into_buffer(options.input, &size); - struct command * commands; - if (options.mode & 2) { - unsigned short original_size = size, remainder; - commands = get_commands_from_file(file_buffer, &size, &remainder); - if (!commands) error_exit(1, "invalid command stream"); - if (options.mode == 2) { - unsigned char * uncompressed = get_uncompressed_data(commands, file_buffer, &size); - if (!uncompressed) error_exit(1, "output data is too large"); - write_raw_data_to_file(options.output, uncompressed, size); - free(uncompressed); - } else - write_commands_and_padding_to_textfile(options.output, commands, size, file_buffer, original_size - remainder, remainder); - } else { - commands = compress(file_buffer, &size, options.method); - (options.mode ? write_commands_to_textfile : write_commands_to_file)(options.output, commands, size, file_buffer, options.alignment); - } - free(file_buffer); - free(commands); - return 0; -} - -struct command * compress (const unsigned char * data, unsigned short * size, unsigned method) { - unsigned char * bitflipped = malloc(*size); - unsigned current; - for (current = 0; current < *size; current ++) bitflipped[current] = bit_flipping_table[data[current]]; - const struct compressor * compressor = compressors; - struct command * result; - if (method < COMPRESSION_METHODS) { - while (method >= compressor -> methods) method -= (compressor ++) -> methods; - result = compressor -> function(data, bitflipped, size, method); - } else { - struct command * compressed_sequences[COMPRESSION_METHODS]; - unsigned short lengths[COMPRESSION_METHODS]; - unsigned flags = 0; - for (current = 0; current < COMPRESSION_METHODS; current ++) { - lengths[current] = *size; - if (flags == compressor -> methods) { - flags = 0; - compressor ++; - } - compressed_sequences[current] = compressor -> function(data, bitflipped, lengths + current, flags ++); - } - result = select_optimal_sequence(compressed_sequences, lengths, size); - for (current = 0; current < COMPRESSION_METHODS; current ++) free(compressed_sequences[current]); - } - free(bitflipped); - return result; -} diff --git a/tools/lz/merging.c b/tools/lz/merging.c deleted file mode 100644 index 79abb6d2..00000000 --- a/tools/lz/merging.c +++ /dev/null @@ -1,102 +0,0 @@ -#include "proto.h" - -struct command * select_optimal_sequence (struct command ** sequences, const unsigned short * lengths, unsigned short * final_length) { - struct command * compressor_sequences[NUM_COMPRESSORS * 2]; - unsigned short compressor_lengths[NUM_COMPRESSORS * 2]; - struct command * inverted_sequences[COMPRESSION_METHODS]; - unsigned short inverted_lengths[COMPRESSION_METHODS]; - unsigned p, current, method = 0; - for (current = 0; current < NUM_COMPRESSORS; current ++) { - compressor_sequences[current] = select_command_sequence(sequences + method, lengths + method, compressors[current].methods, compressor_lengths + current); - compressor_sequences[current + NUM_COMPRESSORS] = select_command_sequence(sequences + method, lengths + method, -(int) compressors[current].methods, - compressor_lengths + (current + NUM_COMPRESSORS)); - for (p = 0; p < compressors[current].methods; p ++) { - inverted_sequences[method + compressors[current].methods - 1 - p] = sequences[method + p]; - inverted_lengths[method + compressors[current].methods - 1 - p] = lengths[method + p]; - } - method += compressors[current].methods; - } - unsigned short final_lengths[8]; - struct command * final_sequences[8] = { - select_command_sequence(compressor_sequences, compressor_lengths, NUM_COMPRESSORS, final_lengths), - select_command_sequence(compressor_sequences, compressor_lengths, -NUM_COMPRESSORS, final_lengths + 1), - select_command_sequence(compressor_sequences + NUM_COMPRESSORS, compressor_lengths + NUM_COMPRESSORS, NUM_COMPRESSORS, final_lengths + 2), - select_command_sequence(compressor_sequences + NUM_COMPRESSORS, compressor_lengths + NUM_COMPRESSORS, -NUM_COMPRESSORS, final_lengths + 3), - select_command_sequence(sequences, lengths, COMPRESSION_METHODS, final_lengths + 4), - select_command_sequence(sequences, lengths, -COMPRESSION_METHODS, final_lengths + 5), - select_command_sequence(inverted_sequences, inverted_lengths, COMPRESSION_METHODS, final_lengths + 6), - select_command_sequence(inverted_sequences, inverted_lengths, -COMPRESSION_METHODS, final_lengths + 7) - }; - for (current = 0; current < (2 * NUM_COMPRESSORS); current ++) free(compressor_sequences[current]); - struct command * result = select_command_sequence(final_sequences, final_lengths, 8, final_length); - for (current = 0; current < 8; current ++) free(final_sequences[current]); - return result; -} - -struct command * select_command_sequence (struct command ** sequences, const unsigned short * lengths, int count, unsigned short * final_length) { - // negative count indicates iterating backwards - unsigned short min_sequence = 0, min_length = compressed_length(*sequences, *lengths); - unsigned short seq, len; - int backwards = 0; - if (count < 0) { - backwards = 1; - count = -count; - } - for (seq = 1; seq < count; seq ++) { - len = compressed_length(sequences[seq], lengths[seq]); - if (len < min_length) { - min_sequence = seq; - min_length = len; - } - } - *final_length = lengths[min_sequence]; - struct command * current = malloc(*final_length * sizeof(struct command)); - memcpy(current, sequences[min_sequence], *final_length * sizeof(struct command)); - struct command * new; - for (seq = 1; seq < count; seq ++) { - if (backwards) seq = count - seq; - new = merge_command_sequences(current, *final_length, sequences[(seq + min_sequence) % count], lengths[(seq + min_sequence) % count], final_length); - if (backwards) seq = count - seq; // restore the value for the loop - free(current); - current = new; - } - return current; -} - -struct command * merge_command_sequences (const struct command * current, unsigned short current_length, const struct command * new, unsigned short new_length, - unsigned short * result_length) { - struct command * result = malloc(sizeof(struct command) * (current_length + new_length)); - struct command * current_command = result; - const struct command * saved_current; - const struct command * saved_new; - unsigned short current_pos, new_pos; - while (current_length) { - if (current -> count == new -> count) { - *(current_command ++) = pick_best_command(2, *(current ++), *(new ++)); - current_length --; - continue; - } - saved_current = current; - saved_new = new; - current_pos = (current ++) -> count; - new_pos = (new ++) -> count; - current_length --; - while (current_pos != new_pos) - if (current_pos < new_pos) { - current_pos += (current ++) -> count; - current_length --; - } else - new_pos += (new ++) -> count; - current_pos = compressed_length(saved_current, current - saved_current); - new_pos = compressed_length(saved_new, new - saved_new); - if (new_pos < current_pos) { - memcpy(current_command, saved_new, sizeof(struct command) * (new - saved_new)); - current_command += new - saved_new; - } else { - memcpy(current_command, saved_current, sizeof(struct command) * (current - saved_current)); - current_command += current - saved_current; - } - } - *result_length = current_command - result; - return result; -} diff --git a/tools/lz/mpcomp.c b/tools/lz/mpcomp.c deleted file mode 100644 index c8829852..00000000 --- a/tools/lz/mpcomp.c +++ /dev/null @@ -1,112 +0,0 @@ -#include "proto.h" - -/* - Multi-pass compressor: performs an initial pass generating a single command for each byte position in the data and - refines the command stream further in subsequent passes. - Methods defined: 16 - Flags values: the flags are a bitfield; each bit triggers some alternate behavior if set: - 1: always emit a literal command (0) for the first byte of the file - 2: when reducing a two-byte repetition (2) command in the overlap elimination pass, don't force it to contain a - whole number of repetitions (i.e., an even count) - 4: don't emit copy commands (4, 5, 6) with a count of 3 - 8: don't emit single-byte repetition (1) commands -*/ - -struct command * try_compress_multi_pass (const unsigned char * data, const unsigned char * flipped, unsigned short * size, unsigned flags) { - struct command * result = calloc(*size, sizeof(struct command)); - unsigned char * reversed = malloc(*size); - short * sources = malloc(*size * sizeof(short)); - unsigned short pos, next, current = 0; - for (pos = 0; pos < *size; pos ++) { - reversed[pos] = data[*size - 1 - pos]; - sources[pos] = -1; - } - for (pos = (flags & 1); pos < *size; pos += (result[pos].count >= MULTIPASS_SKIP_THRESHOLD) ? result[pos].count : 1) { - result[pos] = pick_command_for_pass(data, flipped, reversed, sources, *size, pos, flags); - if ((result[pos].command >= 4) || (result[pos].count < MULTIPASS_SKIP_THRESHOLD)) sources[current ++] = pos; - } - free(reversed); - free(sources); - for (pos = 0; pos < *size; pos ++) { - for (current = 1; current < result[pos].count; current ++) if (result[pos + current].count > result[pos].count) { - result[pos].count = current; - if ((result[pos].command == 2) && (current & 1) && !(flags & 2)) result[pos].count --; - } - if (result[pos].count <= command_size(result[pos])) result[pos] = (struct command) {.command = 0, .count = 0}; - } - for (pos = 0; pos < *size; pos ++) - if (!result[pos].command) { - for (current = 1; (current < MAX_COMMAND_COUNT) && ((pos + current) < *size); current ++) if (result[pos + current].command) break; - result[pos] = (struct command) {.command = 0, .count = current, .value = pos}; - } else if (result[pos].count > MAX_COMMAND_COUNT) { - result[pos + MAX_COMMAND_COUNT] = result[pos]; - result[pos + MAX_COMMAND_COUNT].count -= MAX_COMMAND_COUNT; - if ((result[pos + MAX_COMMAND_COUNT].command >= 4) && (result[pos + MAX_COMMAND_COUNT].value >= 0)) - result[pos + MAX_COMMAND_COUNT].value += (result[pos].command == 6) ? -MAX_COMMAND_COUNT : MAX_COMMAND_COUNT; - result[pos].count = MAX_COMMAND_COUNT; - } - for (next = pos = 0; pos < *size; pos ++) - if (pos == next) - next += result[pos].count; - else - result[pos].command = 7; - repack(&result, size); - return result; -} - -struct command pick_command_for_pass (const unsigned char * data, const unsigned char * flipped, const unsigned char * reversed, const short * sources, - unsigned short length, unsigned short position, unsigned flags) { - struct command result = pick_repetition_for_pass(data, length, position, flags); - if (result.count >= MULTIPASS_SKIP_THRESHOLD) return result; - unsigned char p; - for (p = 0; p < 3; p ++) { - struct command temp = pick_copy_for_pass(data, p[(const unsigned char * []) {data, flipped, reversed}], sources, p + 4, length, position, flags); - if (temp.command == 7) continue; - if (temp.count > result.count) result = temp; - } - if ((result.command >= 4) && (result.value >= (position - LOOKBACK_LIMIT))) result.value -= position; - return result; -} - -struct command pick_repetition_for_pass (const unsigned char * data, unsigned short length, unsigned short position, unsigned flags) { - unsigned short p; - if (data[position]) { - if ((position + 1) >= length) return (struct command) {.command = 1, .count = 1, .value = data[position]}; - struct command result; - if (!(flags & 8) && (data[position] == data[position + 1])) - result = (struct command) {.command = 1, .value = data[position]}; - else - result = (struct command) {.command = 2, .value = data[position] | (data[position + 1] << 8)}; - for (p = 1; ((position + p) < length) && (p < LOOKAHEAD_LIMIT); p ++) if (data[position + p] != data[position + (p & 1)]) break; - result.count = p; - return result; - } else { - for (p = position + 1; (p < length) && (p < (position + LOOKAHEAD_LIMIT)); p ++) if (data[p]) break; - return (struct command) {.command = 3, .count = p - position}; - } -} - -struct command pick_copy_for_pass (const unsigned char * data, const unsigned char * reference, const short * sources, unsigned char command_type, - unsigned short length, unsigned short position, unsigned flags) { - struct command result = {.command = 7, .count = (flags & 4) ? 4 : 3}; - if (length < 3) return result; - unsigned refpos, count; - const unsigned char * current; - unsigned char buffer[6] = {0}; - memcpy(buffer, reference + length - 3, 3); - while (*sources >= 0) { - refpos = *(sources ++); - if (command_type == 6) refpos = length - 1 - refpos; - if (refpos >= (length - 3)) - current = buffer + refpos - (length - 3); - else - current = reference + refpos; - if (memcmp(data + position, current, ((position + 4) > length) ? length - position : 4)) continue; - for (count = 4; (count < (length - position)) && (count < (length - refpos)); count ++) if (data[position + count] != current[count]) break; - if (count > (length - refpos)) count = length - refpos; - if (count > (length - position)) count = length - position; - if (result.count > count) continue; - result = (struct command) {.command = command_type, .count = count, .value = sources[-1]}; - } - return result; -} diff --git a/tools/lz/nullcomp.c b/tools/lz/nullcomp.c deleted file mode 100644 index d4535bd3..00000000 --- a/tools/lz/nullcomp.c +++ /dev/null @@ -1,20 +0,0 @@ -#include "proto.h" - -/* - Null compressor: stores data uncompressed, using literal (0) commands only. - Methods defined: 2 - Flags values: 0 = split a trailing 33-to-64-byte block at the end into two short blocks; 1 = don't -*/ - -struct command * store_uncompressed (__attribute__((unused)) const unsigned char * data, __attribute__((unused)) const unsigned char * bitflipped, unsigned short * size, unsigned flags) { - unsigned short position, block, remainder = *size; - struct command * result = NULL; - *size = 0; - for (position = 0; remainder; position += block, remainder -= block) { - block = (remainder > MAX_COMMAND_COUNT) ? MAX_COMMAND_COUNT : remainder; - if (!(flags & 1) && (block <= (2 * SHORT_COMMAND_COUNT)) && (block > SHORT_COMMAND_COUNT)) block = SHORT_COMMAND_COUNT; - result = realloc(result, sizeof(struct command) * (1 + *size)); - result[(*size) ++] = (struct command) {.command = 0, .count = block, .value = position}; - } - return result; -} diff --git a/tools/lz/options.c b/tools/lz/options.c deleted file mode 100644 index 8e5d05a7..00000000 --- a/tools/lz/options.c +++ /dev/null @@ -1,141 +0,0 @@ -#include "proto.h" - -struct options get_options (int argc, char ** argv) { - struct options result = {.input = NULL, .output = NULL, .mode = 0, .alignment = 0, .method = COMPRESSION_METHODS}; - const char * program_name = *argv; - int compressor = -1; - if (argc == 1) usage(program_name); - for (argv ++; *argv; argv ++) { - if (**argv != '-') break; - if (!1[*argv]) break; - if (!strcmp(*argv, "--")) { - argv ++; - break; - } else if (!(strcmp(*argv, "--text") && strcmp(*argv, "-t"))) - result.mode = 1; - else if (!(strcmp(*argv, "--binary") && strcmp(*argv, "-b"))) - result.mode = 0; - else if (!(strcmp(*argv, "--uncompress") && strcmp(*argv, "-u"))) - result.mode = 2; - else if (!(strcmp(*argv, "--dump") && strcmp(*argv, "-d"))) - result.mode = 3; - else if (!(strcmp(*argv, "--align") && strncmp(*argv, "-a", 2))) - result.alignment = parse_numeric_option_argument(&argv, 12); - else if (!(strcmp(*argv, "--method") && strncmp(*argv, "-m", 2))) - result.method = parse_numeric_option_argument(&argv, COMPRESSION_METHODS - 1); - else if (!(strcmp(*argv, "--compressor") && strncmp(*argv, "-c", 2))) - compressor = parse_compressor_option_argument(&argv); - else if (!(strcmp(*argv, "--optimize") && strcmp(*argv, "-o"))) { - result.method = COMPRESSION_METHODS; - compressor = -1; - } else if (!(strcmp(*argv, "--help") && strcmp(*argv, "-?"))) - usage(program_name); - else if (!(strcmp(*argv, "--list") && strcmp(*argv, "-l"))) - list_compressors(); - else - error_exit(3, "unknown option: %s", *argv); - } - if (compressor >= 0) { - if (result.method >= COMPRESSION_METHODS) result.method = 0; - if (result.method >= compressors[compressor].methods) - error_exit(3, "method for the %s compressor must be between 0 and %u", compressors[compressor].name, compressors[compressor].methods - 1); - while (compressor > 0) result.method += compressors[-- compressor].methods; - } - if (*argv) { - if (strcmp(*argv, "-")) result.input = *argv; - if (*(++ argv)) { - if (argv[1]) error_exit(3, "too many command-line arguments"); - if (strcmp(*argv, "-")) result.output = *argv; - } - } - return result; -} - -unsigned parse_numeric_option_argument (char *** alp, unsigned limit) { - const char * option; - const char * value = get_argument_for_option(alp, &option); - char * error; - unsigned long result = strtoul(value, &error, 10); - if (*error) error_exit(3, "invalid argument to option %s", option); - if (result > limit) error_exit(3, "argument to option %s must be between 0 and %u", option, limit); - return result; -} - -int parse_compressor_option_argument (char *** alp) { - const char * name = get_argument_for_option(alp, NULL); - if (!strcmp(name, "*")) return -1; - int result = -1; - unsigned length = strlen(name); - const struct compressor * compressor; - for (compressor = compressors; compressor -> name; compressor ++) { - if (strncmp(name, compressor -> name, length)) continue; - if (result >= 0) error_exit(3, "ambiguous compressor prefix: %s", name); - result = compressor - compressors; - } - if (result < 0) error_exit(3, "unknown compressor: %s", name); - return result; -} - -const char * get_argument_for_option (char *** alp, const char ** option_name) { - // alp: argument list pointer (i.e., address of the current value of argv after indexing) - // will point at the last consumed argument on exit (since the caller will probably increment it once more) - const char * option; - const char * result; - if (1[**alp] == '-') { - option = *((*alp) ++); - result = **alp; - } else { - option_name_buffer[1] = 1[**alp]; - option = option_name_buffer; - result = **alp + 2; - } - if (!(result && *result)) error_exit(3, "option %s requires an argument", option); - if (option_name) *option_name = option; - return result; -} - -noreturn usage (const char * program_name) { - fprintf(stderr, "Usage: %s [] [ []]\n\n", program_name); - fputs("Execution mode:\n", stderr); - fputs(" -b, --binary Output the command stream as binary data (default).\n", stderr); - fputs(" -t, --text Output the command stream as text.\n", stderr); - fputs(" -u, --uncompress Process a compressed file and output the original data.\n", stderr); - fputs(" -d, --dump Process a compressed file and dump the command stream as\n", stderr); - fputs(" text (as if compressed with the --text option).\n", stderr); - fputs(" -l, --list List compressors and their method numbers.\n", stderr); - fputs(" -?, --help Print this help text and exit.\n", stderr); - fputs("Compression options:\n", stderr); - fputs(" -o, --optimize Use the best combination of compression\n", stderr); - fputs(" methods available (default).\n", stderr); - fputs(" -m, --method Use only one specific compression method.\n", stderr); - fprintf(stderr, " Valid method numbers are between 0 and %u.\n", COMPRESSION_METHODS - 1); - fputs(" -c, --compressor Use the specified compressor: the method\n", stderr); - fputs(" number will be relative to that compressor.\n", stderr); - fputs(" Any prefix of the compressor name may be\n", stderr); - fputs(" specified. Use * to indicate any compressor.\n", stderr); - fputs(" -a, --align Pad the compressed output with zeros until\n", stderr); - fputs(" the size has the specified number of low bits\n", stderr); - fputs(" cleared (default: 0).\n", stderr); - fputs("The source and output filenames can be given as - (or omitted) to use standard\n", stderr); - fputs("input and output. Use -- to indicate that subsequent arguments are file names.\n", stderr); - exit(3); -} - -noreturn list_compressors (void) { - const struct compressor * compressor; - unsigned current, length = 10; - for (compressor = compressors; compressor -> name; compressor ++) if ((current = strlen(compressor -> name)) > length) length = current; - fprintf(stderr, "%-*s Offset Methods\n", length, "Compressor"); - for (current = 0; current < length; current ++) putc('-', stderr); - fputs(" ------ -------\n", stderr); - current = 0; - for (compressor = compressors; compressor -> name; compressor ++) { - fprintf(stderr, "%-*s %6u %7u\n", length, compressor -> name, current, compressor -> methods); - current += compressor -> methods; - } - putc('\n', stderr); - fputs("Note: the offset indicates the compressor's lowest method number when the\n", stderr); - fputs("--compressor option is not given. When that option is used, every compressor's\n", stderr); - fputs("methods are numbered from zero.\n", stderr); - exit(3); -} diff --git a/tools/lz/output.c b/tools/lz/output.c deleted file mode 100644 index 484a9516..00000000 --- a/tools/lz/output.c +++ /dev/null @@ -1,146 +0,0 @@ -#include "proto.h" - -void write_commands_to_textfile (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream, - unsigned char alignment) { - FILE * fp = file ? fopen(file, "w") : stdout; - if (!fp) error_exit(1, "could not open file %s for writing", file); - unsigned length = 0; - while (count --) { - write_command_to_textfile(fp, *commands, input_stream); - length += command_size(*(commands ++)); - } - if (fputs("\tlzend\n", fp) < 0) error_exit(1, "could not write terminator to compressed output"); - length = ~length & ((1 << alignment) - 1); - if (length --) { - int rv = fputs("\tdb 0", fp); - while ((rv >= 0) && length --) rv = fputs(", 0", fp); - if (rv >= 0) rv = -(putc('\n', fp) == EOF); - if (rv < 0) error_exit(1, "could not write padding to compressed output"); - } - if (file) fclose(fp); -} - -void write_commands_and_padding_to_textfile (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream, - unsigned padding_offset, unsigned padding_size) { - FILE * fp = file ? fopen(file, "w") : stdout; - if (!fp) error_exit(1, "could not open file %s for writing", file); - while (count --) write_command_to_textfile(fp, *(commands ++), input_stream); - if (fputs("\tlzend\n", fp) < 0) error_exit(1, "could not write terminator to compressed output"); - if (padding_size) { - input_stream += padding_offset; - int rv = 0; - unsigned pos; - const char * prefix = "\tdb"; - for (pos = 0; (rv >= 0) && (pos < padding_size); pos ++) { - if (input_stream[pos]) - rv = fprintf(fp, "%s $%02hhx", prefix, input_stream[pos]); - else - rv = fprintf(fp, "%s 0", prefix); - prefix = ","; - } - if (rv >= 0) rv = -(putc('\n', fp) == EOF); - if (rv < 0) error_exit(1, "could not write padding to compressed output"); - } - if (file) fclose(fp); -} - -void write_command_to_textfile (FILE * fp, struct command command, const unsigned char * input_stream) { - if ((!command.count) || (command.count > MAX_COMMAND_COUNT)) error_exit(2, "invalid command in output stream"); - int rv, pos; - const char * kind; - switch (command.command) { - case 0: - if ((rv = fprintf(fp, "\tlzdata")) < 0) break; - for (pos = 0; pos < command.count; pos ++) if ((rv = fprintf(fp, "%s$%02hhx", pos ? ", " : " ", input_stream[command.value + pos])) < 0) break; - rv = putc('\n', fp); - break; - case 1: - if ((command.value < 0) || (command.value > 255)) error_exit(2, "invalid command in output stream"); - rv = fprintf(fp, "\tlzrepeat %u, $%02hhx\n", command.count, (unsigned char) command.value); - break; - case 2: - if (command.value < 0) error_exit(2, "invalid command in output stream"); - rv = fprintf(fp, "\tlzrepeat %u, $%02hhx, $%02hhx\n", command.count, (unsigned char) command.value, (unsigned char) (command.value >> 8)); - break; - case 3: - rv = fprintf(fp, "\tlzzero %u\n", command.count); - break; - case 4: - kind = "normal"; - goto copy; - case 5: - kind = "flipped"; - goto copy; - case 6: - kind = "reversed"; - copy: - if ((command.value < -LOOKBACK_LIMIT) || (command.value >= MAX_FILE_SIZE)) error_exit(2, "invalid command in output stream"); - if (command.value < 0) - rv = fprintf(fp, "\tlzcopy %s, %u, %d\n", kind, command.count, command.value); - else - rv = fprintf(fp, "\tlzcopy %s, %u, $%04hx\n", kind, command.count, (unsigned short) command.value); - break; - default: - error_exit(2, "invalid command in output stream"); - } - if (rv < 0) error_exit(1, "could not write command to compressed output"); -} - -void write_commands_to_file (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream, unsigned char alignment) { - FILE * fp = file ? fopen(file, "wb") : stdout; - if (!fp) error_exit(1, "could not open file %s for writing", file); - unsigned length = 0; - while (count --) { - write_command_to_file(fp, *commands, input_stream); - length += command_size(*(commands ++)); - } - if (putc(-1, fp) == EOF) error_exit(1, "could not write terminator to compressed output"); - length = ~length & ((1 << alignment) - 1); - while (length --) if (putc(0, fp) == EOF) error_exit(1, "could not write padding to compressed output"); - if (file) fclose(fp); -} - -void write_command_to_file (FILE * fp, struct command command, const unsigned char * input_stream) { - if ((!command.count) || (command.count > MAX_COMMAND_COUNT)) error_exit(2, "invalid command in output stream"); - unsigned char buf[4]; - unsigned char * pos = buf; - int n; - command.count --; - if (command.count < SHORT_COMMAND_COUNT) - *(pos ++) = (command.command << 5) + command.count; - else { - *(pos ++) = 224 + (command.command << 2) + (command.count >> 8); - *(pos ++) = command.count; - } - switch (command.command) { - case 1: case 2: - if ((command.value < 0) || (command.value >= (1 << (command.command << 3)))) error_exit(2, "invalid command in output stream"); - for (n = 0; n < command.command; n ++) *(pos ++) = command.value >> (n << 3); - case 0: case 3: - break; - default: - if ((command.value < -LOOKBACK_LIMIT) || (command.value >= MAX_FILE_SIZE)) error_exit(2, "invalid command in output stream"); - if (command.value < 0) - *(pos ++) = command.value ^ 127; - else { - *(pos ++) = command.value >> 8; - *(pos ++) = command.value; - } - } - if (fwrite(buf, 1, pos - buf, fp) != (pos - buf)) error_exit(1, "could not write command to compressed output"); - if (command.command) return; - command.count ++; - if (fwrite(input_stream + command.value, 1, command.count, fp) != command.count) error_exit(1, "could not write data to compressed output"); -} - -void write_raw_data_to_file (const char * file, const void * data, unsigned length) { - FILE * fp = file ? fopen(file, "w") : stdout; - if (!fp) error_exit(1, "could not open file %s for writing", file); - while (length) { - unsigned rv = fwrite(data, 1, length, fp); - if (!rv) error_exit(1, "could not write raw data to output"); - data = (const char *) data + rv; - length -= rv; - } - if (file) fclose(fp); -} diff --git a/tools/lz/packing.c b/tools/lz/packing.c deleted file mode 100644 index 0cb9fae9..00000000 --- a/tools/lz/packing.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "proto.h" - -void optimize (struct command * commands, unsigned short count) { - while (count && (commands -> command == 7)) commands ++, count --; - if (count < 2) return; - struct command * end = commands + count; - struct command * next; - for (next = commands + 1; next < end; next ++) { - if (next -> command == 7) continue; - if ( - !(commands -> command) && - (command_size(*next) == next -> count) && - ((commands -> count + next -> count) <= MAX_COMMAND_COUNT) && - ((commands -> count > SHORT_COMMAND_COUNT) || ((commands -> count + next -> count) <= SHORT_COMMAND_COUNT)) - ) { - commands -> count += next -> count; - next -> command = 7; - continue; - } - if (next -> command == commands -> command) - switch (commands -> command) { - case 0: - if ((commands -> value + commands -> count) != next -> value) break; - commands -> count += next -> count; - next -> command = 7; - if (commands -> count <= MAX_COMMAND_COUNT) continue; - next -> command = 0; - next -> value = commands -> value + MAX_COMMAND_COUNT; - next -> count = commands -> count - MAX_COMMAND_COUNT; - commands -> count = MAX_COMMAND_COUNT; - break; - case 1: - if (commands -> value != next -> value) break; - // fallthrough - case 3: - if ((commands -> count + next -> count) <= MAX_COMMAND_COUNT) { - commands -> count += next -> count; - next -> command = 7; - continue; - } - next -> count = (commands -> count + next -> count) - MAX_COMMAND_COUNT; - commands -> count = MAX_COMMAND_COUNT; - } - commands = next; - } -} - -void repack (struct command ** commands, unsigned short * length) { - struct command * new_commands = malloc(sizeof(struct command) * *length); - struct command * current = new_commands; - unsigned short p; - for (p = 0; p < *length; p ++) if (p[*commands].command != 7) *(current ++) = p[*commands]; - free(*commands); - *commands = new_commands; - *length = current - new_commands; -} diff --git a/tools/lz/proto.h b/tools/lz/proto.h deleted file mode 100644 index 84577422..00000000 --- a/tools/lz/proto.h +++ /dev/null @@ -1,107 +0,0 @@ -#include -#include -#include -#include - -#define NUM_COMPRESSORS 4 -#define COMPRESSION_METHODS 96 /* sum of all values for the methods field in compressors */ -#define MAX_FILE_SIZE 32768 -#define SHORT_COMMAND_COUNT 32 -#define MAX_COMMAND_COUNT 1024 -#define LOOKBACK_LIMIT 128 /* highest negative valid count for a copy command */ -#define LOOKAHEAD_LIMIT 3072 /* maximum lookahead distance for the first pass of multi-pass compression */ -#define MULTIPASS_SKIP_THRESHOLD 64 - -#if __STDC_VERSION__ >= 201112L - // forces "noreturn void", which is silly and redundant; this is simpler - #define noreturn _Noreturn void -#else - #define noreturn void /* fallback */ -#endif - -struct command { - unsigned command: 3; // commands 0-6 as per compression spec; command 7 is used as a dummy placeholder - unsigned count: 12; // always equals the uncompressed data length - signed value: 17; // offset for commands 0 (into source) and 4-6 (into decompressed output); repeated bytes for commands 1-2 -}; - -struct compressor { - unsigned methods; - const char * name; - struct command * (* function) (const unsigned char *, const unsigned char *, unsigned short *, unsigned); -}; - -struct options { - const char * input; - const char * output; - unsigned method; // method to use, or >= COMPRESSION_METHODS to try them all - unsigned char mode; // 0: compress, 1: compress to text, 2: uncompress, 3: dump commands as text - unsigned char alignment; // 1 << value -}; - -// global.c -extern const struct compressor compressors[]; -extern const unsigned char bit_flipping_table[]; -extern char option_name_buffer[]; - -// main.c -int main(int, char **); -struct command * compress(const unsigned char *, unsigned short *, unsigned); - -// merging.c -struct command * select_optimal_sequence(struct command **, const unsigned short *, unsigned short *); -struct command * select_command_sequence(struct command **, const unsigned short *, int, unsigned short *); -struct command * merge_command_sequences(const struct command *, unsigned short, const struct command *, unsigned short, unsigned short *); - -// mpcomp.c -struct command * try_compress_multi_pass(const unsigned char *, const unsigned char *, unsigned short *, unsigned); -struct command pick_command_for_pass(const unsigned char *, const unsigned char *, const unsigned char *, const short *, unsigned short, - unsigned short, unsigned); -struct command pick_repetition_for_pass(const unsigned char *, unsigned short, unsigned short, unsigned); -struct command pick_copy_for_pass(const unsigned char *, const unsigned char *, const short *, unsigned char, unsigned short, unsigned short, unsigned); - -// nullcomp.c -struct command * store_uncompressed(const unsigned char *, const unsigned char *, unsigned short *, unsigned); - -// options.c -struct options get_options(int, char **); -unsigned parse_numeric_option_argument(char ***, unsigned); -int parse_compressor_option_argument(char ***); -const char * get_argument_for_option(char ***, const char **); -noreturn usage(const char *); -noreturn list_compressors(void); - -// output.c -void write_commands_to_textfile(const char *, const struct command *, unsigned, const unsigned char *, unsigned char); -void write_commands_and_padding_to_textfile(const char *, const struct command *, unsigned, const unsigned char *, unsigned, unsigned); -void write_command_to_textfile(FILE *, struct command, const unsigned char *); -void write_commands_to_file(const char *, const struct command *, unsigned, const unsigned char *, unsigned char); -void write_command_to_file(FILE *, struct command, const unsigned char *); -void write_raw_data_to_file(const char *, const void *, unsigned); - -// packing.c -void optimize(struct command *, unsigned short); -void repack(struct command **, unsigned short *); - -// repcomp.c -struct command * try_compress_repetitions(const unsigned char *, const unsigned char *, unsigned short *, unsigned); -struct command find_repetition_at_position(const unsigned char *, unsigned short, unsigned short); - -// spcomp.c -struct command * try_compress_single_pass(const unsigned char *, const unsigned char *, unsigned short *, unsigned); -struct command find_best_copy(const unsigned char *, unsigned short, unsigned short, const unsigned char *, unsigned); -unsigned short scan_forwards(const unsigned char *, unsigned short, const unsigned char *, unsigned short, short *); -unsigned short scan_backwards(const unsigned char *, unsigned short, unsigned short, short *); -struct command find_best_repetition(const unsigned char *, unsigned short, unsigned short); - -// uncomp.c -struct command * get_commands_from_file(const unsigned char *, unsigned short * restrict, unsigned short * restrict); -unsigned char * get_uncompressed_data(const struct command *, const unsigned char *, unsigned short *); - -// util.c -noreturn error_exit(int, const char *, ...); -unsigned char * read_file_into_buffer(const char *, unsigned short *); -struct command pick_best_command(unsigned, struct command, ...); -int is_better(struct command, struct command); -short command_size(struct command); -unsigned short compressed_length(const struct command *, unsigned short); diff --git a/tools/lz/repcomp.c b/tools/lz/repcomp.c deleted file mode 100644 index f2bbad8a..00000000 --- a/tools/lz/repcomp.c +++ /dev/null @@ -1,63 +0,0 @@ -#include "proto.h" - -/* - Repetitions compressor: compresses the data only using a subset of the available repetition commands. - Methods defined: 6 - Flags values: the value plus one is taken as a bitfield indicating which kinds of repetition commands are used - (lowest bit to highest: repeat single byte (1), repeat two bytes (2), repeat zeros (3)). -*/ - -struct command * try_compress_repetitions (const unsigned char * data, __attribute__((unused)) const unsigned char * bitflipped, unsigned short * size, unsigned flags) { - unsigned short pos = 0, skipped = 0; - struct command * result = malloc(*size * sizeof(struct command)); - struct command * current = result; - struct command candidate; - flags = (flags + 1) << 1; - while (pos < *size) { - candidate = find_repetition_at_position(data, pos, *size); - if ((candidate.command == 3) && !(flags & 8)) { - candidate.command = 1; - candidate.value = 0; - } - if ((candidate.command == 1) && !(flags & 2)) { - candidate.command = 2; - candidate.value |= candidate.value << 8; - } - if ((flags & (1 << candidate.command)) && (command_size(candidate) <= candidate.count)) { - if (skipped) *(current ++) = (struct command) {.command = 0, .count = skipped, .value = pos - skipped}; - skipped = 0; - *(current ++) = candidate; - pos += candidate.count; - } else { - pos ++; - if ((++ skipped) == MAX_COMMAND_COUNT) { - *(current ++) = (struct command) {.command = 0, .count = MAX_COMMAND_COUNT, .value = pos - MAX_COMMAND_COUNT}; - skipped = 0; - } - } - } - if (skipped) *(current ++) = (struct command) {.command = 0, .count = skipped, .value = pos - skipped}; - *size = current - result; - result = realloc(result, *size * sizeof(struct command)); - return result; -} - -struct command find_repetition_at_position (const unsigned char * data, unsigned short position, unsigned short length) { - if ((position + 1) >= length) return data[position] ? ((struct command) {.command = 7}) : ((struct command) {.command = 3, .count = 1}); - unsigned char value[2] = {data[position], data[position + 1]}; - unsigned repcount, limit = length - position; - if (limit > MAX_COMMAND_COUNT) limit = MAX_COMMAND_COUNT; - for (repcount = 2; (repcount < limit) && (data[position + repcount] == value[repcount & 1]); repcount ++); - struct command result; - result.count = repcount; - if (*value != value[1]) { - if (!*value && (repcount < 3)) return (struct command) {.command = 3, .count = 1}; - result.command = 2; - result.value = ((unsigned) (*value)) | (((unsigned) (value[1])) << 8); - } else if (*value) { - result.command = 1; - result.value = *value; - } else - result.command = 3; - return result; -} diff --git a/tools/lz/spcomp.c b/tools/lz/spcomp.c deleted file mode 100644 index b6184836..00000000 --- a/tools/lz/spcomp.c +++ /dev/null @@ -1,141 +0,0 @@ -#include "proto.h" - -/* - Single-pass compressor: attempts to compress the data in a single pass, selecting the best command at each - position within some constraints. - Methods defined: 72 - Flags values: - Bit fields (will trigger alternate behavior if set): - 1: prefer repetition commands over copy commands of equal count - 2: don't emit a copy or repetition with a count equal to its size when the previous command is a literal (0) that - is not at maximum size (32 or 1024) - 4: don't emit long copy commands - Selector values (pick one from each group and add them to the bit fields): - - Scan delay: number of bytes that are forced into literal (0) commands after each non-literal command: - 0: 0 bytes - 8: 1 byte - 16: 2 bytes - - Copy command preference (when the command counts are tied), in order from most to least: - 0: normal (4), reversed (6), flipped (5) - 24: reversed (6), flipped (5), normal (4) - 48: flipped (5), reversed (6), normal (4) -*/ - -struct command * try_compress_single_pass (const unsigned char * data, const unsigned char * bitflipped, unsigned short * length, unsigned flags) { - struct command * commands = malloc(sizeof(struct command) * *length); - memset(commands, -1, sizeof(struct command) * *length); - struct command * current_command = commands; - unsigned short position = 0, previous_data = 0; - unsigned char scan_delay = 0, scan_delay_flag = (flags >> 3) % 3; - struct command copy, repetition; - while (position < *length) { - copy = find_best_copy(data, position, *length, bitflipped, flags); - repetition = find_best_repetition(data, position, *length); - if (flags & 1) - *current_command = pick_best_command(2, repetition, copy); - else - *current_command = pick_best_command(2, copy, repetition); - *current_command = pick_best_command(2, (struct command) {.command = 0, .count = 1, .value = position}, *current_command); - if ((flags & 2) && (command_size(*current_command) == current_command -> count)) - if (previous_data && (previous_data != SHORT_COMMAND_COUNT) && (previous_data != MAX_COMMAND_COUNT)) - *current_command = (struct command) {.command = 0, .count = 1, .value = position}; - if (scan_delay_flag) { - if (scan_delay >= scan_delay_flag) - scan_delay = 0; - else if (current_command -> command) { - scan_delay ++; - *current_command = (struct command) {.command = 0, .count = 1, .value = position}; - } - } - if (current_command -> command) - previous_data = 0; - else - previous_data += current_command -> count; - position += (current_command ++) -> count; - } - optimize(commands, current_command - commands); - repack(&commands, length); - return commands; -} - -struct command find_best_copy (const unsigned char * data, unsigned short position, unsigned short length, const unsigned char * bitflipped, unsigned flags) { - struct command simple = {.command = 7}; - struct command flipped = simple, backwards = simple; - short count, offset; - if ((count = scan_forwards(data + position, length - position, data, position, &offset))) - simple = (struct command) {.command = 4, .count = count, .value = offset}; - if ((count = scan_forwards(data + position, length - position, bitflipped, position, &offset))) - flipped = (struct command) {.command = 5, .count = count, .value = offset}; - if ((count = scan_backwards(data, length - position, position, &offset))) - backwards = (struct command) {.command = 6, .count = count, .value = offset}; - struct command command; - switch (flags / 24) { - case 0: command = pick_best_command(3, simple, backwards, flipped); break; - case 1: command = pick_best_command(3, backwards, flipped, simple); break; - case 2: command = pick_best_command(3, flipped, backwards, simple); - } - if ((flags & 4) && (command.count > SHORT_COMMAND_COUNT)) command.count = SHORT_COMMAND_COUNT; - return command; -} - -unsigned short scan_forwards (const unsigned char * target, unsigned short limit, const unsigned char * source, unsigned short real_position, short * offset) { - unsigned short best_match, best_length = 0; - unsigned short current_length; - unsigned short position; - for (position = 0; position < real_position; position ++) { - if (source[position] != *target) continue; - for (current_length = 0; (current_length < limit) && (source[position + current_length] == target[current_length]); current_length ++); - if (current_length > MAX_COMMAND_COUNT) current_length = MAX_COMMAND_COUNT; - if (current_length < best_length) continue; - best_match = position; - best_length = current_length; - } - if (!best_length) return 0; - if ((best_match + LOOKBACK_LIMIT) >= real_position) - *offset = best_match - real_position; - else - *offset = best_match; - return best_length; -} - -unsigned short scan_backwards (const unsigned char * data, unsigned short limit, unsigned short real_position, short * offset) { - if (real_position < limit) limit = real_position; - unsigned short best_match, best_length = 0; - unsigned short current_length; - unsigned short position; - for (position = 0; position < real_position; position ++) { - if (data[position] != data[real_position]) continue; - for (current_length = 0; (current_length <= position) && (current_length < limit) && - (data[position - current_length] == data[real_position + current_length]); current_length ++); - if (current_length > MAX_COMMAND_COUNT) current_length = MAX_COMMAND_COUNT; - if (current_length < best_length) continue; - best_match = position; - best_length = current_length; - } - if (!best_length) return 0; - if ((best_match + LOOKBACK_LIMIT) >= real_position) - *offset = best_match - real_position; - else - *offset = best_match; - return best_length; -} - -struct command find_best_repetition (const unsigned char * data, unsigned short position, unsigned short length) { - if ((position + 1) >= length) return data[position] ? ((struct command) {.command = 7}) : ((struct command) {.command = 3, .count = 1}); - unsigned char value[2] = {data[position], data[position + 1]}; - unsigned repcount, limit = length - position; - if (limit > MAX_COMMAND_COUNT) limit = MAX_COMMAND_COUNT; - for (repcount = 2; (repcount < limit) && (data[position + repcount] == value[repcount & 1]); repcount ++); - struct command result; - result.count = repcount; - if (*value != value[1]) { - if (!*value && (repcount < 3)) return (struct command) {.command = 3, .count = 1}; - result.command = 2; - result.value = ((unsigned) (*value)) | (((unsigned) (value[1])) << 8); - } else if (*value) { - result.command = 1; - result.value = *value; - } else - result.command = 3; - return result; -} diff --git a/tools/lz/uncomp.c b/tools/lz/uncomp.c deleted file mode 100644 index b22fc75f..00000000 --- a/tools/lz/uncomp.c +++ /dev/null @@ -1,92 +0,0 @@ -#include "proto.h" - -struct command * get_commands_from_file (const unsigned char * data, unsigned short * restrict size, unsigned short * restrict slack) { - struct command * result = malloc(*size * sizeof(struct command)); - unsigned short remaining = *size; - struct command * current = result; - const unsigned char * rp = data; - while (1) { - if (!(remaining --)) goto error; - current -> command = *rp >> 5; - current -> count = *(rp ++) & 31; - if (current -> command == 7) { - current -> command = current -> count >> 2; - current -> count = (current -> count & 3) << 8; - if (current -> command == 7) { - // long commands inside long commands are not allowed, but if the count is 0x300 here, it means that the original byte was 0xff - if (current -> count == 0x300) break; - goto error; - } - if (!(remaining --)) goto error; - current -> count |= *(rp ++); - } - current -> count ++; - switch (current -> command) { - case 0: - if (remaining <= current -> count) goto error; - current -> value = rp - data; - rp += current -> count; - remaining -= current -> count; - case 3: - break; - case 1: case 2: { - unsigned char p; - if (remaining <= current -> command) goto error; - current -> value = 0; - for (p = 0; p < current -> command; p ++) current -> value |= *(rp ++) << (p << 3); - remaining -= current -> command; - } break; - default: - if (!(remaining --)) goto error; - if ((current -> value = *(rp ++)) & 128) - current -> value = 127 - current -> value; - else { - if (!(remaining --)) goto error; - current -> value = (current -> value << 8) | *(rp ++); - } - } - current ++; - } - if (slack) *slack = *size - (rp - data); - *size = current - result; - return realloc(result, (*size ? *size : 1) * sizeof(struct command)); - error: - free(result); - return NULL; -} - -unsigned char * get_uncompressed_data (const struct command * commands, const unsigned char * compressed, unsigned short * size) { - const struct command * limit = commands + *size; - unsigned char * result = malloc(MAX_FILE_SIZE + MAX_COMMAND_COUNT); - unsigned char * current = result; - unsigned short p; - for (; commands < limit; commands ++) { - switch (commands -> command) { - case 0: - memcpy(current, compressed + commands -> value, commands -> count); - current += commands -> count; - break; - case 1: case 2: - for (p = 0; p < commands -> count; p ++) *(current ++) = commands -> value >> ((p % commands -> command) << 3); - break; - case 3: - memset(current, 0, commands -> count); - current += commands -> count; - break; - default: { - const unsigned char * ref = ((commands -> value < 0) ? current : result) + commands -> value; - for (p = 0; p < commands -> count; p ++) { - current[p] = ref[(commands -> command == 6) ? -(int) p : p]; - if (commands -> command == 5) current[p] = bit_flipping_table[current[p]]; - } - current += commands -> count; - } - } - if ((current - result) > MAX_FILE_SIZE) { - free(result); - return NULL; - } - } - *size = current - result; - return realloc(result, *size ? *size : 1); -} diff --git a/tools/lz/util.c b/tools/lz/util.c deleted file mode 100644 index d77ac7bc..00000000 --- a/tools/lz/util.c +++ /dev/null @@ -1,54 +0,0 @@ -#include "proto.h" - -noreturn error_exit (int error_code, const char * error, ...) { - va_list ap; - va_start(ap, error); - fputs("error: ", stderr); - vfprintf(stderr, error, ap); - va_end(ap); - fputc('\n', stderr); - exit(error_code); -} - -unsigned char * read_file_into_buffer (const char * file, unsigned short * size) { - FILE * fp = file ? fopen(file, "rb") : stdin; - if (!fp) error_exit(1, "could not open file %s for reading", file); - unsigned char * buf = malloc(MAX_FILE_SIZE + 1); - int rv = fread(buf, 1, MAX_FILE_SIZE + 1, fp); - if (file) fclose(fp); - if (rv < 0) error_exit(1, "could not read from file %s", file); - if (rv > MAX_FILE_SIZE) error_exit(1, "file %s is too big", file ? file : ""); - *size = rv; - return buf; -} - -struct command pick_best_command (unsigned count, struct command command, ...) { - struct command result = command; - va_list ap; - va_start(ap, command); - while (-- count) { - command = va_arg(ap, struct command); - if (is_better(command, result)) result = command; - } - va_end(ap); - return result; -} - -int is_better (struct command new, struct command old) { - if (new.command == 7) return 0; - if (old.command == 7) return 1; - short new_savings = new.count - command_size(new), old_savings = old.count - command_size(old); - return new_savings > old_savings; -} - -short command_size (struct command command) { - short header_size = 1 + (command.count > SHORT_COMMAND_COUNT); - if (command.command & 4) return header_size + 1 + (command.value >= 0); - return header_size + command.command[(short []) {command.count, 1, 2, 0}]; -} - -unsigned short compressed_length (const struct command * commands, unsigned short count) { - unsigned short current, total = 0; - for (current = 0; current < count; current ++) if (commands[current].command != 7) total += command_size(commands[current]); - return total; -} diff --git a/tools/lzcompress.c b/tools/lzcompress.c new file mode 100644 index 00000000..97f8cfff --- /dev/null +++ b/tools/lzcompress.c @@ -0,0 +1,841 @@ +#define PROGRAM_NAME "lzcompress" +#define USAGE_OPTS "[-h|--help] [-u|--uncompress] [-m|--matching [-bcdilnr]] [-a|--align alignment] infile outfile" + +#include "common.h" + +/******************************************** LZ format *******************************************/ + +#define SHORT_MAX_LENGTH (1 << 5) // Commands have a 5-bit length +#define LONG_MAX_LENGTH (1 << 10) // LZ_LONG extends length to 10 bits + +// Offsets for lookback commands are 7 bits; above that they use a 16-bit index +#define LOOKBACK_MAX_OFFSET (1 << 7) + +enum lz_command { + LZ_LITERAL = 0, // Read literal data for N bytes + // Sequence commands + LZ_ITERATE = 1, // Write the same byte for N bytes + LZ_ALTERNATE = 2, // Alternate two bytes for N bytes + LZ_ZERO = 3, // Write 0 for N bytes + // Lookback commands + LZ_REPEAT = 4, // Repeat N bytes from the offset + LZ_FLIP = 5, // Repeat N bit-flipped bytes + LZ_REVERSE = 6, // Repeat N bytes in reverse + // Extended commands + LZ_LONG = 7, // Use a new command with a long N + // Sentinel commands + LZ_END = 0xff, // Terminate compressed data +}; + +struct command { + enum lz_command cmd; // Picked compression command + unsigned int length; // Command length N + int offset; // Offset for lookback commands; bytes for sequence commands + uint8_t value; // Uncompressed byte value +}; + +/**************************************** Argument options ****************************************/ + +struct Options { + bool decompress; + bool matching; + uint8_t alignment; + bool skip_initial_byte; + bool no_lookback_3; + bool odd_alternate; + bool iterate_only; + bool literal_only; + bool long_32; + bool prefer_alternate; +}; + +struct Options options = {0}; + +void parse_args(int argc, char *argv[]) { + struct option long_options[] = { + {"uncompress", no_argument, 0, 'u'}, + {"matching", no_argument, 0, 'm'}, + {"align", required_argument, 0, 'a'}, + {"skip-initial-byte", no_argument, 0, 'b'}, + {"no-lookback-3", no_argument, 0, 'c'}, + {"odd-alternate", no_argument, 0, 'd'}, + {"iterate-only", no_argument, 0, 'i'}, + {"literal-only", no_argument, 0, 'l'}, + {"long-32", no_argument, 0, 'n'}, + {"prefer-alternate", no_argument, 0, 'r'}, + {"help", no_argument, 0, 'h'}, + {0} + }; + for (int opt; (opt = getopt_long(argc, argv, "mua:bcdilnrh", long_options)) != -1;) { + switch (opt) { + case 'u': + options.decompress = true; + break; + case 'm': + options.matching = true; + break; + case 'a': + options.alignment = strtoul(optarg, NULL, 0); + break; + case 'c': + options.no_lookback_3 = true; + break; + case 'd': + options.odd_alternate = true; + break; + case 'r': + options.prefer_alternate = true; + break; + case 'l': + options.literal_only = true; + break; + case 'i': + options.iterate_only = true; + break; + case 'n': + options.long_32 = true; + break; + case 'b': + options.skip_initial_byte = true; + break; + case 'h': + fprintf(stderr, "Usage: " PROGRAM_NAME " " USAGE_OPTS "\n\n"); + fputs("Flags to adjust --matching compression:\n", stderr); + fputs(" -a, --align N Alignment of the final compressed byte (2^N)\n", stderr); + fputs(" -b, --skip-initial-byte First byte will always use LZ_LITERAL\n", stderr); + fputs(" -c, --no-lookback-3 Don't use LZ_FLIP/LZ_REVERSE/LZ_LONG for length 3\n", stderr); + fputs(" -d, --odd-alternate Allow odd-length LZ_ALTERNATE\n", stderr); + fputs(" -i, --iterate-only Only use LZ_LITERAL and LZ_ITERATE\n", stderr); + fputs(" -l, --literal-only Only use LZ_LITERAL\n", stderr); + fputs(" -n, --long-32 Use LZ_LONG for length 32\n", stderr); + fputs(" -r, --prefer-alternate Use LZ_ALTERNATE instead of equally-large LZ_ITERATE\n", stderr); + exit(0); + break; + default: + usage_exit(1); + } + } +} + +/************************************** Matching compression **************************************/ + +// Above this length, assume the current command is optimal and jump to its end +#define SKIP_LOOKBACK_MIN_LENGTH (1 << 6) + +struct list { + unsigned int *items; + unsigned int size; + unsigned int capacity; +}; + +struct multimap { + uint32_t key; + struct list *values; + struct multimap *next; +}; + +struct multimap **multimap_find(struct multimap **mp, uint32_t key) { + while (*mp && (*mp)->key != key) { + mp = &(*mp)->next; + } + return mp; +} + +void multimap_put(struct multimap **mp, uint32_t key, unsigned int value) { + mp = multimap_find(mp, key); + if (!*mp) { + *mp = xcalloc(sizeof(**mp)); + (*mp)->key = key; + (*mp)->values = xcalloc(sizeof(*(*mp)->values)); + } + struct list *values = (*mp)->values; + if (values->size == values->capacity) { + values->capacity = (values->capacity + 1) * 2; + values->items = xrealloc(values->items, values->capacity * sizeof(*values->items)); + } + values->items[values->size++] = value; +} + +// Input and output data +struct command *commands = NULL; +unsigned int commands_size = 0; + +uint8_t iterate_f(unsigned int i, __attribute__((unused)) unsigned int len) { + return commands[i].value; +} + +uint8_t alternate_f(unsigned int i, unsigned int len) { + return commands[i + (len & 1)].value; +} + +uint8_t zero_f(__attribute__((unused)) unsigned int i, __attribute__((unused)) unsigned int len) { + return 0; +} + +uint8_t plain_f(uint8_t v) { + return v; +} + +uint8_t flip_f(uint8_t v) { + uint8_t f = 0; + for (unsigned int b = 0; b < 8; b++) { + f |= ((v >> b) & 1) << (7 - b); + } + return f; +} + +void try_sequence_command(enum lz_command cmd, unsigned int i, uint8_t (*f)(unsigned int, unsigned int)) { + unsigned int len; + for (len = 0; i + len < commands_size && commands[i + len].value == f(i, len); len++) {} + if (len > 0 && (cmd != LZ_ALTERNATE || commands[i].length < len + options.prefer_alternate)) { + commands[i].cmd = cmd; + commands[i].length = len; + } +} + +uint32_t data_key(unsigned int i, uint8_t (*cmd_fn)(uint8_t), int dir) { + uint32_t result = 0; + for (unsigned int j = 0; j < 4 && i < commands_size; j++, i += dir) { + result |= (uint32_t)cmd_fn(commands[i].value) << (uint32_t)(8 * j); + } + return result; +} + +unsigned int try_lookback_command( + enum lz_command cmd, + unsigned int i, + unsigned int rep_i, + uint32_t key, + struct multimap *rep_idxs, + uint8_t (*cmd_fn)(uint8_t), + int dir +) { + rep_idxs = *multimap_find(&rep_idxs, key); + struct list *idxs = rep_idxs ? rep_idxs->values : NULL; + + unsigned int len = 0; + for (unsigned int li = 0; idxs && li < idxs->size; li++) { + unsigned int prev_i = idxs->items[li]; + unsigned int rep_len = 0; + for (unsigned int j = i, rj = prev_i; + j < commands_size && rj < commands_size && commands[j].value == cmd_fn(commands[rj].value); + rep_len++, j++, rj += dir) {} + if (rep_len > len || (rep_len == len && prev_i > rep_i)) { + // This repetition is longer than the last one found, + // or it is the same length and closer + len = rep_len; + rep_i = prev_i; + } + } + if (len >= (unsigned int)(3 + options.no_lookback_3) && commands[i].length < len) { + commands[i].cmd = cmd; + commands[i].length = len; + commands[i].offset = i - rep_i - 1; + } + return rep_i; +} + +void pick_best_command(unsigned int i) { + if (options.literal_only) { + return; + } + + // Try LZ_ITERATE: find sequence of one byte + try_sequence_command(LZ_ITERATE, i, iterate_f); + + if (options.iterate_only) { + return; + } + + // Try LZ_ALTERNATE: find sequence of two alternating bytes + try_sequence_command(LZ_ALTERNATE, i, alternate_f); + + // Try LZ_ZERO: find sequence of 0 bytes + try_sequence_command(LZ_ZERO, i, zero_f); + + // Assume that a sequence command is optimal; don't try lookback commands + if (commands[i].length >= SKIP_LOOKBACK_MIN_LENGTH) { + return; + } + + unsigned int rep_i = 0; + + // These map a data key (the first four bytes of a data sequence) + // to a list of indexes where that data is found + static struct multimap *repeat_idxs = NULL, *reverse_idxs = NULL; + + // Try LZ_REPEAT: find repetition of previous data + uint32_t repeat_key = data_key(i, plain_f, 1); + rep_i = try_lookback_command(LZ_REPEAT, i, rep_i, repeat_key, repeat_idxs, plain_f, 1); + + // Try LZ_FLIP: find repetition of previous bit-flipped data + uint32_t flip_key = data_key(i, flip_f, 1); + rep_i = try_lookback_command(LZ_FLIP, i, rep_i, flip_key, repeat_idxs, flip_f, 1); + + // Try LZ_REVERSE: find repetition of previous reversed data + uint32_t reverse_key = data_key(i, plain_f, -1); + rep_i = try_lookback_command(LZ_REVERSE, i, rep_i, repeat_key, reverse_idxs, plain_f, -1); + + multimap_put(&repeat_idxs, repeat_key, i); + multimap_put(&reverse_idxs, reverse_key, i); +} + +unsigned int command_length(const struct command *command) { + switch (command->cmd) { + case LZ_LITERAL: + return 1; + case LZ_ITERATE: + return 2; + case LZ_ALTERNATE: + return 3; + case LZ_ZERO: + return 1; + case LZ_REPEAT: + case LZ_FLIP: + case LZ_REVERSE: + return 2 + (command->offset >= LOOKBACK_MAX_OFFSET); + default: + error_exit("invalid LZ command $%02x\n", command->cmd); + } +} + +void pick_optimized_commands(void) { + // Pick best commands at each index + for (unsigned int i = options.skip_initial_byte; i < commands_size; i++) { + pick_best_command(i); + if (commands[i].length >= SKIP_LOOKBACK_MIN_LENGTH) { + i += commands[i].length - 1; + } + } + + // Reduce lengths so one command won't overlap a subsequent more optimal command + for (unsigned int i = 0; i < commands_size; i++) { + for (unsigned int j = 1; j < commands[i].length; j++) { + if (commands[i + j].length > commands[i].length) { + commands[i].length = j - (!options.odd_alternate && commands[i].cmd == LZ_ALTERNATE && j % 2); + } + } + } + + // Don't use commands whose encoding would be longer than literal data + for (unsigned int i = 0; i < commands_size; i++) { + if (commands[i].length < command_length(&commands[i]) + !options.iterate_only) { + commands[i].cmd = LZ_LITERAL; + commands[i].length = 0; + } + } + + // Compute the lengths of LZ_LITERAL commands + for (unsigned int i = 0; i < commands_size; i += commands[i].length) { + if (commands[i].cmd == LZ_LITERAL) { + unsigned int len; + for (len = 1; i + len < commands_size && commands[i + len].cmd == LZ_LITERAL; len++); + commands[i].length = len; + } + } + + // Limit command lengths to LONG_MAX_LENGTH + for (unsigned int i = 0; i < commands_size; i += commands[i].length) { + if (commands[i].length > LONG_MAX_LENGTH) { + commands[i + LONG_MAX_LENGTH].cmd = commands[i].cmd; + commands[i + LONG_MAX_LENGTH].length = commands[i].length - LONG_MAX_LENGTH; + commands[i].length = LONG_MAX_LENGTH; + } + } +} + +void write_commands(FILE *file) { + unsigned int compressed_size = 0; + + for (unsigned int i = 0; i < commands_size; i += commands[i].length) { + if (commands[i].length + options.long_32 > SHORT_MAX_LENGTH) { + fputc((LZ_LONG << 5) | (commands[i].cmd << 2) | ((commands[i].length - 1) >> 8), file); + fputc((commands[i].length - 1) & 0xff, file); + compressed_size += 2; + } else { + fputc((commands[i].cmd << 5) | (commands[i].length - 1), file); + compressed_size++; + } + switch (commands[i].cmd) { + case LZ_LITERAL: + for (unsigned int j = 0; j < commands[i].length; j++) { + fputc(commands[i + j].value, file); + } + compressed_size += commands[i].length; + break; + case LZ_ITERATE: + fputc(commands[i].value, file); + compressed_size++; + break; + case LZ_ALTERNATE: + fputc(commands[i].value, file); + fputc(commands[i + 1].value, file); + compressed_size += 2; + break; + case LZ_ZERO: + break; + case LZ_REPEAT: + case LZ_FLIP: + case LZ_REVERSE: + if (commands[i].offset < LOOKBACK_MAX_OFFSET) { + fputc(commands[i].offset | 0x80, file); + compressed_size++; + } else { + unsigned int address = i - commands[i].offset - 1; + fputc(address >> 8, file); + fputc(address & 0xff, file); + compressed_size += 2; + } + break; + default: + error_exit("invalid LZ command $%02x in output\n", commands[i].cmd); + } + } + + fputc(LZ_END, file); + compressed_size++; + + while (options.alignment && compressed_size % (1 << options.alignment)) { + fputc(0, file); + compressed_size++; + } +} + +void compress_matching(char const *input_name, char const *output_name) { + long input_size; + uint8_t *input_data = read_u8(input_name, &input_size); + + commands_size = (unsigned int)input_size; + commands = xcalloc(sizeof(*commands) * commands_size); + for (unsigned int i = 0; i < commands_size; i++) { + commands[i].value = input_data[i]; + } + + free(input_data); + + pick_optimized_commands(); + + FILE *output = xfopen(output_name, 'w'); + write_commands(output); + xfclose(output); + + free(commands); +} + +/********************************** Optimized dynamic compression *********************************/ + +// Original `lzcomp` implementation by ax6 +// Dynamic programming `dpcomp` algorithm by mei +// Licensed with the Unlicense into the public domain + +// Uncompressed data +uint8_t *raw_data = NULL; +unsigned int raw_data_size = 0; + +// best_sizes[i] = the best compressed length for the first i bytes of input +// Note that this is nondecreasing, since truncating even in the middle of a command won't enlarge it +unsigned int *best_sizes = NULL; +// best_commands[i] = the last command of the commands that yields best_sizes[i] +struct command *best_commands = NULL; + +unsigned int min(unsigned int a, unsigned int b) { + return a < b ? a : b; +} + +unsigned int command_size(const struct command *command) { + unsigned int header_size = 1 + (command->length > SHORT_MAX_LENGTH); + switch (command->cmd) { + case LZ_LITERAL: + return header_size + command->length; + case LZ_ITERATE: + return header_size + 1; + case LZ_ALTERNATE: + return header_size + 2; + case LZ_ZERO: + return header_size; + case LZ_REPEAT: + case LZ_FLIP: + case LZ_REVERSE: + return header_size + 1 + (command->offset >= 0); + default: + error_exit("invalid LZ command $%02x\n", command->cmd); + } +} + +void consider(unsigned int pos, const struct command *command) { + unsigned int new_size = best_sizes[pos - command->length] + command_size(command); + if (new_size < best_sizes[pos]) { + best_sizes[pos] = new_size; + best_commands[pos] = *command; + } +} + +unsigned int match_right(unsigned int pos, unsigned int at) { + unsigned int n = 0; + while (pos + n < raw_data_size && raw_data[pos + n] == raw_data[at + n]) { + n++; + } + return n; +} + +unsigned int match_flipped(unsigned int pos, unsigned int at) { + unsigned int n = 0; + while (pos + n < raw_data_size && flip_f(raw_data[pos + n]) == raw_data[at + n]) { + n++; + } + return n; +} + +unsigned int match_left(unsigned int pos, unsigned int at) { + unsigned int n = 0; + while (pos + n < raw_data_size && n <= at && raw_data[pos + n] == raw_data[at - n]) { + n++; + } + return n; +} + +int encode_offset(int pos, int at) { + return at - pos >= -LOOKBACK_MAX_OFFSET ? at - pos : at; +} + +void find_optimal_commands(void) { + best_sizes = xmalloc(sizeof(*best_sizes) * (raw_data_size + 1)); + best_commands = xmalloc(sizeof(*best_commands) * (raw_data_size + 1)); + + best_sizes[0] = 0; + for (unsigned int i = 1; i <= raw_data_size; i++) { + best_sizes[i] = -1u; + } + + for (unsigned int i = 1; i <= raw_data_size; i++) { + uint8_t byte = raw_data[i - 1]; + for (unsigned int prev = i > LONG_MAX_LENGTH ? i - LONG_MAX_LENGTH : 0; prev < i; prev++) { + consider(i, &(struct command){LZ_LITERAL, i - prev, prev, 0}); + } + + unsigned int len = 0; + do { + len++; + if (len >= 2) { + consider(i, &(struct command){byte ? LZ_ITERATE : LZ_ZERO, len, byte, 0}); + } + } while (len < LONG_MAX_LENGTH && len < i && raw_data[i - (len + 1)] == byte); + + if (i > 1) { + len = 1; + do { + len++; + if (len >= 3) { + int bytes = (raw_data[i - len + 1] << 8) | (raw_data[i - len]); + consider(i, &(struct command){LZ_ALTERNATE, len, bytes, 0}); + } + } while (len < LONG_MAX_LENGTH && len < i && raw_data[i - (len + 1)] == raw_data[i - (len - 1)]); + } + + for (unsigned int at = 0; at < i - 1; at++) { + unsigned int k = min(LONG_MAX_LENGTH, match_right(i - 1, at)); + for (unsigned int j = 2; j <= k; j++) { + consider(i + j - 1, &(struct command){LZ_REPEAT, j, encode_offset(i - 1, at), 0}); + } + + k = min(LONG_MAX_LENGTH, match_left(i - 1, at)); + for (unsigned int j = 2; j <= k; j++) { + consider(i + j - 1, &(struct command){LZ_REVERSE, j, encode_offset(i - 1, at), 0}); + } + + k = min(LONG_MAX_LENGTH, match_flipped(i - 1, at)); + for (unsigned int j = 2; j <= k; j++) { + consider(i + j - 1, &(struct command){LZ_FLIP, j, encode_offset(i - 1, at), 0}); + } + } + } + + unsigned int command_count = 0; + for (unsigned int pos = raw_data_size; pos > 0; pos -= best_commands[pos].length) { + command_count++; + } + commands_size = command_count; + + commands = xmalloc(sizeof(*commands) * command_count); + for (unsigned int pos = raw_data_size; pos > 0; pos -= best_commands[pos].length) { + commands[--command_count] = best_commands[pos]; + } +} + +void write_commands_dynamic(FILE *file) { + unsigned int compressed_size = 0; + + for (unsigned int i = 0; i < commands_size; i++) { + struct command command = commands[i]; + + uint8_t buf[4] = {0}; + uint8_t *buf_pos = buf; + + command.length--; + + if (command.length >= LONG_MAX_LENGTH) { + error_exit("invalid LZ command $%02x (length %u) in output\n", command.cmd, command.length); + } + + if (command.length < SHORT_MAX_LENGTH) { + *(buf_pos++) = (command.cmd << 5) + command.length; + } else { + *(buf_pos++) = (LZ_LONG << 5) + (command.cmd << 2) + (command.length >> 8); + *(buf_pos++) = command.length & 0xff; + } + + switch (command.cmd) { + case LZ_LITERAL: + break; + case LZ_ITERATE: + if (command.offset < 0 || command.offset > 0xff) { + error_exit("invalid LZ command $%02x (byte $%x) in output\n", command.cmd, command.offset); + } + *(buf_pos++) = command.offset & 0xff; + break; + case LZ_ALTERNATE: + if (command.offset < 0 || command.offset > 0xffff) { + error_exit("invalid LZ command $%02x (bytes $%x) in output\n", command.cmd, command.offset); + } + *(buf_pos++) = command.offset & 0xff; + *(buf_pos++) = command.offset >> 8; + break; + case LZ_ZERO: + break; + case LZ_REPEAT: + case LZ_FLIP: + case LZ_REVERSE: + if (command.offset < -LOOKBACK_MAX_OFFSET) { + error_exit("invalid LZ command $%02x (offset %d) in output\n", command.cmd, command.offset); + } + if (command.offset < 0) { + *(buf_pos++) = command.offset ^ 0x7f; + } else { + *(buf_pos++) = command.offset >> 8; + *(buf_pos++) = command.offset & 0xff; + } + break; + default: + error_exit("invalid LZ command $%02x in output\n", command.cmd); + } + + fwrite(buf, 1, buf_pos - buf, file); + if (!command.cmd) { + command.length++; + fwrite(raw_data + command.offset, 1, command.length, file); + } + + compressed_size += command_size(&commands[i]); + } + + putc(LZ_END, file); + + unsigned int padding_size = ~compressed_size & ((1 << options.alignment) - 1); + while (padding_size--) { + putc(0, file); + } +} + +void compress_dynamic(char const *input_name, char const *output_name) { + long input_size; + raw_data = read_u8(input_name, &input_size); + + raw_data_size = (unsigned int)input_size; + find_optimal_commands(); + + free(best_commands); + free(best_sizes); + + FILE *output = xfopen(output_name, 'w'); + write_commands_dynamic(output); + xfclose(output); + + free(raw_data); + free(commands); +} + +/****************************************** Decompression *****************************************/ + +struct command *read_commands(char const *filename, unsigned int *num_commands) { + long filesize; + uint8_t *data = read_u8(filename, &filesize); + + // There will be at most one command per byte of compressed data, so + // `filesize` is a valid upper bound for the count of `command_data` + struct command *command_data = xmalloc(sizeof(*command_data) * filesize); + *num_commands = 0; + + for (long i = 0; i < filesize;) { + uint8_t byte = data[i++]; + struct command *command = &command_data[(*num_commands)++]; + + unsigned int following_len = 0; + + if (byte == LZ_END) { + command->cmd = LZ_END; + } else { + command->cmd = byte >> 5; + command->length = byte & 31; + if (command->cmd == LZ_LONG) { + if (i >= filesize) { + error_exit("incomplete long command at end of input\n"); + } + command->cmd = command->length >> 2; + command->length = ((command->length & 3) << 8) | data[i++]; + } + command->length++; + } + + switch (command->cmd) { + case LZ_END: + following_len = filesize - i; + break; + case LZ_LITERAL: + following_len = command->length; + break; + case LZ_ITERATE: + following_len = 1; + break; + case LZ_ALTERNATE: + following_len = 2; + break; + case LZ_ZERO: + break; + case LZ_REPEAT: + case LZ_FLIP: + case LZ_REVERSE: + if (i >= filesize) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + command->offset = data[i++]; + if (command->offset & 0x80) { + command->offset = -((command->offset & 0x7f) + 1); + } else { + if (i >= filesize) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + command->offset = (command->offset << 8) | data[i++]; + } + break; + default: + error_exit("invalid LZ command $%02x\n", command->cmd); + } + + if (i + following_len > filesize) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + for (unsigned int j = 0; j < following_len; j++) { + command_data[(*num_commands)++].value = data[i++]; + } + } + + free(data); + return command_data; +} + +void decompress(char const *input_name, char const *output_name) { + unsigned int input_size; + struct command *input_data = read_commands(input_name, &input_size); + + unsigned int output_capacity = input_size * 2; + uint8_t *output_data = xmalloc(output_capacity); + unsigned int output_size = 0; + + for (unsigned int i = 0; i < input_size;) { + struct command *command = &input_data[i++]; + + if (command->cmd == LZ_END) { + break; + } + + if (output_size + command->length > output_capacity) { + output_capacity = output_capacity * 2 + command->length; + output_data = xrealloc(output_data, output_capacity); + } + + switch (command->cmd) { + case LZ_LITERAL: + if (i + command->length > input_size) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = input_data[i++].value; + } + break; + case LZ_ITERATE: + if (i + 1 > input_size) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + uint8_t byte = input_data[i++].value; + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = byte; + } + break; + case LZ_ALTERNATE: + if (i + 2 > input_size) { + error_exit("incomplete $%02x command at end of input\n", command->cmd); + } + uint8_t bytes[2] = {input_data[i].value, input_data[i + 1].value}; + i += 2; + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = bytes[j & 1]; + } + break; + case LZ_ZERO: + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = 0; + } + break; + case LZ_REPEAT: { + unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset; + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = output_data[base + j]; + } + break; + } + case LZ_FLIP: { + unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset; + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = flip_f(output_data[base + j]); + } + break; + } + case LZ_REVERSE: { + unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset; + for (unsigned int j = 0; j < command->length; j++) { + output_data[output_size++] = output_data[base - j]; + } + break; + } + default: + error_exit("invalid LZ command $%02x in input\n", command->cmd); + } + } + + write_u8(output_name, output_data, output_size); + + free(input_data); + free(output_data); +} + +/********************************************** Main **********************************************/ + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + argc -= optind; + argv += optind; + if (argc != 2) { + usage_exit(1); + } + + if (options.decompress) { + decompress(argv[0], argv[1]); + } else if (options.matching) { + compress_matching(argv[0], argv[1]); + } else { + compress_dynamic(argv[0], argv[1]); + } + + return 0; +}