mirror of
https://github.com/pret/pokegold.git
synced 2026-08-23 09:04:52 -05:00
Replace lzcomp with lzcompress (#148)
This commit is contained in:
Rangi
2
Makefile
2
Makefile
@@ -207,7 +207,7 @@ pokesilver_vc.gbc: RGBFIXFLAGS += -t POKEMON_SLV -i AAXE
|
||||
include gfx/lz.mk
|
||||
|
||||
%.lz: %
|
||||
tools/lzcomp $(LZFLAGS) -- $< $@
|
||||
tools/lzcompress $(LZFLAGS) -- $< $@
|
||||
|
||||
|
||||
### Pokemon and trainer sprite rules
|
||||
|
||||
58
gfx/lz.mk
58
gfx/lz.mk
@@ -1,17 +1,17 @@
|
||||
# Delete these rules if you don't care about matching.
|
||||
|
||||
%.lz: LZFLAGS = --compressor multipass
|
||||
%.lz: LZFLAGS = --matching
|
||||
|
||||
gfx/battle_anims/%.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/battle_anims/%.lz: LZFLAGS += --odd-alternate --align 4
|
||||
|
||||
gfx/battle/dude.2bpp.lz: LZFLAGS += --align 4
|
||||
|
||||
gfx/diploma/diploma.2bpp.lz: LZFLAGS += --method 8 --align 1
|
||||
gfx/diploma/diploma.2bpp.lz: LZFLAGS += --prefer-alternate --align 1
|
||||
|
||||
gfx/memory_game/memory_game.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/memory_game/memory_game.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
|
||||
gfx/intro/%.lz: LZFLAGS += --align 4
|
||||
gfx/intro/water1.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/intro/water1.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
|
||||
gfx/new_game/shrink1.2bpp.lz: LZFLAGS += --align 4
|
||||
gfx/new_game/shrink2.2bpp.lz: LZFLAGS += --align 4
|
||||
@@ -20,41 +20,41 @@ gfx/pc/pc.2bpp.lz: LZFLAGS += --align 4
|
||||
|
||||
gfx/player/chris_back.2bpp.lz: LZFLAGS += --align 4
|
||||
|
||||
gfx/pokedex/pokedex.2bpp.lz: LZFLAGS += --method 8 --align 1
|
||||
gfx/pokedex/pokedex_sgb.2bpp.lz: LZFLAGS += --method 8 --align 1
|
||||
gfx/pokedex/pokedex.2bpp.lz: LZFLAGS += --prefer-alternate --align 1
|
||||
gfx/pokedex/pokedex_sgb.2bpp.lz: LZFLAGS += --prefer-alternate --align 1
|
||||
gfx/pokedex/question_mark.2bpp.lz: LZFLAGS += --align 4
|
||||
gfx/pokedex/slowpoke.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/pokedex/slowpoke.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
|
||||
gfx/pokegear/pokegear.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/pokegear/pokegear.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
gfx/pokegear/pokegear_sprites.2bpp.lz: LZFLAGS += --align 4
|
||||
gfx/pokegear/town_map.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/pokegear/town_map.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
|
||||
gfx/pokemon/hitmontop/back_gold.2bpp.lz: LZFLAGS += --method 4
|
||||
gfx/pokemon/miltank/back_gold.2bpp.lz: LZFLAGS += --method 4
|
||||
gfx/pokemon/hitmontop/back_gold.2bpp.lz: LZFLAGS += --no-lookback-3
|
||||
gfx/pokemon/miltank/back_gold.2bpp.lz: LZFLAGS += --no-lookback-3
|
||||
|
||||
gfx/slots/slots_1.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/slots/slots_1.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/slots/slots_2.2bpp.lz: LZFLAGS += --align 4
|
||||
gfx/slots/slots_3.2bpp.lz: LZFLAGS += --align 4
|
||||
|
||||
gfx/tilesets/%.lz: LZFLAGS += --align 4
|
||||
gfx/tilesets/game_corner.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/house.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/johto.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/tilesets/johto_modern.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/tilesets/kanto.2bpp.lz: LZFLAGS = --compressor null --method 1 --align 1
|
||||
gfx/tilesets/mansion.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/mart.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/players_room.2bpp.lz: LZFLAGS += --method 4 --align 4
|
||||
gfx/tilesets/radio_tower.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/ruins_of_alph.2bpp.lz: LZFLAGS += --method 6 --align 4
|
||||
gfx/tilesets/tower.2bpp.lz: LZFLAGS += --method 2 --align 4
|
||||
gfx/tilesets/game_corner.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/tilesets/house.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/tilesets/johto.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
gfx/tilesets/johto_modern.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
gfx/tilesets/kanto.2bpp.lz: LZFLAGS += --literal-only --align 1
|
||||
gfx/tilesets/mansion.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/tilesets/mart.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/tilesets/players_room.2bpp.lz: LZFLAGS += --no-lookback-3 --align 4
|
||||
gfx/tilesets/radio_tower.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
gfx/tilesets/ruins_of_alph.2bpp.lz: LZFLAGS += --odd-alternate --no-lookback-3 --align 4
|
||||
gfx/tilesets/tower.2bpp.lz: LZFLAGS += --odd-alternate --align 4
|
||||
|
||||
gfx/title/hooh_gold.2bpp.lz: LZFLAGS += --align 3
|
||||
gfx/title/logo_bottom_gold.2bpp.lz: LZFLAGS += --method 9 --align 1
|
||||
gfx/title/logo_bottom_silver.2bpp.lz: LZFLAGS += --method 9 --align 1
|
||||
gfx/title/logo_top_gold.2bpp.lz: LZFLAGS += --method 9 --align 1
|
||||
gfx/title/logo_top_silver.2bpp.lz: LZFLAGS += --method 9 --align 1
|
||||
gfx/title/logo_bottom_gold.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1
|
||||
gfx/title/logo_bottom_silver.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1
|
||||
gfx/title/logo_top_gold.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1
|
||||
gfx/title/logo_top_silver.2bpp.lz: LZFLAGS += --skip-initial-byte --prefer-alternate --align 1
|
||||
gfx/title/lugia_silver.2bpp.lz: LZFLAGS += --align 4
|
||||
|
||||
gfx/unown_puzzle/%.lz: LZFLAGS += --align 4
|
||||
gfx/unown_puzzle/start_cancel.2bpp.lz: LZFLAGS += --method 8 --align 1
|
||||
gfx/unown_puzzle/start_cancel.2bpp.lz: LZFLAGS += --prefer-alternate --align 1
|
||||
|
||||
2
tools/.gitignore
vendored
2
tools/.gitignore
vendored
@@ -1,6 +1,6 @@
|
||||
gbcpal
|
||||
gfx
|
||||
lzcomp
|
||||
lzcompress
|
||||
make_patch
|
||||
png_dimensions
|
||||
scan_includes
|
||||
|
||||
@@ -6,7 +6,7 @@ CFLAGS := -O3 -std=c11 -Wall -Wextra -pedantic
|
||||
tools := \
|
||||
gbcpal \
|
||||
gfx \
|
||||
lzcomp \
|
||||
lzcompress \
|
||||
make_patch \
|
||||
png_dimensions \
|
||||
scan_includes \
|
||||
@@ -18,9 +18,5 @@ all: $(tools)
|
||||
clean:
|
||||
$(RM) $(tools)
|
||||
|
||||
lzcomp: CFLAGS += -Wno-strict-overflow -Wno-sign-compare
|
||||
lzcomp: $(wildcard lz/*.c) $(wildcard lz/*.h)
|
||||
$(CC) $(CFLAGS) -o $@ lz/*.c
|
||||
|
||||
%: %.c common.h
|
||||
$(CC) $(CFLAGS) -o $@ $<
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
const struct compressor compressors[] = {
|
||||
// NOTE: the "flags" field for each compressor will be set to the chosen/current method number minus the base
|
||||
// number for that particular compressor. That means that each compressor will use a zero-based flags value.
|
||||
{.methods = 72, .name = "singlepass", .function = &try_compress_single_pass}, // 0-71
|
||||
{.methods = 2, .name = "null", .function = &store_uncompressed}, // 72-73
|
||||
{.methods = 6, .name = "repetitions", .function = &try_compress_repetitions}, // 74-79
|
||||
{.methods = 16, .name = "multipass", .function = &try_compress_multi_pass}, // 80-95
|
||||
{0} // end of the list
|
||||
};
|
||||
|
||||
const unsigned char bit_flipping_table[] = {
|
||||
// For each byte, the table contains that same byte with its bits flipped around (for instance,
|
||||
// 0x58 (01011000 binary) becomes 0x1a (00011010 binary)). This is faster than flipping bits
|
||||
// manually at runtime.
|
||||
0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0,
|
||||
0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8,
|
||||
0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4,
|
||||
0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc,
|
||||
0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2,
|
||||
0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa,
|
||||
0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6,
|
||||
0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe,
|
||||
0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1,
|
||||
0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9,
|
||||
0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5,
|
||||
0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd,
|
||||
0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3,
|
||||
0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb,
|
||||
0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7,
|
||||
0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff
|
||||
};
|
||||
|
||||
char option_name_buffer[] = "-?"; // used to extract the name of a short option (separated from its argument)
|
||||
@@ -1,54 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
int main (int argc, char ** argv) {
|
||||
struct options options = get_options(argc, argv);
|
||||
unsigned short size;
|
||||
unsigned char * file_buffer = read_file_into_buffer(options.input, &size);
|
||||
struct command * commands;
|
||||
if (options.mode & 2) {
|
||||
unsigned short original_size = size, remainder;
|
||||
commands = get_commands_from_file(file_buffer, &size, &remainder);
|
||||
if (!commands) error_exit(1, "invalid command stream");
|
||||
if (options.mode == 2) {
|
||||
unsigned char * uncompressed = get_uncompressed_data(commands, file_buffer, &size);
|
||||
if (!uncompressed) error_exit(1, "output data is too large");
|
||||
write_raw_data_to_file(options.output, uncompressed, size);
|
||||
free(uncompressed);
|
||||
} else
|
||||
write_commands_and_padding_to_textfile(options.output, commands, size, file_buffer, original_size - remainder, remainder);
|
||||
} else {
|
||||
commands = compress(file_buffer, &size, options.method);
|
||||
(options.mode ? write_commands_to_textfile : write_commands_to_file)(options.output, commands, size, file_buffer, options.alignment);
|
||||
}
|
||||
free(file_buffer);
|
||||
free(commands);
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct command * compress (const unsigned char * data, unsigned short * size, unsigned method) {
|
||||
unsigned char * bitflipped = malloc(*size);
|
||||
unsigned current;
|
||||
for (current = 0; current < *size; current ++) bitflipped[current] = bit_flipping_table[data[current]];
|
||||
const struct compressor * compressor = compressors;
|
||||
struct command * result;
|
||||
if (method < COMPRESSION_METHODS) {
|
||||
while (method >= compressor -> methods) method -= (compressor ++) -> methods;
|
||||
result = compressor -> function(data, bitflipped, size, method);
|
||||
} else {
|
||||
struct command * compressed_sequences[COMPRESSION_METHODS];
|
||||
unsigned short lengths[COMPRESSION_METHODS];
|
||||
unsigned flags = 0;
|
||||
for (current = 0; current < COMPRESSION_METHODS; current ++) {
|
||||
lengths[current] = *size;
|
||||
if (flags == compressor -> methods) {
|
||||
flags = 0;
|
||||
compressor ++;
|
||||
}
|
||||
compressed_sequences[current] = compressor -> function(data, bitflipped, lengths + current, flags ++);
|
||||
}
|
||||
result = select_optimal_sequence(compressed_sequences, lengths, size);
|
||||
for (current = 0; current < COMPRESSION_METHODS; current ++) free(compressed_sequences[current]);
|
||||
}
|
||||
free(bitflipped);
|
||||
return result;
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
struct command * select_optimal_sequence (struct command ** sequences, const unsigned short * lengths, unsigned short * final_length) {
|
||||
struct command * compressor_sequences[NUM_COMPRESSORS * 2];
|
||||
unsigned short compressor_lengths[NUM_COMPRESSORS * 2];
|
||||
struct command * inverted_sequences[COMPRESSION_METHODS];
|
||||
unsigned short inverted_lengths[COMPRESSION_METHODS];
|
||||
unsigned p, current, method = 0;
|
||||
for (current = 0; current < NUM_COMPRESSORS; current ++) {
|
||||
compressor_sequences[current] = select_command_sequence(sequences + method, lengths + method, compressors[current].methods, compressor_lengths + current);
|
||||
compressor_sequences[current + NUM_COMPRESSORS] = select_command_sequence(sequences + method, lengths + method, -(int) compressors[current].methods,
|
||||
compressor_lengths + (current + NUM_COMPRESSORS));
|
||||
for (p = 0; p < compressors[current].methods; p ++) {
|
||||
inverted_sequences[method + compressors[current].methods - 1 - p] = sequences[method + p];
|
||||
inverted_lengths[method + compressors[current].methods - 1 - p] = lengths[method + p];
|
||||
}
|
||||
method += compressors[current].methods;
|
||||
}
|
||||
unsigned short final_lengths[8];
|
||||
struct command * final_sequences[8] = {
|
||||
select_command_sequence(compressor_sequences, compressor_lengths, NUM_COMPRESSORS, final_lengths),
|
||||
select_command_sequence(compressor_sequences, compressor_lengths, -NUM_COMPRESSORS, final_lengths + 1),
|
||||
select_command_sequence(compressor_sequences + NUM_COMPRESSORS, compressor_lengths + NUM_COMPRESSORS, NUM_COMPRESSORS, final_lengths + 2),
|
||||
select_command_sequence(compressor_sequences + NUM_COMPRESSORS, compressor_lengths + NUM_COMPRESSORS, -NUM_COMPRESSORS, final_lengths + 3),
|
||||
select_command_sequence(sequences, lengths, COMPRESSION_METHODS, final_lengths + 4),
|
||||
select_command_sequence(sequences, lengths, -COMPRESSION_METHODS, final_lengths + 5),
|
||||
select_command_sequence(inverted_sequences, inverted_lengths, COMPRESSION_METHODS, final_lengths + 6),
|
||||
select_command_sequence(inverted_sequences, inverted_lengths, -COMPRESSION_METHODS, final_lengths + 7)
|
||||
};
|
||||
for (current = 0; current < (2 * NUM_COMPRESSORS); current ++) free(compressor_sequences[current]);
|
||||
struct command * result = select_command_sequence(final_sequences, final_lengths, 8, final_length);
|
||||
for (current = 0; current < 8; current ++) free(final_sequences[current]);
|
||||
return result;
|
||||
}
|
||||
|
||||
struct command * select_command_sequence (struct command ** sequences, const unsigned short * lengths, int count, unsigned short * final_length) {
|
||||
// negative count indicates iterating backwards
|
||||
unsigned short min_sequence = 0, min_length = compressed_length(*sequences, *lengths);
|
||||
unsigned short seq, len;
|
||||
int backwards = 0;
|
||||
if (count < 0) {
|
||||
backwards = 1;
|
||||
count = -count;
|
||||
}
|
||||
for (seq = 1; seq < count; seq ++) {
|
||||
len = compressed_length(sequences[seq], lengths[seq]);
|
||||
if (len < min_length) {
|
||||
min_sequence = seq;
|
||||
min_length = len;
|
||||
}
|
||||
}
|
||||
*final_length = lengths[min_sequence];
|
||||
struct command * current = malloc(*final_length * sizeof(struct command));
|
||||
memcpy(current, sequences[min_sequence], *final_length * sizeof(struct command));
|
||||
struct command * new;
|
||||
for (seq = 1; seq < count; seq ++) {
|
||||
if (backwards) seq = count - seq;
|
||||
new = merge_command_sequences(current, *final_length, sequences[(seq + min_sequence) % count], lengths[(seq + min_sequence) % count], final_length);
|
||||
if (backwards) seq = count - seq; // restore the value for the loop
|
||||
free(current);
|
||||
current = new;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
struct command * merge_command_sequences (const struct command * current, unsigned short current_length, const struct command * new, unsigned short new_length,
|
||||
unsigned short * result_length) {
|
||||
struct command * result = malloc(sizeof(struct command) * (current_length + new_length));
|
||||
struct command * current_command = result;
|
||||
const struct command * saved_current;
|
||||
const struct command * saved_new;
|
||||
unsigned short current_pos, new_pos;
|
||||
while (current_length) {
|
||||
if (current -> count == new -> count) {
|
||||
*(current_command ++) = pick_best_command(2, *(current ++), *(new ++));
|
||||
current_length --;
|
||||
continue;
|
||||
}
|
||||
saved_current = current;
|
||||
saved_new = new;
|
||||
current_pos = (current ++) -> count;
|
||||
new_pos = (new ++) -> count;
|
||||
current_length --;
|
||||
while (current_pos != new_pos)
|
||||
if (current_pos < new_pos) {
|
||||
current_pos += (current ++) -> count;
|
||||
current_length --;
|
||||
} else
|
||||
new_pos += (new ++) -> count;
|
||||
current_pos = compressed_length(saved_current, current - saved_current);
|
||||
new_pos = compressed_length(saved_new, new - saved_new);
|
||||
if (new_pos < current_pos) {
|
||||
memcpy(current_command, saved_new, sizeof(struct command) * (new - saved_new));
|
||||
current_command += new - saved_new;
|
||||
} else {
|
||||
memcpy(current_command, saved_current, sizeof(struct command) * (current - saved_current));
|
||||
current_command += current - saved_current;
|
||||
}
|
||||
}
|
||||
*result_length = current_command - result;
|
||||
return result;
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
/*
|
||||
Multi-pass compressor: performs an initial pass generating a single command for each byte position in the data and
|
||||
refines the command stream further in subsequent passes.
|
||||
Methods defined: 16
|
||||
Flags values: the flags are a bitfield; each bit triggers some alternate behavior if set:
|
||||
1: always emit a literal command (0) for the first byte of the file
|
||||
2: when reducing a two-byte repetition (2) command in the overlap elimination pass, don't force it to contain a
|
||||
whole number of repetitions (i.e., an even count)
|
||||
4: don't emit copy commands (4, 5, 6) with a count of 3
|
||||
8: don't emit single-byte repetition (1) commands
|
||||
*/
|
||||
|
||||
struct command * try_compress_multi_pass (const unsigned char * data, const unsigned char * flipped, unsigned short * size, unsigned flags) {
|
||||
struct command * result = calloc(*size, sizeof(struct command));
|
||||
unsigned char * reversed = malloc(*size);
|
||||
short * sources = malloc(*size * sizeof(short));
|
||||
unsigned short pos, next, current = 0;
|
||||
for (pos = 0; pos < *size; pos ++) {
|
||||
reversed[pos] = data[*size - 1 - pos];
|
||||
sources[pos] = -1;
|
||||
}
|
||||
for (pos = (flags & 1); pos < *size; pos += (result[pos].count >= MULTIPASS_SKIP_THRESHOLD) ? result[pos].count : 1) {
|
||||
result[pos] = pick_command_for_pass(data, flipped, reversed, sources, *size, pos, flags);
|
||||
if ((result[pos].command >= 4) || (result[pos].count < MULTIPASS_SKIP_THRESHOLD)) sources[current ++] = pos;
|
||||
}
|
||||
free(reversed);
|
||||
free(sources);
|
||||
for (pos = 0; pos < *size; pos ++) {
|
||||
for (current = 1; current < result[pos].count; current ++) if (result[pos + current].count > result[pos].count) {
|
||||
result[pos].count = current;
|
||||
if ((result[pos].command == 2) && (current & 1) && !(flags & 2)) result[pos].count --;
|
||||
}
|
||||
if (result[pos].count <= command_size(result[pos])) result[pos] = (struct command) {.command = 0, .count = 0};
|
||||
}
|
||||
for (pos = 0; pos < *size; pos ++)
|
||||
if (!result[pos].command) {
|
||||
for (current = 1; (current < MAX_COMMAND_COUNT) && ((pos + current) < *size); current ++) if (result[pos + current].command) break;
|
||||
result[pos] = (struct command) {.command = 0, .count = current, .value = pos};
|
||||
} else if (result[pos].count > MAX_COMMAND_COUNT) {
|
||||
result[pos + MAX_COMMAND_COUNT] = result[pos];
|
||||
result[pos + MAX_COMMAND_COUNT].count -= MAX_COMMAND_COUNT;
|
||||
if ((result[pos + MAX_COMMAND_COUNT].command >= 4) && (result[pos + MAX_COMMAND_COUNT].value >= 0))
|
||||
result[pos + MAX_COMMAND_COUNT].value += (result[pos].command == 6) ? -MAX_COMMAND_COUNT : MAX_COMMAND_COUNT;
|
||||
result[pos].count = MAX_COMMAND_COUNT;
|
||||
}
|
||||
for (next = pos = 0; pos < *size; pos ++)
|
||||
if (pos == next)
|
||||
next += result[pos].count;
|
||||
else
|
||||
result[pos].command = 7;
|
||||
repack(&result, size);
|
||||
return result;
|
||||
}
|
||||
|
||||
struct command pick_command_for_pass (const unsigned char * data, const unsigned char * flipped, const unsigned char * reversed, const short * sources,
|
||||
unsigned short length, unsigned short position, unsigned flags) {
|
||||
struct command result = pick_repetition_for_pass(data, length, position, flags);
|
||||
if (result.count >= MULTIPASS_SKIP_THRESHOLD) return result;
|
||||
unsigned char p;
|
||||
for (p = 0; p < 3; p ++) {
|
||||
struct command temp = pick_copy_for_pass(data, p[(const unsigned char * []) {data, flipped, reversed}], sources, p + 4, length, position, flags);
|
||||
if (temp.command == 7) continue;
|
||||
if (temp.count > result.count) result = temp;
|
||||
}
|
||||
if ((result.command >= 4) && (result.value >= (position - LOOKBACK_LIMIT))) result.value -= position;
|
||||
return result;
|
||||
}
|
||||
|
||||
struct command pick_repetition_for_pass (const unsigned char * data, unsigned short length, unsigned short position, unsigned flags) {
|
||||
unsigned short p;
|
||||
if (data[position]) {
|
||||
if ((position + 1) >= length) return (struct command) {.command = 1, .count = 1, .value = data[position]};
|
||||
struct command result;
|
||||
if (!(flags & 8) && (data[position] == data[position + 1]))
|
||||
result = (struct command) {.command = 1, .value = data[position]};
|
||||
else
|
||||
result = (struct command) {.command = 2, .value = data[position] | (data[position + 1] << 8)};
|
||||
for (p = 1; ((position + p) < length) && (p < LOOKAHEAD_LIMIT); p ++) if (data[position + p] != data[position + (p & 1)]) break;
|
||||
result.count = p;
|
||||
return result;
|
||||
} else {
|
||||
for (p = position + 1; (p < length) && (p < (position + LOOKAHEAD_LIMIT)); p ++) if (data[p]) break;
|
||||
return (struct command) {.command = 3, .count = p - position};
|
||||
}
|
||||
}
|
||||
|
||||
struct command pick_copy_for_pass (const unsigned char * data, const unsigned char * reference, const short * sources, unsigned char command_type,
|
||||
unsigned short length, unsigned short position, unsigned flags) {
|
||||
struct command result = {.command = 7, .count = (flags & 4) ? 4 : 3};
|
||||
if (length < 3) return result;
|
||||
unsigned refpos, count;
|
||||
const unsigned char * current;
|
||||
unsigned char buffer[6] = {0};
|
||||
memcpy(buffer, reference + length - 3, 3);
|
||||
while (*sources >= 0) {
|
||||
refpos = *(sources ++);
|
||||
if (command_type == 6) refpos = length - 1 - refpos;
|
||||
if (refpos >= (length - 3))
|
||||
current = buffer + refpos - (length - 3);
|
||||
else
|
||||
current = reference + refpos;
|
||||
if (memcmp(data + position, current, ((position + 4) > length) ? length - position : 4)) continue;
|
||||
for (count = 4; (count < (length - position)) && (count < (length - refpos)); count ++) if (data[position + count] != current[count]) break;
|
||||
if (count > (length - refpos)) count = length - refpos;
|
||||
if (count > (length - position)) count = length - position;
|
||||
if (result.count > count) continue;
|
||||
result = (struct command) {.command = command_type, .count = count, .value = sources[-1]};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
/*
|
||||
Null compressor: stores data uncompressed, using literal (0) commands only.
|
||||
Methods defined: 2
|
||||
Flags values: 0 = split a trailing 33-to-64-byte block at the end into two short blocks; 1 = don't
|
||||
*/
|
||||
|
||||
struct command * store_uncompressed (__attribute__((unused)) const unsigned char * data, __attribute__((unused)) const unsigned char * bitflipped, unsigned short * size, unsigned flags) {
|
||||
unsigned short position, block, remainder = *size;
|
||||
struct command * result = NULL;
|
||||
*size = 0;
|
||||
for (position = 0; remainder; position += block, remainder -= block) {
|
||||
block = (remainder > MAX_COMMAND_COUNT) ? MAX_COMMAND_COUNT : remainder;
|
||||
if (!(flags & 1) && (block <= (2 * SHORT_COMMAND_COUNT)) && (block > SHORT_COMMAND_COUNT)) block = SHORT_COMMAND_COUNT;
|
||||
result = realloc(result, sizeof(struct command) * (1 + *size));
|
||||
result[(*size) ++] = (struct command) {.command = 0, .count = block, .value = position};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
struct options get_options (int argc, char ** argv) {
|
||||
struct options result = {.input = NULL, .output = NULL, .mode = 0, .alignment = 0, .method = COMPRESSION_METHODS};
|
||||
const char * program_name = *argv;
|
||||
int compressor = -1;
|
||||
if (argc == 1) usage(program_name);
|
||||
for (argv ++; *argv; argv ++) {
|
||||
if (**argv != '-') break;
|
||||
if (!1[*argv]) break;
|
||||
if (!strcmp(*argv, "--")) {
|
||||
argv ++;
|
||||
break;
|
||||
} else if (!(strcmp(*argv, "--text") && strcmp(*argv, "-t")))
|
||||
result.mode = 1;
|
||||
else if (!(strcmp(*argv, "--binary") && strcmp(*argv, "-b")))
|
||||
result.mode = 0;
|
||||
else if (!(strcmp(*argv, "--uncompress") && strcmp(*argv, "-u")))
|
||||
result.mode = 2;
|
||||
else if (!(strcmp(*argv, "--dump") && strcmp(*argv, "-d")))
|
||||
result.mode = 3;
|
||||
else if (!(strcmp(*argv, "--align") && strncmp(*argv, "-a", 2)))
|
||||
result.alignment = parse_numeric_option_argument(&argv, 12);
|
||||
else if (!(strcmp(*argv, "--method") && strncmp(*argv, "-m", 2)))
|
||||
result.method = parse_numeric_option_argument(&argv, COMPRESSION_METHODS - 1);
|
||||
else if (!(strcmp(*argv, "--compressor") && strncmp(*argv, "-c", 2)))
|
||||
compressor = parse_compressor_option_argument(&argv);
|
||||
else if (!(strcmp(*argv, "--optimize") && strcmp(*argv, "-o"))) {
|
||||
result.method = COMPRESSION_METHODS;
|
||||
compressor = -1;
|
||||
} else if (!(strcmp(*argv, "--help") && strcmp(*argv, "-?")))
|
||||
usage(program_name);
|
||||
else if (!(strcmp(*argv, "--list") && strcmp(*argv, "-l")))
|
||||
list_compressors();
|
||||
else
|
||||
error_exit(3, "unknown option: %s", *argv);
|
||||
}
|
||||
if (compressor >= 0) {
|
||||
if (result.method >= COMPRESSION_METHODS) result.method = 0;
|
||||
if (result.method >= compressors[compressor].methods)
|
||||
error_exit(3, "method for the %s compressor must be between 0 and %u", compressors[compressor].name, compressors[compressor].methods - 1);
|
||||
while (compressor > 0) result.method += compressors[-- compressor].methods;
|
||||
}
|
||||
if (*argv) {
|
||||
if (strcmp(*argv, "-")) result.input = *argv;
|
||||
if (*(++ argv)) {
|
||||
if (argv[1]) error_exit(3, "too many command-line arguments");
|
||||
if (strcmp(*argv, "-")) result.output = *argv;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
unsigned parse_numeric_option_argument (char *** alp, unsigned limit) {
|
||||
const char * option;
|
||||
const char * value = get_argument_for_option(alp, &option);
|
||||
char * error;
|
||||
unsigned long result = strtoul(value, &error, 10);
|
||||
if (*error) error_exit(3, "invalid argument to option %s", option);
|
||||
if (result > limit) error_exit(3, "argument to option %s must be between 0 and %u", option, limit);
|
||||
return result;
|
||||
}
|
||||
|
||||
int parse_compressor_option_argument (char *** alp) {
|
||||
const char * name = get_argument_for_option(alp, NULL);
|
||||
if (!strcmp(name, "*")) return -1;
|
||||
int result = -1;
|
||||
unsigned length = strlen(name);
|
||||
const struct compressor * compressor;
|
||||
for (compressor = compressors; compressor -> name; compressor ++) {
|
||||
if (strncmp(name, compressor -> name, length)) continue;
|
||||
if (result >= 0) error_exit(3, "ambiguous compressor prefix: %s", name);
|
||||
result = compressor - compressors;
|
||||
}
|
||||
if (result < 0) error_exit(3, "unknown compressor: %s", name);
|
||||
return result;
|
||||
}
|
||||
|
||||
const char * get_argument_for_option (char *** alp, const char ** option_name) {
|
||||
// alp: argument list pointer (i.e., address of the current value of argv after indexing)
|
||||
// will point at the last consumed argument on exit (since the caller will probably increment it once more)
|
||||
const char * option;
|
||||
const char * result;
|
||||
if (1[**alp] == '-') {
|
||||
option = *((*alp) ++);
|
||||
result = **alp;
|
||||
} else {
|
||||
option_name_buffer[1] = 1[**alp];
|
||||
option = option_name_buffer;
|
||||
result = **alp + 2;
|
||||
}
|
||||
if (!(result && *result)) error_exit(3, "option %s requires an argument", option);
|
||||
if (option_name) *option_name = option;
|
||||
return result;
|
||||
}
|
||||
|
||||
noreturn usage (const char * program_name) {
|
||||
fprintf(stderr, "Usage: %s [<options>] [<source file> [<output>]]\n\n", program_name);
|
||||
fputs("Execution mode:\n", stderr);
|
||||
fputs(" -b, --binary Output the command stream as binary data (default).\n", stderr);
|
||||
fputs(" -t, --text Output the command stream as text.\n", stderr);
|
||||
fputs(" -u, --uncompress Process a compressed file and output the original data.\n", stderr);
|
||||
fputs(" -d, --dump Process a compressed file and dump the command stream as\n", stderr);
|
||||
fputs(" text (as if compressed with the --text option).\n", stderr);
|
||||
fputs(" -l, --list List compressors and their method numbers.\n", stderr);
|
||||
fputs(" -?, --help Print this help text and exit.\n", stderr);
|
||||
fputs("Compression options:\n", stderr);
|
||||
fputs(" -o, --optimize Use the best combination of compression\n", stderr);
|
||||
fputs(" methods available (default).\n", stderr);
|
||||
fputs(" -m<number>, --method <number> Use only one specific compression method.\n", stderr);
|
||||
fprintf(stderr, " Valid method numbers are between 0 and %u.\n", COMPRESSION_METHODS - 1);
|
||||
fputs(" -c<name>, --compressor <name> Use the specified compressor: the method\n", stderr);
|
||||
fputs(" number will be relative to that compressor.\n", stderr);
|
||||
fputs(" Any prefix of the compressor name may be\n", stderr);
|
||||
fputs(" specified. Use * to indicate any compressor.\n", stderr);
|
||||
fputs(" -a<number>, --align <number> Pad the compressed output with zeros until\n", stderr);
|
||||
fputs(" the size has the specified number of low bits\n", stderr);
|
||||
fputs(" cleared (default: 0).\n", stderr);
|
||||
fputs("The source and output filenames can be given as - (or omitted) to use standard\n", stderr);
|
||||
fputs("input and output. Use -- to indicate that subsequent arguments are file names.\n", stderr);
|
||||
exit(3);
|
||||
}
|
||||
|
||||
noreturn list_compressors (void) {
|
||||
const struct compressor * compressor;
|
||||
unsigned current, length = 10;
|
||||
for (compressor = compressors; compressor -> name; compressor ++) if ((current = strlen(compressor -> name)) > length) length = current;
|
||||
fprintf(stderr, "%-*s Offset Methods\n", length, "Compressor");
|
||||
for (current = 0; current < length; current ++) putc('-', stderr);
|
||||
fputs(" ------ -------\n", stderr);
|
||||
current = 0;
|
||||
for (compressor = compressors; compressor -> name; compressor ++) {
|
||||
fprintf(stderr, "%-*s %6u %7u\n", length, compressor -> name, current, compressor -> methods);
|
||||
current += compressor -> methods;
|
||||
}
|
||||
putc('\n', stderr);
|
||||
fputs("Note: the offset indicates the compressor's lowest method number when the\n", stderr);
|
||||
fputs("--compressor option is not given. When that option is used, every compressor's\n", stderr);
|
||||
fputs("methods are numbered from zero.\n", stderr);
|
||||
exit(3);
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
void write_commands_to_textfile (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream,
|
||||
unsigned char alignment) {
|
||||
FILE * fp = file ? fopen(file, "w") : stdout;
|
||||
if (!fp) error_exit(1, "could not open file %s for writing", file);
|
||||
unsigned length = 0;
|
||||
while (count --) {
|
||||
write_command_to_textfile(fp, *commands, input_stream);
|
||||
length += command_size(*(commands ++));
|
||||
}
|
||||
if (fputs("\tlzend\n", fp) < 0) error_exit(1, "could not write terminator to compressed output");
|
||||
length = ~length & ((1 << alignment) - 1);
|
||||
if (length --) {
|
||||
int rv = fputs("\tdb 0", fp);
|
||||
while ((rv >= 0) && length --) rv = fputs(", 0", fp);
|
||||
if (rv >= 0) rv = -(putc('\n', fp) == EOF);
|
||||
if (rv < 0) error_exit(1, "could not write padding to compressed output");
|
||||
}
|
||||
if (file) fclose(fp);
|
||||
}
|
||||
|
||||
void write_commands_and_padding_to_textfile (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream,
|
||||
unsigned padding_offset, unsigned padding_size) {
|
||||
FILE * fp = file ? fopen(file, "w") : stdout;
|
||||
if (!fp) error_exit(1, "could not open file %s for writing", file);
|
||||
while (count --) write_command_to_textfile(fp, *(commands ++), input_stream);
|
||||
if (fputs("\tlzend\n", fp) < 0) error_exit(1, "could not write terminator to compressed output");
|
||||
if (padding_size) {
|
||||
input_stream += padding_offset;
|
||||
int rv = 0;
|
||||
unsigned pos;
|
||||
const char * prefix = "\tdb";
|
||||
for (pos = 0; (rv >= 0) && (pos < padding_size); pos ++) {
|
||||
if (input_stream[pos])
|
||||
rv = fprintf(fp, "%s $%02hhx", prefix, input_stream[pos]);
|
||||
else
|
||||
rv = fprintf(fp, "%s 0", prefix);
|
||||
prefix = ",";
|
||||
}
|
||||
if (rv >= 0) rv = -(putc('\n', fp) == EOF);
|
||||
if (rv < 0) error_exit(1, "could not write padding to compressed output");
|
||||
}
|
||||
if (file) fclose(fp);
|
||||
}
|
||||
|
||||
void write_command_to_textfile (FILE * fp, struct command command, const unsigned char * input_stream) {
|
||||
if ((!command.count) || (command.count > MAX_COMMAND_COUNT)) error_exit(2, "invalid command in output stream");
|
||||
int rv, pos;
|
||||
const char * kind;
|
||||
switch (command.command) {
|
||||
case 0:
|
||||
if ((rv = fprintf(fp, "\tlzdata")) < 0) break;
|
||||
for (pos = 0; pos < command.count; pos ++) if ((rv = fprintf(fp, "%s$%02hhx", pos ? ", " : " ", input_stream[command.value + pos])) < 0) break;
|
||||
rv = putc('\n', fp);
|
||||
break;
|
||||
case 1:
|
||||
if ((command.value < 0) || (command.value > 255)) error_exit(2, "invalid command in output stream");
|
||||
rv = fprintf(fp, "\tlzrepeat %u, $%02hhx\n", command.count, (unsigned char) command.value);
|
||||
break;
|
||||
case 2:
|
||||
if (command.value < 0) error_exit(2, "invalid command in output stream");
|
||||
rv = fprintf(fp, "\tlzrepeat %u, $%02hhx, $%02hhx\n", command.count, (unsigned char) command.value, (unsigned char) (command.value >> 8));
|
||||
break;
|
||||
case 3:
|
||||
rv = fprintf(fp, "\tlzzero %u\n", command.count);
|
||||
break;
|
||||
case 4:
|
||||
kind = "normal";
|
||||
goto copy;
|
||||
case 5:
|
||||
kind = "flipped";
|
||||
goto copy;
|
||||
case 6:
|
||||
kind = "reversed";
|
||||
copy:
|
||||
if ((command.value < -LOOKBACK_LIMIT) || (command.value >= MAX_FILE_SIZE)) error_exit(2, "invalid command in output stream");
|
||||
if (command.value < 0)
|
||||
rv = fprintf(fp, "\tlzcopy %s, %u, %d\n", kind, command.count, command.value);
|
||||
else
|
||||
rv = fprintf(fp, "\tlzcopy %s, %u, $%04hx\n", kind, command.count, (unsigned short) command.value);
|
||||
break;
|
||||
default:
|
||||
error_exit(2, "invalid command in output stream");
|
||||
}
|
||||
if (rv < 0) error_exit(1, "could not write command to compressed output");
|
||||
}
|
||||
|
||||
void write_commands_to_file (const char * file, const struct command * commands, unsigned count, const unsigned char * input_stream, unsigned char alignment) {
|
||||
FILE * fp = file ? fopen(file, "wb") : stdout;
|
||||
if (!fp) error_exit(1, "could not open file %s for writing", file);
|
||||
unsigned length = 0;
|
||||
while (count --) {
|
||||
write_command_to_file(fp, *commands, input_stream);
|
||||
length += command_size(*(commands ++));
|
||||
}
|
||||
if (putc(-1, fp) == EOF) error_exit(1, "could not write terminator to compressed output");
|
||||
length = ~length & ((1 << alignment) - 1);
|
||||
while (length --) if (putc(0, fp) == EOF) error_exit(1, "could not write padding to compressed output");
|
||||
if (file) fclose(fp);
|
||||
}
|
||||
|
||||
void write_command_to_file (FILE * fp, struct command command, const unsigned char * input_stream) {
|
||||
if ((!command.count) || (command.count > MAX_COMMAND_COUNT)) error_exit(2, "invalid command in output stream");
|
||||
unsigned char buf[4];
|
||||
unsigned char * pos = buf;
|
||||
int n;
|
||||
command.count --;
|
||||
if (command.count < SHORT_COMMAND_COUNT)
|
||||
*(pos ++) = (command.command << 5) + command.count;
|
||||
else {
|
||||
*(pos ++) = 224 + (command.command << 2) + (command.count >> 8);
|
||||
*(pos ++) = command.count;
|
||||
}
|
||||
switch (command.command) {
|
||||
case 1: case 2:
|
||||
if ((command.value < 0) || (command.value >= (1 << (command.command << 3)))) error_exit(2, "invalid command in output stream");
|
||||
for (n = 0; n < command.command; n ++) *(pos ++) = command.value >> (n << 3);
|
||||
case 0: case 3:
|
||||
break;
|
||||
default:
|
||||
if ((command.value < -LOOKBACK_LIMIT) || (command.value >= MAX_FILE_SIZE)) error_exit(2, "invalid command in output stream");
|
||||
if (command.value < 0)
|
||||
*(pos ++) = command.value ^ 127;
|
||||
else {
|
||||
*(pos ++) = command.value >> 8;
|
||||
*(pos ++) = command.value;
|
||||
}
|
||||
}
|
||||
if (fwrite(buf, 1, pos - buf, fp) != (pos - buf)) error_exit(1, "could not write command to compressed output");
|
||||
if (command.command) return;
|
||||
command.count ++;
|
||||
if (fwrite(input_stream + command.value, 1, command.count, fp) != command.count) error_exit(1, "could not write data to compressed output");
|
||||
}
|
||||
|
||||
void write_raw_data_to_file (const char * file, const void * data, unsigned length) {
|
||||
FILE * fp = file ? fopen(file, "w") : stdout;
|
||||
if (!fp) error_exit(1, "could not open file %s for writing", file);
|
||||
while (length) {
|
||||
unsigned rv = fwrite(data, 1, length, fp);
|
||||
if (!rv) error_exit(1, "could not write raw data to output");
|
||||
data = (const char *) data + rv;
|
||||
length -= rv;
|
||||
}
|
||||
if (file) fclose(fp);
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
void optimize (struct command * commands, unsigned short count) {
|
||||
while (count && (commands -> command == 7)) commands ++, count --;
|
||||
if (count < 2) return;
|
||||
struct command * end = commands + count;
|
||||
struct command * next;
|
||||
for (next = commands + 1; next < end; next ++) {
|
||||
if (next -> command == 7) continue;
|
||||
if (
|
||||
!(commands -> command) &&
|
||||
(command_size(*next) == next -> count) &&
|
||||
((commands -> count + next -> count) <= MAX_COMMAND_COUNT) &&
|
||||
((commands -> count > SHORT_COMMAND_COUNT) || ((commands -> count + next -> count) <= SHORT_COMMAND_COUNT))
|
||||
) {
|
||||
commands -> count += next -> count;
|
||||
next -> command = 7;
|
||||
continue;
|
||||
}
|
||||
if (next -> command == commands -> command)
|
||||
switch (commands -> command) {
|
||||
case 0:
|
||||
if ((commands -> value + commands -> count) != next -> value) break;
|
||||
commands -> count += next -> count;
|
||||
next -> command = 7;
|
||||
if (commands -> count <= MAX_COMMAND_COUNT) continue;
|
||||
next -> command = 0;
|
||||
next -> value = commands -> value + MAX_COMMAND_COUNT;
|
||||
next -> count = commands -> count - MAX_COMMAND_COUNT;
|
||||
commands -> count = MAX_COMMAND_COUNT;
|
||||
break;
|
||||
case 1:
|
||||
if (commands -> value != next -> value) break;
|
||||
// fallthrough
|
||||
case 3:
|
||||
if ((commands -> count + next -> count) <= MAX_COMMAND_COUNT) {
|
||||
commands -> count += next -> count;
|
||||
next -> command = 7;
|
||||
continue;
|
||||
}
|
||||
next -> count = (commands -> count + next -> count) - MAX_COMMAND_COUNT;
|
||||
commands -> count = MAX_COMMAND_COUNT;
|
||||
}
|
||||
commands = next;
|
||||
}
|
||||
}
|
||||
|
||||
void repack (struct command ** commands, unsigned short * length) {
|
||||
struct command * new_commands = malloc(sizeof(struct command) * *length);
|
||||
struct command * current = new_commands;
|
||||
unsigned short p;
|
||||
for (p = 0; p < *length; p ++) if (p[*commands].command != 7) *(current ++) = p[*commands];
|
||||
free(*commands);
|
||||
*commands = new_commands;
|
||||
*length = current - new_commands;
|
||||
}
|
||||
107
tools/lz/proto.h
107
tools/lz/proto.h
@@ -1,107 +0,0 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#define NUM_COMPRESSORS 4
|
||||
#define COMPRESSION_METHODS 96 /* sum of all values for the methods field in compressors */
|
||||
#define MAX_FILE_SIZE 32768
|
||||
#define SHORT_COMMAND_COUNT 32
|
||||
#define MAX_COMMAND_COUNT 1024
|
||||
#define LOOKBACK_LIMIT 128 /* highest negative valid count for a copy command */
|
||||
#define LOOKAHEAD_LIMIT 3072 /* maximum lookahead distance for the first pass of multi-pass compression */
|
||||
#define MULTIPASS_SKIP_THRESHOLD 64
|
||||
|
||||
#if __STDC_VERSION__ >= 201112L
|
||||
// <noreturn.h> forces "noreturn void", which is silly and redundant; this is simpler
|
||||
#define noreturn _Noreturn void
|
||||
#else
|
||||
#define noreturn void /* fallback */
|
||||
#endif
|
||||
|
||||
struct command {
|
||||
unsigned command: 3; // commands 0-6 as per compression spec; command 7 is used as a dummy placeholder
|
||||
unsigned count: 12; // always equals the uncompressed data length
|
||||
signed value: 17; // offset for commands 0 (into source) and 4-6 (into decompressed output); repeated bytes for commands 1-2
|
||||
};
|
||||
|
||||
struct compressor {
|
||||
unsigned methods;
|
||||
const char * name;
|
||||
struct command * (* function) (const unsigned char *, const unsigned char *, unsigned short *, unsigned);
|
||||
};
|
||||
|
||||
struct options {
|
||||
const char * input;
|
||||
const char * output;
|
||||
unsigned method; // method to use, or >= COMPRESSION_METHODS to try them all
|
||||
unsigned char mode; // 0: compress, 1: compress to text, 2: uncompress, 3: dump commands as text
|
||||
unsigned char alignment; // 1 << value
|
||||
};
|
||||
|
||||
// global.c
|
||||
extern const struct compressor compressors[];
|
||||
extern const unsigned char bit_flipping_table[];
|
||||
extern char option_name_buffer[];
|
||||
|
||||
// main.c
|
||||
int main(int, char **);
|
||||
struct command * compress(const unsigned char *, unsigned short *, unsigned);
|
||||
|
||||
// merging.c
|
||||
struct command * select_optimal_sequence(struct command **, const unsigned short *, unsigned short *);
|
||||
struct command * select_command_sequence(struct command **, const unsigned short *, int, unsigned short *);
|
||||
struct command * merge_command_sequences(const struct command *, unsigned short, const struct command *, unsigned short, unsigned short *);
|
||||
|
||||
// mpcomp.c
|
||||
struct command * try_compress_multi_pass(const unsigned char *, const unsigned char *, unsigned short *, unsigned);
|
||||
struct command pick_command_for_pass(const unsigned char *, const unsigned char *, const unsigned char *, const short *, unsigned short,
|
||||
unsigned short, unsigned);
|
||||
struct command pick_repetition_for_pass(const unsigned char *, unsigned short, unsigned short, unsigned);
|
||||
struct command pick_copy_for_pass(const unsigned char *, const unsigned char *, const short *, unsigned char, unsigned short, unsigned short, unsigned);
|
||||
|
||||
// nullcomp.c
|
||||
struct command * store_uncompressed(const unsigned char *, const unsigned char *, unsigned short *, unsigned);
|
||||
|
||||
// options.c
|
||||
struct options get_options(int, char **);
|
||||
unsigned parse_numeric_option_argument(char ***, unsigned);
|
||||
int parse_compressor_option_argument(char ***);
|
||||
const char * get_argument_for_option(char ***, const char **);
|
||||
noreturn usage(const char *);
|
||||
noreturn list_compressors(void);
|
||||
|
||||
// output.c
|
||||
void write_commands_to_textfile(const char *, const struct command *, unsigned, const unsigned char *, unsigned char);
|
||||
void write_commands_and_padding_to_textfile(const char *, const struct command *, unsigned, const unsigned char *, unsigned, unsigned);
|
||||
void write_command_to_textfile(FILE *, struct command, const unsigned char *);
|
||||
void write_commands_to_file(const char *, const struct command *, unsigned, const unsigned char *, unsigned char);
|
||||
void write_command_to_file(FILE *, struct command, const unsigned char *);
|
||||
void write_raw_data_to_file(const char *, const void *, unsigned);
|
||||
|
||||
// packing.c
|
||||
void optimize(struct command *, unsigned short);
|
||||
void repack(struct command **, unsigned short *);
|
||||
|
||||
// repcomp.c
|
||||
struct command * try_compress_repetitions(const unsigned char *, const unsigned char *, unsigned short *, unsigned);
|
||||
struct command find_repetition_at_position(const unsigned char *, unsigned short, unsigned short);
|
||||
|
||||
// spcomp.c
|
||||
struct command * try_compress_single_pass(const unsigned char *, const unsigned char *, unsigned short *, unsigned);
|
||||
struct command find_best_copy(const unsigned char *, unsigned short, unsigned short, const unsigned char *, unsigned);
|
||||
unsigned short scan_forwards(const unsigned char *, unsigned short, const unsigned char *, unsigned short, short *);
|
||||
unsigned short scan_backwards(const unsigned char *, unsigned short, unsigned short, short *);
|
||||
struct command find_best_repetition(const unsigned char *, unsigned short, unsigned short);
|
||||
|
||||
// uncomp.c
|
||||
struct command * get_commands_from_file(const unsigned char *, unsigned short * restrict, unsigned short * restrict);
|
||||
unsigned char * get_uncompressed_data(const struct command *, const unsigned char *, unsigned short *);
|
||||
|
||||
// util.c
|
||||
noreturn error_exit(int, const char *, ...);
|
||||
unsigned char * read_file_into_buffer(const char *, unsigned short *);
|
||||
struct command pick_best_command(unsigned, struct command, ...);
|
||||
int is_better(struct command, struct command);
|
||||
short command_size(struct command);
|
||||
unsigned short compressed_length(const struct command *, unsigned short);
|
||||
@@ -1,63 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
/*
|
||||
Repetitions compressor: compresses the data only using a subset of the available repetition commands.
|
||||
Methods defined: 6
|
||||
Flags values: the value plus one is taken as a bitfield indicating which kinds of repetition commands are used
|
||||
(lowest bit to highest: repeat single byte (1), repeat two bytes (2), repeat zeros (3)).
|
||||
*/
|
||||
|
||||
struct command * try_compress_repetitions (const unsigned char * data, __attribute__((unused)) const unsigned char * bitflipped, unsigned short * size, unsigned flags) {
|
||||
unsigned short pos = 0, skipped = 0;
|
||||
struct command * result = malloc(*size * sizeof(struct command));
|
||||
struct command * current = result;
|
||||
struct command candidate;
|
||||
flags = (flags + 1) << 1;
|
||||
while (pos < *size) {
|
||||
candidate = find_repetition_at_position(data, pos, *size);
|
||||
if ((candidate.command == 3) && !(flags & 8)) {
|
||||
candidate.command = 1;
|
||||
candidate.value = 0;
|
||||
}
|
||||
if ((candidate.command == 1) && !(flags & 2)) {
|
||||
candidate.command = 2;
|
||||
candidate.value |= candidate.value << 8;
|
||||
}
|
||||
if ((flags & (1 << candidate.command)) && (command_size(candidate) <= candidate.count)) {
|
||||
if (skipped) *(current ++) = (struct command) {.command = 0, .count = skipped, .value = pos - skipped};
|
||||
skipped = 0;
|
||||
*(current ++) = candidate;
|
||||
pos += candidate.count;
|
||||
} else {
|
||||
pos ++;
|
||||
if ((++ skipped) == MAX_COMMAND_COUNT) {
|
||||
*(current ++) = (struct command) {.command = 0, .count = MAX_COMMAND_COUNT, .value = pos - MAX_COMMAND_COUNT};
|
||||
skipped = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (skipped) *(current ++) = (struct command) {.command = 0, .count = skipped, .value = pos - skipped};
|
||||
*size = current - result;
|
||||
result = realloc(result, *size * sizeof(struct command));
|
||||
return result;
|
||||
}
|
||||
|
||||
struct command find_repetition_at_position (const unsigned char * data, unsigned short position, unsigned short length) {
|
||||
if ((position + 1) >= length) return data[position] ? ((struct command) {.command = 7}) : ((struct command) {.command = 3, .count = 1});
|
||||
unsigned char value[2] = {data[position], data[position + 1]};
|
||||
unsigned repcount, limit = length - position;
|
||||
if (limit > MAX_COMMAND_COUNT) limit = MAX_COMMAND_COUNT;
|
||||
for (repcount = 2; (repcount < limit) && (data[position + repcount] == value[repcount & 1]); repcount ++);
|
||||
struct command result;
|
||||
result.count = repcount;
|
||||
if (*value != value[1]) {
|
||||
if (!*value && (repcount < 3)) return (struct command) {.command = 3, .count = 1};
|
||||
result.command = 2;
|
||||
result.value = ((unsigned) (*value)) | (((unsigned) (value[1])) << 8);
|
||||
} else if (*value) {
|
||||
result.command = 1;
|
||||
result.value = *value;
|
||||
} else
|
||||
result.command = 3;
|
||||
return result;
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
/*
|
||||
Single-pass compressor: attempts to compress the data in a single pass, selecting the best command at each
|
||||
position within some constraints.
|
||||
Methods defined: 72
|
||||
Flags values:
|
||||
Bit fields (will trigger alternate behavior if set):
|
||||
1: prefer repetition commands over copy commands of equal count
|
||||
2: don't emit a copy or repetition with a count equal to its size when the previous command is a literal (0) that
|
||||
is not at maximum size (32 or 1024)
|
||||
4: don't emit long copy commands
|
||||
Selector values (pick one from each group and add them to the bit fields):
|
||||
- Scan delay: number of bytes that are forced into literal (0) commands after each non-literal command:
|
||||
0: 0 bytes
|
||||
8: 1 byte
|
||||
16: 2 bytes
|
||||
- Copy command preference (when the command counts are tied), in order from most to least:
|
||||
0: normal (4), reversed (6), flipped (5)
|
||||
24: reversed (6), flipped (5), normal (4)
|
||||
48: flipped (5), reversed (6), normal (4)
|
||||
*/
|
||||
|
||||
struct command * try_compress_single_pass (const unsigned char * data, const unsigned char * bitflipped, unsigned short * length, unsigned flags) {
|
||||
struct command * commands = malloc(sizeof(struct command) * *length);
|
||||
memset(commands, -1, sizeof(struct command) * *length);
|
||||
struct command * current_command = commands;
|
||||
unsigned short position = 0, previous_data = 0;
|
||||
unsigned char scan_delay = 0, scan_delay_flag = (flags >> 3) % 3;
|
||||
struct command copy, repetition;
|
||||
while (position < *length) {
|
||||
copy = find_best_copy(data, position, *length, bitflipped, flags);
|
||||
repetition = find_best_repetition(data, position, *length);
|
||||
if (flags & 1)
|
||||
*current_command = pick_best_command(2, repetition, copy);
|
||||
else
|
||||
*current_command = pick_best_command(2, copy, repetition);
|
||||
*current_command = pick_best_command(2, (struct command) {.command = 0, .count = 1, .value = position}, *current_command);
|
||||
if ((flags & 2) && (command_size(*current_command) == current_command -> count))
|
||||
if (previous_data && (previous_data != SHORT_COMMAND_COUNT) && (previous_data != MAX_COMMAND_COUNT))
|
||||
*current_command = (struct command) {.command = 0, .count = 1, .value = position};
|
||||
if (scan_delay_flag) {
|
||||
if (scan_delay >= scan_delay_flag)
|
||||
scan_delay = 0;
|
||||
else if (current_command -> command) {
|
||||
scan_delay ++;
|
||||
*current_command = (struct command) {.command = 0, .count = 1, .value = position};
|
||||
}
|
||||
}
|
||||
if (current_command -> command)
|
||||
previous_data = 0;
|
||||
else
|
||||
previous_data += current_command -> count;
|
||||
position += (current_command ++) -> count;
|
||||
}
|
||||
optimize(commands, current_command - commands);
|
||||
repack(&commands, length);
|
||||
return commands;
|
||||
}
|
||||
|
||||
struct command find_best_copy (const unsigned char * data, unsigned short position, unsigned short length, const unsigned char * bitflipped, unsigned flags) {
|
||||
struct command simple = {.command = 7};
|
||||
struct command flipped = simple, backwards = simple;
|
||||
short count, offset;
|
||||
if ((count = scan_forwards(data + position, length - position, data, position, &offset)))
|
||||
simple = (struct command) {.command = 4, .count = count, .value = offset};
|
||||
if ((count = scan_forwards(data + position, length - position, bitflipped, position, &offset)))
|
||||
flipped = (struct command) {.command = 5, .count = count, .value = offset};
|
||||
if ((count = scan_backwards(data, length - position, position, &offset)))
|
||||
backwards = (struct command) {.command = 6, .count = count, .value = offset};
|
||||
struct command command;
|
||||
switch (flags / 24) {
|
||||
case 0: command = pick_best_command(3, simple, backwards, flipped); break;
|
||||
case 1: command = pick_best_command(3, backwards, flipped, simple); break;
|
||||
case 2: command = pick_best_command(3, flipped, backwards, simple);
|
||||
}
|
||||
if ((flags & 4) && (command.count > SHORT_COMMAND_COUNT)) command.count = SHORT_COMMAND_COUNT;
|
||||
return command;
|
||||
}
|
||||
|
||||
unsigned short scan_forwards (const unsigned char * target, unsigned short limit, const unsigned char * source, unsigned short real_position, short * offset) {
|
||||
unsigned short best_match, best_length = 0;
|
||||
unsigned short current_length;
|
||||
unsigned short position;
|
||||
for (position = 0; position < real_position; position ++) {
|
||||
if (source[position] != *target) continue;
|
||||
for (current_length = 0; (current_length < limit) && (source[position + current_length] == target[current_length]); current_length ++);
|
||||
if (current_length > MAX_COMMAND_COUNT) current_length = MAX_COMMAND_COUNT;
|
||||
if (current_length < best_length) continue;
|
||||
best_match = position;
|
||||
best_length = current_length;
|
||||
}
|
||||
if (!best_length) return 0;
|
||||
if ((best_match + LOOKBACK_LIMIT) >= real_position)
|
||||
*offset = best_match - real_position;
|
||||
else
|
||||
*offset = best_match;
|
||||
return best_length;
|
||||
}
|
||||
|
||||
unsigned short scan_backwards (const unsigned char * data, unsigned short limit, unsigned short real_position, short * offset) {
|
||||
if (real_position < limit) limit = real_position;
|
||||
unsigned short best_match, best_length = 0;
|
||||
unsigned short current_length;
|
||||
unsigned short position;
|
||||
for (position = 0; position < real_position; position ++) {
|
||||
if (data[position] != data[real_position]) continue;
|
||||
for (current_length = 0; (current_length <= position) && (current_length < limit) &&
|
||||
(data[position - current_length] == data[real_position + current_length]); current_length ++);
|
||||
if (current_length > MAX_COMMAND_COUNT) current_length = MAX_COMMAND_COUNT;
|
||||
if (current_length < best_length) continue;
|
||||
best_match = position;
|
||||
best_length = current_length;
|
||||
}
|
||||
if (!best_length) return 0;
|
||||
if ((best_match + LOOKBACK_LIMIT) >= real_position)
|
||||
*offset = best_match - real_position;
|
||||
else
|
||||
*offset = best_match;
|
||||
return best_length;
|
||||
}
|
||||
|
||||
struct command find_best_repetition (const unsigned char * data, unsigned short position, unsigned short length) {
|
||||
if ((position + 1) >= length) return data[position] ? ((struct command) {.command = 7}) : ((struct command) {.command = 3, .count = 1});
|
||||
unsigned char value[2] = {data[position], data[position + 1]};
|
||||
unsigned repcount, limit = length - position;
|
||||
if (limit > MAX_COMMAND_COUNT) limit = MAX_COMMAND_COUNT;
|
||||
for (repcount = 2; (repcount < limit) && (data[position + repcount] == value[repcount & 1]); repcount ++);
|
||||
struct command result;
|
||||
result.count = repcount;
|
||||
if (*value != value[1]) {
|
||||
if (!*value && (repcount < 3)) return (struct command) {.command = 3, .count = 1};
|
||||
result.command = 2;
|
||||
result.value = ((unsigned) (*value)) | (((unsigned) (value[1])) << 8);
|
||||
} else if (*value) {
|
||||
result.command = 1;
|
||||
result.value = *value;
|
||||
} else
|
||||
result.command = 3;
|
||||
return result;
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
struct command * get_commands_from_file (const unsigned char * data, unsigned short * restrict size, unsigned short * restrict slack) {
|
||||
struct command * result = malloc(*size * sizeof(struct command));
|
||||
unsigned short remaining = *size;
|
||||
struct command * current = result;
|
||||
const unsigned char * rp = data;
|
||||
while (1) {
|
||||
if (!(remaining --)) goto error;
|
||||
current -> command = *rp >> 5;
|
||||
current -> count = *(rp ++) & 31;
|
||||
if (current -> command == 7) {
|
||||
current -> command = current -> count >> 2;
|
||||
current -> count = (current -> count & 3) << 8;
|
||||
if (current -> command == 7) {
|
||||
// long commands inside long commands are not allowed, but if the count is 0x300 here, it means that the original byte was 0xff
|
||||
if (current -> count == 0x300) break;
|
||||
goto error;
|
||||
}
|
||||
if (!(remaining --)) goto error;
|
||||
current -> count |= *(rp ++);
|
||||
}
|
||||
current -> count ++;
|
||||
switch (current -> command) {
|
||||
case 0:
|
||||
if (remaining <= current -> count) goto error;
|
||||
current -> value = rp - data;
|
||||
rp += current -> count;
|
||||
remaining -= current -> count;
|
||||
case 3:
|
||||
break;
|
||||
case 1: case 2: {
|
||||
unsigned char p;
|
||||
if (remaining <= current -> command) goto error;
|
||||
current -> value = 0;
|
||||
for (p = 0; p < current -> command; p ++) current -> value |= *(rp ++) << (p << 3);
|
||||
remaining -= current -> command;
|
||||
} break;
|
||||
default:
|
||||
if (!(remaining --)) goto error;
|
||||
if ((current -> value = *(rp ++)) & 128)
|
||||
current -> value = 127 - current -> value;
|
||||
else {
|
||||
if (!(remaining --)) goto error;
|
||||
current -> value = (current -> value << 8) | *(rp ++);
|
||||
}
|
||||
}
|
||||
current ++;
|
||||
}
|
||||
if (slack) *slack = *size - (rp - data);
|
||||
*size = current - result;
|
||||
return realloc(result, (*size ? *size : 1) * sizeof(struct command));
|
||||
error:
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
unsigned char * get_uncompressed_data (const struct command * commands, const unsigned char * compressed, unsigned short * size) {
|
||||
const struct command * limit = commands + *size;
|
||||
unsigned char * result = malloc(MAX_FILE_SIZE + MAX_COMMAND_COUNT);
|
||||
unsigned char * current = result;
|
||||
unsigned short p;
|
||||
for (; commands < limit; commands ++) {
|
||||
switch (commands -> command) {
|
||||
case 0:
|
||||
memcpy(current, compressed + commands -> value, commands -> count);
|
||||
current += commands -> count;
|
||||
break;
|
||||
case 1: case 2:
|
||||
for (p = 0; p < commands -> count; p ++) *(current ++) = commands -> value >> ((p % commands -> command) << 3);
|
||||
break;
|
||||
case 3:
|
||||
memset(current, 0, commands -> count);
|
||||
current += commands -> count;
|
||||
break;
|
||||
default: {
|
||||
const unsigned char * ref = ((commands -> value < 0) ? current : result) + commands -> value;
|
||||
for (p = 0; p < commands -> count; p ++) {
|
||||
current[p] = ref[(commands -> command == 6) ? -(int) p : p];
|
||||
if (commands -> command == 5) current[p] = bit_flipping_table[current[p]];
|
||||
}
|
||||
current += commands -> count;
|
||||
}
|
||||
}
|
||||
if ((current - result) > MAX_FILE_SIZE) {
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
*size = current - result;
|
||||
return realloc(result, *size ? *size : 1);
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
#include "proto.h"
|
||||
|
||||
noreturn error_exit (int error_code, const char * error, ...) {
|
||||
va_list ap;
|
||||
va_start(ap, error);
|
||||
fputs("error: ", stderr);
|
||||
vfprintf(stderr, error, ap);
|
||||
va_end(ap);
|
||||
fputc('\n', stderr);
|
||||
exit(error_code);
|
||||
}
|
||||
|
||||
unsigned char * read_file_into_buffer (const char * file, unsigned short * size) {
|
||||
FILE * fp = file ? fopen(file, "rb") : stdin;
|
||||
if (!fp) error_exit(1, "could not open file %s for reading", file);
|
||||
unsigned char * buf = malloc(MAX_FILE_SIZE + 1);
|
||||
int rv = fread(buf, 1, MAX_FILE_SIZE + 1, fp);
|
||||
if (file) fclose(fp);
|
||||
if (rv < 0) error_exit(1, "could not read from file %s", file);
|
||||
if (rv > MAX_FILE_SIZE) error_exit(1, "file %s is too big", file ? file : "<standard input>");
|
||||
*size = rv;
|
||||
return buf;
|
||||
}
|
||||
|
||||
struct command pick_best_command (unsigned count, struct command command, ...) {
|
||||
struct command result = command;
|
||||
va_list ap;
|
||||
va_start(ap, command);
|
||||
while (-- count) {
|
||||
command = va_arg(ap, struct command);
|
||||
if (is_better(command, result)) result = command;
|
||||
}
|
||||
va_end(ap);
|
||||
return result;
|
||||
}
|
||||
|
||||
int is_better (struct command new, struct command old) {
|
||||
if (new.command == 7) return 0;
|
||||
if (old.command == 7) return 1;
|
||||
short new_savings = new.count - command_size(new), old_savings = old.count - command_size(old);
|
||||
return new_savings > old_savings;
|
||||
}
|
||||
|
||||
short command_size (struct command command) {
|
||||
short header_size = 1 + (command.count > SHORT_COMMAND_COUNT);
|
||||
if (command.command & 4) return header_size + 1 + (command.value >= 0);
|
||||
return header_size + command.command[(short []) {command.count, 1, 2, 0}];
|
||||
}
|
||||
|
||||
unsigned short compressed_length (const struct command * commands, unsigned short count) {
|
||||
unsigned short current, total = 0;
|
||||
for (current = 0; current < count; current ++) if (commands[current].command != 7) total += command_size(commands[current]);
|
||||
return total;
|
||||
}
|
||||
841
tools/lzcompress.c
Normal file
841
tools/lzcompress.c
Normal file
@@ -0,0 +1,841 @@
|
||||
#define PROGRAM_NAME "lzcompress"
|
||||
#define USAGE_OPTS "[-h|--help] [-u|--uncompress] [-m|--matching [-bcdilnr]] [-a|--align alignment] infile outfile"
|
||||
|
||||
#include "common.h"
|
||||
|
||||
/******************************************** LZ format *******************************************/
|
||||
|
||||
#define SHORT_MAX_LENGTH (1 << 5) // Commands have a 5-bit length
|
||||
#define LONG_MAX_LENGTH (1 << 10) // LZ_LONG extends length to 10 bits
|
||||
|
||||
// Offsets for lookback commands are 7 bits; above that they use a 16-bit index
|
||||
#define LOOKBACK_MAX_OFFSET (1 << 7)
|
||||
|
||||
enum lz_command {
|
||||
LZ_LITERAL = 0, // Read literal data for N bytes
|
||||
// Sequence commands
|
||||
LZ_ITERATE = 1, // Write the same byte for N bytes
|
||||
LZ_ALTERNATE = 2, // Alternate two bytes for N bytes
|
||||
LZ_ZERO = 3, // Write 0 for N bytes
|
||||
// Lookback commands
|
||||
LZ_REPEAT = 4, // Repeat N bytes from the offset
|
||||
LZ_FLIP = 5, // Repeat N bit-flipped bytes
|
||||
LZ_REVERSE = 6, // Repeat N bytes in reverse
|
||||
// Extended commands
|
||||
LZ_LONG = 7, // Use a new command with a long N
|
||||
// Sentinel commands
|
||||
LZ_END = 0xff, // Terminate compressed data
|
||||
};
|
||||
|
||||
struct command {
|
||||
enum lz_command cmd; // Picked compression command
|
||||
unsigned int length; // Command length N
|
||||
int offset; // Offset for lookback commands; bytes for sequence commands
|
||||
uint8_t value; // Uncompressed byte value
|
||||
};
|
||||
|
||||
/**************************************** Argument options ****************************************/
|
||||
|
||||
struct Options {
|
||||
bool decompress;
|
||||
bool matching;
|
||||
uint8_t alignment;
|
||||
bool skip_initial_byte;
|
||||
bool no_lookback_3;
|
||||
bool odd_alternate;
|
||||
bool iterate_only;
|
||||
bool literal_only;
|
||||
bool long_32;
|
||||
bool prefer_alternate;
|
||||
};
|
||||
|
||||
struct Options options = {0};
|
||||
|
||||
void parse_args(int argc, char *argv[]) {
|
||||
struct option long_options[] = {
|
||||
{"uncompress", no_argument, 0, 'u'},
|
||||
{"matching", no_argument, 0, 'm'},
|
||||
{"align", required_argument, 0, 'a'},
|
||||
{"skip-initial-byte", no_argument, 0, 'b'},
|
||||
{"no-lookback-3", no_argument, 0, 'c'},
|
||||
{"odd-alternate", no_argument, 0, 'd'},
|
||||
{"iterate-only", no_argument, 0, 'i'},
|
||||
{"literal-only", no_argument, 0, 'l'},
|
||||
{"long-32", no_argument, 0, 'n'},
|
||||
{"prefer-alternate", no_argument, 0, 'r'},
|
||||
{"help", no_argument, 0, 'h'},
|
||||
{0}
|
||||
};
|
||||
for (int opt; (opt = getopt_long(argc, argv, "mua:bcdilnrh", long_options)) != -1;) {
|
||||
switch (opt) {
|
||||
case 'u':
|
||||
options.decompress = true;
|
||||
break;
|
||||
case 'm':
|
||||
options.matching = true;
|
||||
break;
|
||||
case 'a':
|
||||
options.alignment = strtoul(optarg, NULL, 0);
|
||||
break;
|
||||
case 'c':
|
||||
options.no_lookback_3 = true;
|
||||
break;
|
||||
case 'd':
|
||||
options.odd_alternate = true;
|
||||
break;
|
||||
case 'r':
|
||||
options.prefer_alternate = true;
|
||||
break;
|
||||
case 'l':
|
||||
options.literal_only = true;
|
||||
break;
|
||||
case 'i':
|
||||
options.iterate_only = true;
|
||||
break;
|
||||
case 'n':
|
||||
options.long_32 = true;
|
||||
break;
|
||||
case 'b':
|
||||
options.skip_initial_byte = true;
|
||||
break;
|
||||
case 'h':
|
||||
fprintf(stderr, "Usage: " PROGRAM_NAME " " USAGE_OPTS "\n\n");
|
||||
fputs("Flags to adjust --matching compression:\n", stderr);
|
||||
fputs(" -a, --align N Alignment of the final compressed byte (2^N)\n", stderr);
|
||||
fputs(" -b, --skip-initial-byte First byte will always use LZ_LITERAL\n", stderr);
|
||||
fputs(" -c, --no-lookback-3 Don't use LZ_FLIP/LZ_REVERSE/LZ_LONG for length 3\n", stderr);
|
||||
fputs(" -d, --odd-alternate Allow odd-length LZ_ALTERNATE\n", stderr);
|
||||
fputs(" -i, --iterate-only Only use LZ_LITERAL and LZ_ITERATE\n", stderr);
|
||||
fputs(" -l, --literal-only Only use LZ_LITERAL\n", stderr);
|
||||
fputs(" -n, --long-32 Use LZ_LONG for length 32\n", stderr);
|
||||
fputs(" -r, --prefer-alternate Use LZ_ALTERNATE instead of equally-large LZ_ITERATE\n", stderr);
|
||||
exit(0);
|
||||
break;
|
||||
default:
|
||||
usage_exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/************************************** Matching compression **************************************/
|
||||
|
||||
// Above this length, assume the current command is optimal and jump to its end
|
||||
#define SKIP_LOOKBACK_MIN_LENGTH (1 << 6)
|
||||
|
||||
struct list {
|
||||
unsigned int *items;
|
||||
unsigned int size;
|
||||
unsigned int capacity;
|
||||
};
|
||||
|
||||
struct multimap {
|
||||
uint32_t key;
|
||||
struct list *values;
|
||||
struct multimap *next;
|
||||
};
|
||||
|
||||
struct multimap **multimap_find(struct multimap **mp, uint32_t key) {
|
||||
while (*mp && (*mp)->key != key) {
|
||||
mp = &(*mp)->next;
|
||||
}
|
||||
return mp;
|
||||
}
|
||||
|
||||
void multimap_put(struct multimap **mp, uint32_t key, unsigned int value) {
|
||||
mp = multimap_find(mp, key);
|
||||
if (!*mp) {
|
||||
*mp = xcalloc(sizeof(**mp));
|
||||
(*mp)->key = key;
|
||||
(*mp)->values = xcalloc(sizeof(*(*mp)->values));
|
||||
}
|
||||
struct list *values = (*mp)->values;
|
||||
if (values->size == values->capacity) {
|
||||
values->capacity = (values->capacity + 1) * 2;
|
||||
values->items = xrealloc(values->items, values->capacity * sizeof(*values->items));
|
||||
}
|
||||
values->items[values->size++] = value;
|
||||
}
|
||||
|
||||
// Input and output data
|
||||
struct command *commands = NULL;
|
||||
unsigned int commands_size = 0;
|
||||
|
||||
uint8_t iterate_f(unsigned int i, __attribute__((unused)) unsigned int len) {
|
||||
return commands[i].value;
|
||||
}
|
||||
|
||||
uint8_t alternate_f(unsigned int i, unsigned int len) {
|
||||
return commands[i + (len & 1)].value;
|
||||
}
|
||||
|
||||
uint8_t zero_f(__attribute__((unused)) unsigned int i, __attribute__((unused)) unsigned int len) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint8_t plain_f(uint8_t v) {
|
||||
return v;
|
||||
}
|
||||
|
||||
uint8_t flip_f(uint8_t v) {
|
||||
uint8_t f = 0;
|
||||
for (unsigned int b = 0; b < 8; b++) {
|
||||
f |= ((v >> b) & 1) << (7 - b);
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
void try_sequence_command(enum lz_command cmd, unsigned int i, uint8_t (*f)(unsigned int, unsigned int)) {
|
||||
unsigned int len;
|
||||
for (len = 0; i + len < commands_size && commands[i + len].value == f(i, len); len++) {}
|
||||
if (len > 0 && (cmd != LZ_ALTERNATE || commands[i].length < len + options.prefer_alternate)) {
|
||||
commands[i].cmd = cmd;
|
||||
commands[i].length = len;
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t data_key(unsigned int i, uint8_t (*cmd_fn)(uint8_t), int dir) {
|
||||
uint32_t result = 0;
|
||||
for (unsigned int j = 0; j < 4 && i < commands_size; j++, i += dir) {
|
||||
result |= (uint32_t)cmd_fn(commands[i].value) << (uint32_t)(8 * j);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
unsigned int try_lookback_command(
|
||||
enum lz_command cmd,
|
||||
unsigned int i,
|
||||
unsigned int rep_i,
|
||||
uint32_t key,
|
||||
struct multimap *rep_idxs,
|
||||
uint8_t (*cmd_fn)(uint8_t),
|
||||
int dir
|
||||
) {
|
||||
rep_idxs = *multimap_find(&rep_idxs, key);
|
||||
struct list *idxs = rep_idxs ? rep_idxs->values : NULL;
|
||||
|
||||
unsigned int len = 0;
|
||||
for (unsigned int li = 0; idxs && li < idxs->size; li++) {
|
||||
unsigned int prev_i = idxs->items[li];
|
||||
unsigned int rep_len = 0;
|
||||
for (unsigned int j = i, rj = prev_i;
|
||||
j < commands_size && rj < commands_size && commands[j].value == cmd_fn(commands[rj].value);
|
||||
rep_len++, j++, rj += dir) {}
|
||||
if (rep_len > len || (rep_len == len && prev_i > rep_i)) {
|
||||
// This repetition is longer than the last one found,
|
||||
// or it is the same length and closer
|
||||
len = rep_len;
|
||||
rep_i = prev_i;
|
||||
}
|
||||
}
|
||||
if (len >= (unsigned int)(3 + options.no_lookback_3) && commands[i].length < len) {
|
||||
commands[i].cmd = cmd;
|
||||
commands[i].length = len;
|
||||
commands[i].offset = i - rep_i - 1;
|
||||
}
|
||||
return rep_i;
|
||||
}
|
||||
|
||||
void pick_best_command(unsigned int i) {
|
||||
if (options.literal_only) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Try LZ_ITERATE: find sequence of one byte
|
||||
try_sequence_command(LZ_ITERATE, i, iterate_f);
|
||||
|
||||
if (options.iterate_only) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Try LZ_ALTERNATE: find sequence of two alternating bytes
|
||||
try_sequence_command(LZ_ALTERNATE, i, alternate_f);
|
||||
|
||||
// Try LZ_ZERO: find sequence of 0 bytes
|
||||
try_sequence_command(LZ_ZERO, i, zero_f);
|
||||
|
||||
// Assume that a sequence command is optimal; don't try lookback commands
|
||||
if (commands[i].length >= SKIP_LOOKBACK_MIN_LENGTH) {
|
||||
return;
|
||||
}
|
||||
|
||||
unsigned int rep_i = 0;
|
||||
|
||||
// These map a data key (the first four bytes of a data sequence)
|
||||
// to a list of indexes where that data is found
|
||||
static struct multimap *repeat_idxs = NULL, *reverse_idxs = NULL;
|
||||
|
||||
// Try LZ_REPEAT: find repetition of previous data
|
||||
uint32_t repeat_key = data_key(i, plain_f, 1);
|
||||
rep_i = try_lookback_command(LZ_REPEAT, i, rep_i, repeat_key, repeat_idxs, plain_f, 1);
|
||||
|
||||
// Try LZ_FLIP: find repetition of previous bit-flipped data
|
||||
uint32_t flip_key = data_key(i, flip_f, 1);
|
||||
rep_i = try_lookback_command(LZ_FLIP, i, rep_i, flip_key, repeat_idxs, flip_f, 1);
|
||||
|
||||
// Try LZ_REVERSE: find repetition of previous reversed data
|
||||
uint32_t reverse_key = data_key(i, plain_f, -1);
|
||||
rep_i = try_lookback_command(LZ_REVERSE, i, rep_i, repeat_key, reverse_idxs, plain_f, -1);
|
||||
|
||||
multimap_put(&repeat_idxs, repeat_key, i);
|
||||
multimap_put(&reverse_idxs, reverse_key, i);
|
||||
}
|
||||
|
||||
unsigned int command_length(const struct command *command) {
|
||||
switch (command->cmd) {
|
||||
case LZ_LITERAL:
|
||||
return 1;
|
||||
case LZ_ITERATE:
|
||||
return 2;
|
||||
case LZ_ALTERNATE:
|
||||
return 3;
|
||||
case LZ_ZERO:
|
||||
return 1;
|
||||
case LZ_REPEAT:
|
||||
case LZ_FLIP:
|
||||
case LZ_REVERSE:
|
||||
return 2 + (command->offset >= LOOKBACK_MAX_OFFSET);
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x\n", command->cmd);
|
||||
}
|
||||
}
|
||||
|
||||
void pick_optimized_commands(void) {
|
||||
// Pick best commands at each index
|
||||
for (unsigned int i = options.skip_initial_byte; i < commands_size; i++) {
|
||||
pick_best_command(i);
|
||||
if (commands[i].length >= SKIP_LOOKBACK_MIN_LENGTH) {
|
||||
i += commands[i].length - 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Reduce lengths so one command won't overlap a subsequent more optimal command
|
||||
for (unsigned int i = 0; i < commands_size; i++) {
|
||||
for (unsigned int j = 1; j < commands[i].length; j++) {
|
||||
if (commands[i + j].length > commands[i].length) {
|
||||
commands[i].length = j - (!options.odd_alternate && commands[i].cmd == LZ_ALTERNATE && j % 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Don't use commands whose encoding would be longer than literal data
|
||||
for (unsigned int i = 0; i < commands_size; i++) {
|
||||
if (commands[i].length < command_length(&commands[i]) + !options.iterate_only) {
|
||||
commands[i].cmd = LZ_LITERAL;
|
||||
commands[i].length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the lengths of LZ_LITERAL commands
|
||||
for (unsigned int i = 0; i < commands_size; i += commands[i].length) {
|
||||
if (commands[i].cmd == LZ_LITERAL) {
|
||||
unsigned int len;
|
||||
for (len = 1; i + len < commands_size && commands[i + len].cmd == LZ_LITERAL; len++);
|
||||
commands[i].length = len;
|
||||
}
|
||||
}
|
||||
|
||||
// Limit command lengths to LONG_MAX_LENGTH
|
||||
for (unsigned int i = 0; i < commands_size; i += commands[i].length) {
|
||||
if (commands[i].length > LONG_MAX_LENGTH) {
|
||||
commands[i + LONG_MAX_LENGTH].cmd = commands[i].cmd;
|
||||
commands[i + LONG_MAX_LENGTH].length = commands[i].length - LONG_MAX_LENGTH;
|
||||
commands[i].length = LONG_MAX_LENGTH;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void write_commands(FILE *file) {
|
||||
unsigned int compressed_size = 0;
|
||||
|
||||
for (unsigned int i = 0; i < commands_size; i += commands[i].length) {
|
||||
if (commands[i].length + options.long_32 > SHORT_MAX_LENGTH) {
|
||||
fputc((LZ_LONG << 5) | (commands[i].cmd << 2) | ((commands[i].length - 1) >> 8), file);
|
||||
fputc((commands[i].length - 1) & 0xff, file);
|
||||
compressed_size += 2;
|
||||
} else {
|
||||
fputc((commands[i].cmd << 5) | (commands[i].length - 1), file);
|
||||
compressed_size++;
|
||||
}
|
||||
switch (commands[i].cmd) {
|
||||
case LZ_LITERAL:
|
||||
for (unsigned int j = 0; j < commands[i].length; j++) {
|
||||
fputc(commands[i + j].value, file);
|
||||
}
|
||||
compressed_size += commands[i].length;
|
||||
break;
|
||||
case LZ_ITERATE:
|
||||
fputc(commands[i].value, file);
|
||||
compressed_size++;
|
||||
break;
|
||||
case LZ_ALTERNATE:
|
||||
fputc(commands[i].value, file);
|
||||
fputc(commands[i + 1].value, file);
|
||||
compressed_size += 2;
|
||||
break;
|
||||
case LZ_ZERO:
|
||||
break;
|
||||
case LZ_REPEAT:
|
||||
case LZ_FLIP:
|
||||
case LZ_REVERSE:
|
||||
if (commands[i].offset < LOOKBACK_MAX_OFFSET) {
|
||||
fputc(commands[i].offset | 0x80, file);
|
||||
compressed_size++;
|
||||
} else {
|
||||
unsigned int address = i - commands[i].offset - 1;
|
||||
fputc(address >> 8, file);
|
||||
fputc(address & 0xff, file);
|
||||
compressed_size += 2;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x in output\n", commands[i].cmd);
|
||||
}
|
||||
}
|
||||
|
||||
fputc(LZ_END, file);
|
||||
compressed_size++;
|
||||
|
||||
while (options.alignment && compressed_size % (1 << options.alignment)) {
|
||||
fputc(0, file);
|
||||
compressed_size++;
|
||||
}
|
||||
}
|
||||
|
||||
void compress_matching(char const *input_name, char const *output_name) {
|
||||
long input_size;
|
||||
uint8_t *input_data = read_u8(input_name, &input_size);
|
||||
|
||||
commands_size = (unsigned int)input_size;
|
||||
commands = xcalloc(sizeof(*commands) * commands_size);
|
||||
for (unsigned int i = 0; i < commands_size; i++) {
|
||||
commands[i].value = input_data[i];
|
||||
}
|
||||
|
||||
free(input_data);
|
||||
|
||||
pick_optimized_commands();
|
||||
|
||||
FILE *output = xfopen(output_name, 'w');
|
||||
write_commands(output);
|
||||
xfclose(output);
|
||||
|
||||
free(commands);
|
||||
}
|
||||
|
||||
/********************************** Optimized dynamic compression *********************************/
|
||||
|
||||
// Original `lzcomp` implementation by ax6 <https://github.com/aaaaaa123456789/lzcomp>
|
||||
// Dynamic programming `dpcomp` algorithm by mei <https://github.com/meithecatte/lzcomp>
|
||||
// Licensed with the Unlicense into the public domain <https://unlicense.org>
|
||||
|
||||
// Uncompressed data
|
||||
uint8_t *raw_data = NULL;
|
||||
unsigned int raw_data_size = 0;
|
||||
|
||||
// best_sizes[i] = the best compressed length for the first i bytes of input
|
||||
// Note that this is nondecreasing, since truncating even in the middle of a command won't enlarge it
|
||||
unsigned int *best_sizes = NULL;
|
||||
// best_commands[i] = the last command of the commands that yields best_sizes[i]
|
||||
struct command *best_commands = NULL;
|
||||
|
||||
unsigned int min(unsigned int a, unsigned int b) {
|
||||
return a < b ? a : b;
|
||||
}
|
||||
|
||||
unsigned int command_size(const struct command *command) {
|
||||
unsigned int header_size = 1 + (command->length > SHORT_MAX_LENGTH);
|
||||
switch (command->cmd) {
|
||||
case LZ_LITERAL:
|
||||
return header_size + command->length;
|
||||
case LZ_ITERATE:
|
||||
return header_size + 1;
|
||||
case LZ_ALTERNATE:
|
||||
return header_size + 2;
|
||||
case LZ_ZERO:
|
||||
return header_size;
|
||||
case LZ_REPEAT:
|
||||
case LZ_FLIP:
|
||||
case LZ_REVERSE:
|
||||
return header_size + 1 + (command->offset >= 0);
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x\n", command->cmd);
|
||||
}
|
||||
}
|
||||
|
||||
void consider(unsigned int pos, const struct command *command) {
|
||||
unsigned int new_size = best_sizes[pos - command->length] + command_size(command);
|
||||
if (new_size < best_sizes[pos]) {
|
||||
best_sizes[pos] = new_size;
|
||||
best_commands[pos] = *command;
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int match_right(unsigned int pos, unsigned int at) {
|
||||
unsigned int n = 0;
|
||||
while (pos + n < raw_data_size && raw_data[pos + n] == raw_data[at + n]) {
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
unsigned int match_flipped(unsigned int pos, unsigned int at) {
|
||||
unsigned int n = 0;
|
||||
while (pos + n < raw_data_size && flip_f(raw_data[pos + n]) == raw_data[at + n]) {
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
unsigned int match_left(unsigned int pos, unsigned int at) {
|
||||
unsigned int n = 0;
|
||||
while (pos + n < raw_data_size && n <= at && raw_data[pos + n] == raw_data[at - n]) {
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
int encode_offset(int pos, int at) {
|
||||
return at - pos >= -LOOKBACK_MAX_OFFSET ? at - pos : at;
|
||||
}
|
||||
|
||||
void find_optimal_commands(void) {
|
||||
best_sizes = xmalloc(sizeof(*best_sizes) * (raw_data_size + 1));
|
||||
best_commands = xmalloc(sizeof(*best_commands) * (raw_data_size + 1));
|
||||
|
||||
best_sizes[0] = 0;
|
||||
for (unsigned int i = 1; i <= raw_data_size; i++) {
|
||||
best_sizes[i] = -1u;
|
||||
}
|
||||
|
||||
for (unsigned int i = 1; i <= raw_data_size; i++) {
|
||||
uint8_t byte = raw_data[i - 1];
|
||||
for (unsigned int prev = i > LONG_MAX_LENGTH ? i - LONG_MAX_LENGTH : 0; prev < i; prev++) {
|
||||
consider(i, &(struct command){LZ_LITERAL, i - prev, prev, 0});
|
||||
}
|
||||
|
||||
unsigned int len = 0;
|
||||
do {
|
||||
len++;
|
||||
if (len >= 2) {
|
||||
consider(i, &(struct command){byte ? LZ_ITERATE : LZ_ZERO, len, byte, 0});
|
||||
}
|
||||
} while (len < LONG_MAX_LENGTH && len < i && raw_data[i - (len + 1)] == byte);
|
||||
|
||||
if (i > 1) {
|
||||
len = 1;
|
||||
do {
|
||||
len++;
|
||||
if (len >= 3) {
|
||||
int bytes = (raw_data[i - len + 1] << 8) | (raw_data[i - len]);
|
||||
consider(i, &(struct command){LZ_ALTERNATE, len, bytes, 0});
|
||||
}
|
||||
} while (len < LONG_MAX_LENGTH && len < i && raw_data[i - (len + 1)] == raw_data[i - (len - 1)]);
|
||||
}
|
||||
|
||||
for (unsigned int at = 0; at < i - 1; at++) {
|
||||
unsigned int k = min(LONG_MAX_LENGTH, match_right(i - 1, at));
|
||||
for (unsigned int j = 2; j <= k; j++) {
|
||||
consider(i + j - 1, &(struct command){LZ_REPEAT, j, encode_offset(i - 1, at), 0});
|
||||
}
|
||||
|
||||
k = min(LONG_MAX_LENGTH, match_left(i - 1, at));
|
||||
for (unsigned int j = 2; j <= k; j++) {
|
||||
consider(i + j - 1, &(struct command){LZ_REVERSE, j, encode_offset(i - 1, at), 0});
|
||||
}
|
||||
|
||||
k = min(LONG_MAX_LENGTH, match_flipped(i - 1, at));
|
||||
for (unsigned int j = 2; j <= k; j++) {
|
||||
consider(i + j - 1, &(struct command){LZ_FLIP, j, encode_offset(i - 1, at), 0});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsigned int command_count = 0;
|
||||
for (unsigned int pos = raw_data_size; pos > 0; pos -= best_commands[pos].length) {
|
||||
command_count++;
|
||||
}
|
||||
commands_size = command_count;
|
||||
|
||||
commands = xmalloc(sizeof(*commands) * command_count);
|
||||
for (unsigned int pos = raw_data_size; pos > 0; pos -= best_commands[pos].length) {
|
||||
commands[--command_count] = best_commands[pos];
|
||||
}
|
||||
}
|
||||
|
||||
void write_commands_dynamic(FILE *file) {
|
||||
unsigned int compressed_size = 0;
|
||||
|
||||
for (unsigned int i = 0; i < commands_size; i++) {
|
||||
struct command command = commands[i];
|
||||
|
||||
uint8_t buf[4] = {0};
|
||||
uint8_t *buf_pos = buf;
|
||||
|
||||
command.length--;
|
||||
|
||||
if (command.length >= LONG_MAX_LENGTH) {
|
||||
error_exit("invalid LZ command $%02x (length %u) in output\n", command.cmd, command.length);
|
||||
}
|
||||
|
||||
if (command.length < SHORT_MAX_LENGTH) {
|
||||
*(buf_pos++) = (command.cmd << 5) + command.length;
|
||||
} else {
|
||||
*(buf_pos++) = (LZ_LONG << 5) + (command.cmd << 2) + (command.length >> 8);
|
||||
*(buf_pos++) = command.length & 0xff;
|
||||
}
|
||||
|
||||
switch (command.cmd) {
|
||||
case LZ_LITERAL:
|
||||
break;
|
||||
case LZ_ITERATE:
|
||||
if (command.offset < 0 || command.offset > 0xff) {
|
||||
error_exit("invalid LZ command $%02x (byte $%x) in output\n", command.cmd, command.offset);
|
||||
}
|
||||
*(buf_pos++) = command.offset & 0xff;
|
||||
break;
|
||||
case LZ_ALTERNATE:
|
||||
if (command.offset < 0 || command.offset > 0xffff) {
|
||||
error_exit("invalid LZ command $%02x (bytes $%x) in output\n", command.cmd, command.offset);
|
||||
}
|
||||
*(buf_pos++) = command.offset & 0xff;
|
||||
*(buf_pos++) = command.offset >> 8;
|
||||
break;
|
||||
case LZ_ZERO:
|
||||
break;
|
||||
case LZ_REPEAT:
|
||||
case LZ_FLIP:
|
||||
case LZ_REVERSE:
|
||||
if (command.offset < -LOOKBACK_MAX_OFFSET) {
|
||||
error_exit("invalid LZ command $%02x (offset %d) in output\n", command.cmd, command.offset);
|
||||
}
|
||||
if (command.offset < 0) {
|
||||
*(buf_pos++) = command.offset ^ 0x7f;
|
||||
} else {
|
||||
*(buf_pos++) = command.offset >> 8;
|
||||
*(buf_pos++) = command.offset & 0xff;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x in output\n", command.cmd);
|
||||
}
|
||||
|
||||
fwrite(buf, 1, buf_pos - buf, file);
|
||||
if (!command.cmd) {
|
||||
command.length++;
|
||||
fwrite(raw_data + command.offset, 1, command.length, file);
|
||||
}
|
||||
|
||||
compressed_size += command_size(&commands[i]);
|
||||
}
|
||||
|
||||
putc(LZ_END, file);
|
||||
|
||||
unsigned int padding_size = ~compressed_size & ((1 << options.alignment) - 1);
|
||||
while (padding_size--) {
|
||||
putc(0, file);
|
||||
}
|
||||
}
|
||||
|
||||
void compress_dynamic(char const *input_name, char const *output_name) {
|
||||
long input_size;
|
||||
raw_data = read_u8(input_name, &input_size);
|
||||
|
||||
raw_data_size = (unsigned int)input_size;
|
||||
find_optimal_commands();
|
||||
|
||||
free(best_commands);
|
||||
free(best_sizes);
|
||||
|
||||
FILE *output = xfopen(output_name, 'w');
|
||||
write_commands_dynamic(output);
|
||||
xfclose(output);
|
||||
|
||||
free(raw_data);
|
||||
free(commands);
|
||||
}
|
||||
|
||||
/****************************************** Decompression *****************************************/
|
||||
|
||||
struct command *read_commands(char const *filename, unsigned int *num_commands) {
|
||||
long filesize;
|
||||
uint8_t *data = read_u8(filename, &filesize);
|
||||
|
||||
// There will be at most one command per byte of compressed data, so
|
||||
// `filesize` is a valid upper bound for the count of `command_data`
|
||||
struct command *command_data = xmalloc(sizeof(*command_data) * filesize);
|
||||
*num_commands = 0;
|
||||
|
||||
for (long i = 0; i < filesize;) {
|
||||
uint8_t byte = data[i++];
|
||||
struct command *command = &command_data[(*num_commands)++];
|
||||
|
||||
unsigned int following_len = 0;
|
||||
|
||||
if (byte == LZ_END) {
|
||||
command->cmd = LZ_END;
|
||||
} else {
|
||||
command->cmd = byte >> 5;
|
||||
command->length = byte & 31;
|
||||
if (command->cmd == LZ_LONG) {
|
||||
if (i >= filesize) {
|
||||
error_exit("incomplete long command at end of input\n");
|
||||
}
|
||||
command->cmd = command->length >> 2;
|
||||
command->length = ((command->length & 3) << 8) | data[i++];
|
||||
}
|
||||
command->length++;
|
||||
}
|
||||
|
||||
switch (command->cmd) {
|
||||
case LZ_END:
|
||||
following_len = filesize - i;
|
||||
break;
|
||||
case LZ_LITERAL:
|
||||
following_len = command->length;
|
||||
break;
|
||||
case LZ_ITERATE:
|
||||
following_len = 1;
|
||||
break;
|
||||
case LZ_ALTERNATE:
|
||||
following_len = 2;
|
||||
break;
|
||||
case LZ_ZERO:
|
||||
break;
|
||||
case LZ_REPEAT:
|
||||
case LZ_FLIP:
|
||||
case LZ_REVERSE:
|
||||
if (i >= filesize) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
command->offset = data[i++];
|
||||
if (command->offset & 0x80) {
|
||||
command->offset = -((command->offset & 0x7f) + 1);
|
||||
} else {
|
||||
if (i >= filesize) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
command->offset = (command->offset << 8) | data[i++];
|
||||
}
|
||||
break;
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x\n", command->cmd);
|
||||
}
|
||||
|
||||
if (i + following_len > filesize) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
for (unsigned int j = 0; j < following_len; j++) {
|
||||
command_data[(*num_commands)++].value = data[i++];
|
||||
}
|
||||
}
|
||||
|
||||
free(data);
|
||||
return command_data;
|
||||
}
|
||||
|
||||
void decompress(char const *input_name, char const *output_name) {
|
||||
unsigned int input_size;
|
||||
struct command *input_data = read_commands(input_name, &input_size);
|
||||
|
||||
unsigned int output_capacity = input_size * 2;
|
||||
uint8_t *output_data = xmalloc(output_capacity);
|
||||
unsigned int output_size = 0;
|
||||
|
||||
for (unsigned int i = 0; i < input_size;) {
|
||||
struct command *command = &input_data[i++];
|
||||
|
||||
if (command->cmd == LZ_END) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (output_size + command->length > output_capacity) {
|
||||
output_capacity = output_capacity * 2 + command->length;
|
||||
output_data = xrealloc(output_data, output_capacity);
|
||||
}
|
||||
|
||||
switch (command->cmd) {
|
||||
case LZ_LITERAL:
|
||||
if (i + command->length > input_size) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = input_data[i++].value;
|
||||
}
|
||||
break;
|
||||
case LZ_ITERATE:
|
||||
if (i + 1 > input_size) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
uint8_t byte = input_data[i++].value;
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = byte;
|
||||
}
|
||||
break;
|
||||
case LZ_ALTERNATE:
|
||||
if (i + 2 > input_size) {
|
||||
error_exit("incomplete $%02x command at end of input\n", command->cmd);
|
||||
}
|
||||
uint8_t bytes[2] = {input_data[i].value, input_data[i + 1].value};
|
||||
i += 2;
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = bytes[j & 1];
|
||||
}
|
||||
break;
|
||||
case LZ_ZERO:
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = 0;
|
||||
}
|
||||
break;
|
||||
case LZ_REPEAT: {
|
||||
unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset;
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = output_data[base + j];
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LZ_FLIP: {
|
||||
unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset;
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = flip_f(output_data[base + j]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case LZ_REVERSE: {
|
||||
unsigned int base = (command->offset < 0 ? output_size : 0) + command->offset;
|
||||
for (unsigned int j = 0; j < command->length; j++) {
|
||||
output_data[output_size++] = output_data[base - j];
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
error_exit("invalid LZ command $%02x in input\n", command->cmd);
|
||||
}
|
||||
}
|
||||
|
||||
write_u8(output_name, output_data, output_size);
|
||||
|
||||
free(input_data);
|
||||
free(output_data);
|
||||
}
|
||||
|
||||
/********************************************** Main **********************************************/
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
parse_args(argc, argv);
|
||||
|
||||
argc -= optind;
|
||||
argv += optind;
|
||||
if (argc != 2) {
|
||||
usage_exit(1);
|
||||
}
|
||||
|
||||
if (options.decompress) {
|
||||
decompress(argv[0], argv[1]);
|
||||
} else if (options.matching) {
|
||||
compress_matching(argv[0], argv[1]);
|
||||
} else {
|
||||
compress_dynamic(argv[0], argv[1]);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user