diff --git a/tools/aif2pcm_old/.gitignore b/tools/aif2pcm_old/.gitignore deleted file mode 100644 index 31531797e..000000000 --- a/tools/aif2pcm_old/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -aif2pcm - diff --git a/tools/aif2pcm_old/LICENSE b/tools/aif2pcm_old/LICENSE deleted file mode 100644 index 966b92bd6..000000000 --- a/tools/aif2pcm_old/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2016 huderlem -Copyright (c) 2005, 2006 by Marco Trillo - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/tools/aif2pcm_old/Makefile b/tools/aif2pcm_old/Makefile deleted file mode 100644 index af7d19fe9..000000000 --- a/tools/aif2pcm_old/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -CC = gcc - -CFLAGS = -Wall -Wextra -Wno-switch -Werror -std=c11 -O2 - -LIBS = -lm - -SRCS = main.c extended.c - -.PHONY: all clean - -all: aif2pcm - @: - -aif2pcm: $(SRCS) - $(CC) $(CFLAGS) $(SRCS) -o $@ $(LDFLAGS) $(LIBS) - -clean: - $(RM) aif2pcm aif2pcm.exe diff --git a/tools/aif2pcm_old/extended.c b/tools/aif2pcm_old/extended.c deleted file mode 100644 index 94449164b..000000000 --- a/tools/aif2pcm_old/extended.c +++ /dev/null @@ -1,172 +0,0 @@ -/* $Id: extended.c,v 1.8 2006/12/23 11:17:49 toad32767 Exp $ */ -/*- - * Copyright (c) 2005, 2006 by Marco Trillo - * - * Permission is hereby granted, free of charge, to any - * person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the - * Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice - * shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY - * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS - * OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include -#include -#include - -/* - * Infinite & NAN values - * for non-IEEE systems - */ -#ifndef HUGE_VAL -#ifdef HUGE -#define INFINITE_VALUE HUGE -#define NAN_VALUE HUGE -#endif -#else -#define INFINITE_VALUE HUGE_VAL -#define NAN_VALUE HUGE_VAL -#endif - -/* - * IEEE 754 Extended Precision - * - * Implementation here is the 80-bit extended precision - * format of Motorola 68881, Motorola 68882 and Motorola - * 68040 FPUs, as well as Intel 80x87 FPUs. - * - * See: - * http://www.freescale.com/files/32bit/doc/fact_sheet/BR509.pdf - */ -/* - * Exponent range: [-16383,16383] - * Precision for mantissa: 64 bits with no hidden bit - * Bias: 16383 - */ - -/* - * Write IEEE Extended Precision Numbers - */ -void -ieee754_write_extended(double in, uint8_t* out) -{ - int sgn, exp, shift; - double fraction, t; - unsigned int lexp, hexp; - unsigned long low, high; - - if (in == 0.0) { - memset(out, 0, 10); - return; - } - if (in < 0.0) { - in = fabs(in); - sgn = 1; - } else - sgn = 0; - - fraction = frexp(in, &exp); - - if (exp == 0 || exp > 16384) { - if (exp > 16384) /* infinite value */ - low = high = 0; - else { - low = 0x80000000; - high = 0; - } - exp = 32767; - goto done; - } - fraction = ldexp(fraction, 32); - t = floor(fraction); - low = (unsigned long) t; - fraction -= t; - t = floor(ldexp(fraction, 32)); - high = (unsigned long) t; - - /* Convert exponents < -16382 to -16382 (then they will be - * stored as -16383) */ - if (exp < -16382) { - shift = 0 - exp - 16382; - high >>= shift; - high |= (low << (32 - shift)); - low >>= shift; - exp = -16382; - } - exp += 16383 - 1; /* bias */ - -done: - lexp = ((unsigned int) exp) >> 8; - hexp = ((unsigned int) exp) & 0xFF; - - /* big endian */ - out[0] = ((uint8_t) sgn) << 7; - out[0] |= (uint8_t) lexp; - out[1] = (uint8_t) hexp; - out[2] = (uint8_t) (low >> 24); - out[3] = (uint8_t) ((low >> 16) & 0xFF); - out[4] = (uint8_t) ((low >> 8) & 0xFF); - out[5] = (uint8_t) (low & 0xFF); - out[6] = (uint8_t) (high >> 24); - out[7] = (uint8_t) ((high >> 16) & 0xFF); - out[8] = (uint8_t) ((high >> 8) & 0xFF); - out[9] = (uint8_t) (high & 0xFF); - - return; -} - - -/* - * Read IEEE Extended Precision Numbers - */ -double -ieee754_read_extended(uint8_t* in) -{ - int sgn, exp; - unsigned long low, high; - double out; - - /* Extract the components from the big endian buffer */ - sgn = (int) (in[0] >> 7); - exp = ((int) (in[0] & 0x7F) << 8) | ((int) in[1]); - low = (((unsigned long) in[2]) << 24) - | (((unsigned long) in[3]) << 16) - | (((unsigned long) in[4]) << 8) | (unsigned long) in[5]; - high = (((unsigned long) in[6]) << 24) - | (((unsigned long) in[7]) << 16) - | (((unsigned long) in[8]) << 8) | (unsigned long) in[9]; - - if (exp == 0 && low == 0 && high == 0) - return (sgn ? -0.0 : 0.0); - - switch (exp) { - case 32767: - if (low == 0 && high == 0) - return (sgn ? -INFINITE_VALUE : INFINITE_VALUE); - else - return (sgn ? -NAN_VALUE : NAN_VALUE); - default: - exp -= 16383; /* unbias exponent */ - - } - - out = ldexp((double) low, -31 + exp); - out += ldexp((double) high, -63 + exp); - - return (sgn ? -out : out); -} diff --git a/tools/aif2pcm_old/main.c b/tools/aif2pcm_old/main.c deleted file mode 100644 index cd5ac4a50..000000000 --- a/tools/aif2pcm_old/main.c +++ /dev/null @@ -1,888 +0,0 @@ -// Copyright(c) 2016 huderlem -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#include -#include -#include -#include -#include -#include - -/* extended.c */ -void ieee754_write_extended (double, uint8_t*); -double ieee754_read_extended (uint8_t*); - -#ifdef _MSC_VER - -#define FATAL_ERROR(format, ...) \ -do \ -{ \ - fprintf(stderr, format, __VA_ARGS__); \ - exit(1); \ -} while (0) - -#else - -#define FATAL_ERROR(format, ...) \ -do \ -{ \ - fprintf(stderr, format, ##__VA_ARGS__); \ - exit(1); \ -} while (0) - -#endif // _MSC_VER - -typedef struct { - unsigned long num_samples; - uint8_t *samples; - uint8_t midi_note; - bool has_loop; - unsigned long loop_offset; - double sample_rate; - unsigned long real_num_samples; -} AifData; - -struct Bytes { - unsigned long length; - uint8_t *data; -}; - -struct Marker { - unsigned short id; - unsigned long position; - // don't care about the name -}; - -struct Bytes *read_bytearray(const char *filename) -{ - struct Bytes *bytes = malloc(sizeof(struct Bytes)); - FILE *f = fopen(filename, "rb"); - if (!f) - { - FATAL_ERROR("Failed to open '%s' for reading!\n", filename); - } - fseek(f, 0, SEEK_END); - bytes->length = ftell(f); - fseek(f, 0, SEEK_SET); - bytes->data = malloc(bytes->length); - unsigned long read = fread(bytes->data, bytes->length, 1, f); - fclose(f); - if (read <= 0) - { - FATAL_ERROR("Failed to read data from '%s'!\n", filename); - } - return bytes; -} - -void write_bytearray(const char *filename, struct Bytes *bytes) -{ - FILE *f = fopen(filename, "wb"); - if (!f) - { - FATAL_ERROR("Failed to open '%s' for writing!\n", filename); - } - fwrite(bytes->data, bytes->length, 1, f); - fclose(f); -} - -void free_bytearray(struct Bytes *bytes) -{ - free(bytes->data); - free(bytes); -} - -char *get_file_extension(char *filename) -{ - char *index = strrchr(filename, '.'); - if (!index || index == filename) - { - return NULL; - } - return index + 1; -} - -char *new_file_extension(char *filename, char *ext) -{ - char *index = strrchr(filename, '.'); - if (!index || index == filename) - { - index = filename + strlen(filename); - } - int length = index - filename; - char *new_filename = malloc(length + 1 + strlen(ext) + 1); - if (new_filename) - { - strcpy(new_filename, filename); - new_filename[length] = '.'; - strcpy(new_filename + length + 1, ext); - } - return new_filename; -} - -void read_aif(struct Bytes *aif, AifData *aif_data) -{ - aif_data->has_loop = false; - aif_data->num_samples = 0; - - unsigned long pos = 0; - char chunk_name[5]; chunk_name[4] = '\0'; - char chunk_type[5]; chunk_type[4] = '\0'; - - // Check for FORM Chunk - memcpy(chunk_name, &aif->data[pos], 4); - pos += 4; - if (strcmp(chunk_name, "FORM") != 0) - { - FATAL_ERROR("Input .aif file has invalid header Chunk '%s'!\n", chunk_name); - } - - // Read size of whole file. - unsigned long whole_chunk_size = aif->data[pos++] << 24; - whole_chunk_size |= (aif->data[pos++] << 16); - whole_chunk_size |= (aif->data[pos++] << 8); - whole_chunk_size |= (uint8_t)aif->data[pos++]; - - unsigned long expected_whole_chunk_size = aif->length - 8; - if (whole_chunk_size != expected_whole_chunk_size) - { - FATAL_ERROR("FORM Chunk ckSize '%lu' doesn't match actual size '%lu'!\n", whole_chunk_size, expected_whole_chunk_size); - } - - // Check for AIFF Form Type - memcpy(chunk_type, &aif->data[pos], 4); - pos += 4; - if (strcmp(chunk_type, "AIFF") != 0) - { - FATAL_ERROR("FORM Type is '%s', but it must be AIFF!", chunk_type); - } - - struct Marker *markers = NULL; - unsigned short num_markers = 0, loop_start = 0, loop_end = 0; - unsigned long num_sample_frames = 0; - - // Read all the Chunks to populate the AifData struct. - while ((pos + 8) < aif->length) - { - // Read Chunk id - memcpy(chunk_name, &aif->data[pos], 4); - pos += 4; - - unsigned long chunk_size = (aif->data[pos++] << 24); - chunk_size |= (aif->data[pos++] << 16); - chunk_size |= (aif->data[pos++] << 8); - chunk_size |= aif->data[pos++]; - - if ((pos + chunk_size) > aif->length) - { - FATAL_ERROR("%s chunk at 0x%lx reached end of file before finishing\n", chunk_name, pos); - } - - if (strcmp(chunk_name, "COMM") == 0) - { - short num_channels = (aif->data[pos++] << 8); - num_channels |= (uint8_t)aif->data[pos++]; - if (num_channels != 1) - { - FATAL_ERROR("numChannels (%d) in the COMM Chunk must be 1!\n", num_channels); - } - - num_sample_frames = (aif->data[pos++] << 24); - num_sample_frames |= (aif->data[pos++] << 16); - num_sample_frames |= (aif->data[pos++] << 8); - num_sample_frames |= (uint8_t)aif->data[pos++]; - - short sample_size = (aif->data[pos++] << 8); - sample_size |= (uint8_t)aif->data[pos++]; - if (sample_size != 8) - { - FATAL_ERROR("sampleSize (%d) in the COMM Chunk must be 8!\n", sample_size); - } - - double sample_rate = ieee754_read_extended((uint8_t*)(aif->data + pos)); - pos += 10; - - aif_data->sample_rate = sample_rate; - - if (aif_data->num_samples == 0) - { - aif_data->num_samples = num_sample_frames; - } - } - else if (strcmp(chunk_name, "MARK") == 0) - { - num_markers = (aif->data[pos++] << 8); - num_markers |= (uint8_t)aif->data[pos++]; - - if (markers) - { - FATAL_ERROR("More than one MARK Chunk in file!\n"); - } - - markers = calloc(num_markers, sizeof(struct Marker)); - - // Read each marker. - for (int i = 0; i < num_markers; i++) - { - unsigned short marker_id = (aif->data[pos++] << 8); - marker_id |= (uint8_t)aif->data[pos++]; - - unsigned long marker_position = (aif->data[pos++] << 24); - marker_position |= (aif->data[pos++] << 16); - marker_position |= (aif->data[pos++] << 8); - marker_position |= (uint8_t)aif->data[pos++]; - - // Marker name is a Pascal-style string. - uint8_t marker_name_size = aif->data[pos++]; - // We don't actually need the marker name for anything anymore. - /*char *marker_name = (char *)malloc((marker_name_size + 1) * sizeof(char)); - memcpy(marker_name, &aif->data[pos], marker_name_size); - marker_name[marker_name_size] = '\0';*/ - pos += marker_name_size + !(marker_name_size & 1); - - markers[i].id = marker_id; - markers[i].position = marker_position; - } - } - else if (strcmp(chunk_name, "INST") == 0) - { - uint8_t midi_note = (uint8_t)aif->data[pos++]; - - aif_data->midi_note = midi_note; - - // Skip over data we don't need. - pos += 7; - - unsigned short loop_type = (aif->data[pos++] << 8); - loop_type |= (uint8_t)aif->data[pos++]; - - if (loop_type) - { - loop_start = (aif->data[pos++] << 8); - loop_start |= (uint8_t)aif->data[pos++]; - - loop_end = (aif->data[pos++] << 8); - loop_end |= (uint8_t)aif->data[pos++]; - } - else - { - // Skip NoLooping sustain loop. - pos += 4; - } - - // Skip release loop, we don't need it. - pos += 6; - } - else if (strcmp(chunk_name, "SSND") == 0) - { - // Skip offset and blockSize - pos += 8; - - unsigned long num_samples = chunk_size - 8; - uint8_t *sample_data = (uint8_t *)malloc(num_samples * sizeof(uint8_t)); - memcpy(sample_data, &aif->data[pos], num_samples); - - aif_data->samples = sample_data; - aif_data->real_num_samples = num_samples; - pos += chunk_size - 8; - } - else - { - // Skip over unsupported chunks. - pos += chunk_size; - } - } - - if (markers) - { - // Resolve loop points. - struct Marker *cur_marker = markers; - - // Grab loop start point. - for (int i = 0; i < num_markers; i++, cur_marker++) - { - if (cur_marker->id == loop_start) - { - aif_data->loop_offset = cur_marker->position; - aif_data->has_loop = true; - break; - } - } - - cur_marker = markers; - - // Grab loop end point. - for (int i = 0; i < num_markers; i++, cur_marker++) - { - if (cur_marker->id == loop_end) - { - if (cur_marker->position < aif_data->loop_offset) { - aif_data->loop_offset = cur_marker->position; - aif_data->has_loop = true; - } - aif_data->num_samples = cur_marker->position; - break; - } - } - - free(markers); - } -} - -// This is a table of deltas between sample values in compressed PCM data. -const int gDeltaEncodingTable[] = { - 0, 1, 4, 9, 16, 25, 36, 49, - -64, -49, -36, -25, -16, -9, -4, -1, -}; - -struct Bytes *delta_decompress(struct Bytes *delta, unsigned int expected_length) -{ - struct Bytes *pcm = malloc(sizeof(struct Bytes)); - pcm->length = expected_length; - pcm->data = malloc(pcm->length + 0x40); - - uint8_t hi, lo; - unsigned int i = 0; - unsigned int j = 0; - int k; - int8_t base; - while (i < delta->length) - { - base = (int8_t)delta->data[i++]; - pcm->data[j++] = (uint8_t)base; - if (i >= delta->length) - { - break; - } - if (j >= pcm->length) - { - break; - } - lo = delta->data[i] & 0xf; - base += gDeltaEncodingTable[lo]; - pcm->data[j++] = base; - i++; - if (i >= delta->length) - { - break; - } - if (j >= pcm->length) - { - break; - } - for (k = 0; k < 31; k++) - { - hi = (delta->data[i] >> 4) & 0xf; - base += gDeltaEncodingTable[hi]; - pcm->data[j++] = base; - if (j >= pcm->length) - { - break; - } - lo = delta->data[i] & 0xf; - base += gDeltaEncodingTable[lo]; - pcm->data[j++] = base; - i++; - if (i >= delta->length) - { - break; - } - if (j >= pcm->length) - { - break; - } - } - if (j >= pcm->length) - { - break; - } - } - - pcm->length = j; - return pcm; -} - -int get_delta_index(uint8_t sample, uint8_t prev_sample) -{ - int best_error = INT_MAX; - int best_index = -1; - - for (int i = 0; i < 16; i++) - { - uint8_t new_sample = prev_sample + gDeltaEncodingTable[i]; - int error = sample > new_sample ? sample - new_sample : new_sample - sample; - - if (error < best_error) - { - best_error = error; - best_index = i; - } - } - - return best_index; -} - -struct Bytes *delta_compress(struct Bytes *pcm) -{ - struct Bytes *delta = malloc(sizeof(struct Bytes)); - // estimate the length so we can malloc - int num_blocks = pcm->length / 64; - delta->length = num_blocks * 33; - - int extra = pcm->length % 64; - if (extra) - { - delta->length += 1; - extra -= 1; - } - if (extra) - { - delta->length += 1; - extra -= 1; - } - if (extra) - { - delta->length += (extra + 1) / 2; - } - - delta->data = malloc(delta->length + 33); - - unsigned int i = 0; - unsigned int j = 0; - int k; - uint8_t base; - int delta_index; - - while (i < pcm->length) - { - base = pcm->data[i++]; - delta->data[j++] = base; - - if (i >= pcm->length) - { - break; - } - delta_index = get_delta_index(pcm->data[i++], base); - base += gDeltaEncodingTable[delta_index]; - delta->data[j++] = delta_index; - - for (k = 0; k < 31; k++) - { - if (i >= pcm->length) - { - break; - } - delta_index = get_delta_index(pcm->data[i++], base); - base += gDeltaEncodingTable[delta_index]; - delta->data[j] = (delta_index << 4); - - if (i >= pcm->length) - { - break; - } - delta_index = get_delta_index(pcm->data[i++], base); - base += gDeltaEncodingTable[delta_index]; - delta->data[j++] |= delta_index; - } - } - - delta->length = j; - - return delta; -} - -#define STORE_U32_LE(dest, value) \ -do { \ - *(dest) = (value) & 0xff; \ - *((dest) + 1) = ((value) >> 8) & 0xff; \ - *((dest) + 2) = ((value) >> 16) & 0xff; \ - *((dest) + 3) = ((value) >> 24) & 0xff; \ -} while (0) - -#define LOAD_U32_LE(var, src) \ -do { \ - (var) = *(src); \ - (var) |= (*((src) + 1) << 8); \ - (var) |= (*((src) + 2) << 16); \ - (var) |= (*((src) + 3) << 24); \ -} while (0) - -// Reads an .aif file and produces a .pcm file containing an array of 8-bit samples. -void aif2pcm(const char *aif_filename, const char *pcm_filename, bool compress) -{ - struct Bytes *aif = read_bytearray(aif_filename); - AifData aif_data = {0,0,0,0,0,0,0}; - read_aif(aif, &aif_data); - - int header_size = 0x10; - struct Bytes *pcm; - struct Bytes output = {0,0}; - - if (compress) - { - struct Bytes *input = malloc(sizeof(struct Bytes)); - input->data = aif_data.samples; - input->length = aif_data.real_num_samples; - pcm = delta_compress(input); - free(input); - } - else - { - pcm = malloc(sizeof(struct Bytes)); - pcm->data = aif_data.samples; - pcm->length = aif_data.real_num_samples; - } - output.length = header_size + pcm->length; - output.data = malloc(output.length); - - uint32_t pitch_adjust = (uint32_t)(aif_data.sample_rate * 1024); - uint32_t loop_offset = (uint32_t)(aif_data.loop_offset); - uint32_t adjusted_num_samples = (uint32_t)(aif_data.num_samples - 1); - uint32_t flags = 0; - if (aif_data.has_loop) flags |= 0x40000000; - if (compress) flags |= 1; - STORE_U32_LE(output.data + 0, flags); - STORE_U32_LE(output.data + 4, pitch_adjust); - STORE_U32_LE(output.data + 8, loop_offset); - STORE_U32_LE(output.data + 12, adjusted_num_samples); - memcpy(&output.data[header_size], pcm->data, pcm->length); - write_bytearray(pcm_filename, &output); - - free(aif->data); - free(aif); - free(pcm); - free(output.data); - free(aif_data.samples); -} - -// Reads a .pcm file containing an array of 8-bit samples and produces an .aif file. -// See http://www-mmsp.ece.mcgill.ca/documents/audioformats/aiff/Docs/AIFF-1.3.pdf for .aif file specification. -void pcm2aif(const char *pcm_filename, const char *aif_filename, uint32_t base_note) -{ - struct Bytes *pcm = read_bytearray(pcm_filename); - - AifData *aif_data = malloc(sizeof(AifData)); - - uint32_t flags; - LOAD_U32_LE(flags, pcm->data + 0); - aif_data->has_loop = flags & 0x40000000; - bool compressed = flags & 1; - - uint32_t pitch_adjust; - LOAD_U32_LE(pitch_adjust, pcm->data + 4); - aif_data->sample_rate = pitch_adjust / 1024.0; - - LOAD_U32_LE(aif_data->loop_offset, pcm->data + 8); - LOAD_U32_LE(aif_data->num_samples, pcm->data + 12); - aif_data->num_samples += 1; - - if (compressed) - { - struct Bytes *delta = pcm; - uint8_t *pcm_data = pcm->data; - delta->length -= 0x10; - delta->data += 0x10; - pcm = delta_decompress(delta, aif_data->num_samples); - free(pcm_data); - free(delta); - } - else - { - pcm->length -= 0x10; - pcm->data += 0x10; - } - - aif_data->samples = malloc(pcm->length); - memcpy(aif_data->samples, pcm->data, pcm->length); - - struct Bytes *aif = malloc(sizeof(struct Bytes)); - aif->length = 54 + 60 + pcm->length; - aif->data = malloc(aif->length); - - long pos = 0; - - // First, write the FORM header chunk. - // FORM Chunk ckID - aif->data[pos++] = 'F'; - aif->data[pos++] = 'O'; - aif->data[pos++] = 'R'; - aif->data[pos++] = 'M'; - - // FORM Chunk ckSize - unsigned long form_size = pos; - unsigned long data_size = aif->length - 8; - aif->data[pos++] = ((data_size >> 24) & 0xFF); - aif->data[pos++] = ((data_size >> 16) & 0xFF); - aif->data[pos++] = ((data_size >> 8) & 0xFF); - aif->data[pos++] = (data_size & 0xFF); - - // FORM Chunk formType - aif->data[pos++] = 'A'; - aif->data[pos++] = 'I'; - aif->data[pos++] = 'F'; - aif->data[pos++] = 'F'; - - // Next, write the Common Chunk - // Common Chunk ckID - aif->data[pos++] = 'C'; - aif->data[pos++] = 'O'; - aif->data[pos++] = 'M'; - aif->data[pos++] = 'M'; - - // Common Chunk ckSize - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 18; - - // Common Chunk numChannels - aif->data[pos++] = 0; - aif->data[pos++] = 1; // 1 channel - - // Common Chunk numSampleFrames - aif->data[pos++] = ((aif_data->num_samples >> 24) & 0xFF); - aif->data[pos++] = ((aif_data->num_samples >> 16) & 0xFF); - aif->data[pos++] = ((aif_data->num_samples >> 8) & 0xFF); - aif->data[pos++] = (aif_data->num_samples & 0xFF); - - // Common Chunk sampleSize - aif->data[pos++] = 0; - aif->data[pos++] = 8; // 8 bits per sample - - // Common Chunk sampleRate - //double sample_rate = pitch_adjust / 1024.0; - uint8_t sample_rate_buffer[10]; - ieee754_write_extended(aif_data->sample_rate, sample_rate_buffer); - for (int i = 0; i < 10; i++) - { - aif->data[pos++] = sample_rate_buffer[i]; - } - - if (aif_data->has_loop) - { - - // Marker Chunk ckID - aif->data[pos++] = 'M'; - aif->data[pos++] = 'A'; - aif->data[pos++] = 'R'; - aif->data[pos++] = 'K'; - - // Marker Chunk ckSize - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 12 + (aif_data->has_loop ? 12 : 0); - - // Marker Chunk numMarkers - aif->data[pos++] = 0; - aif->data[pos++] = (aif_data->has_loop ? 2 : 1); - - // Marker loop start - aif->data[pos++] = 0; - aif->data[pos++] = 1; // id = 1 - - long loop_start = aif_data->loop_offset; - aif->data[pos++] = ((loop_start >> 24) & 0xFF); - aif->data[pos++] = ((loop_start >> 16) & 0xFF); - aif->data[pos++] = ((loop_start >> 8) & 0xFF); - aif->data[pos++] = (loop_start & 0xFF); // position - - aif->data[pos++] = 5; // pascal-style string length - aif->data[pos++] = 'S'; - aif->data[pos++] = 'T'; - aif->data[pos++] = 'A'; - aif->data[pos++] = 'R'; - aif->data[pos++] = 'T'; // markerName - - // Marker loop end - aif->data[pos++] = 0; - aif->data[pos++] = (aif_data->has_loop ? 2 : 1); // id = 2 - - long loop_end = aif_data->num_samples; - aif->data[pos++] = ((loop_end >> 24) & 0xFF); - aif->data[pos++] = ((loop_end >> 16) & 0xFF); - aif->data[pos++] = ((loop_end >> 8) & 0xFF); - aif->data[pos++] = (loop_end & 0xFF); // position - - aif->data[pos++] = 3; // pascal-style string length - aif->data[pos++] = 'E'; - aif->data[pos++] = 'N'; - aif->data[pos++] = 'D'; - } - - // Instrument Chunk ckID - aif->data[pos++] = 'I'; - aif->data[pos++] = 'N'; - aif->data[pos++] = 'S'; - aif->data[pos++] = 'T'; - - // Instrument Chunk ckSize - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 20; - - aif->data[pos++] = base_note; // baseNote - aif->data[pos++] = 0; // detune - aif->data[pos++] = 0; // lowNote - aif->data[pos++] = 127; // highNote - aif->data[pos++] = 1; // lowVelocity - aif->data[pos++] = 127; // highVelocity - aif->data[pos++] = 0; // gain (hi) - aif->data[pos++] = 0; // gain (lo) - - // Instrument Chunk sustainLoop - aif->data[pos++] = 0; - aif->data[pos++] = 1; // playMode = ForwardLooping - - aif->data[pos++] = 0; - aif->data[pos++] = 1; // beginLoop marker id - - aif->data[pos++] = 0; - aif->data[pos++] = 2; // endLoop marker id - - // Instrument Chunk releaseLoop - aif->data[pos++] = 0; - aif->data[pos++] = 1; // playMode = ForwardLooping - - aif->data[pos++] = 0; - aif->data[pos++] = 1; // beginLoop marker id - - aif->data[pos++] = 0; - aif->data[pos++] = 2; // endLoop marker id - - // Finally, write the Sound Data Chunk - // Sound Data Chunk ckID - aif->data[pos++] = 'S'; - aif->data[pos++] = 'S'; - aif->data[pos++] = 'N'; - aif->data[pos++] = 'D'; - - // Sound Data Chunk ckSize - unsigned long sound_data_size = pcm->length + 8; - aif->data[pos++] = ((sound_data_size >> 24) & 0xFF); - aif->data[pos++] = ((sound_data_size >> 16) & 0xFF); - aif->data[pos++] = ((sound_data_size >> 8) & 0xFF); - aif->data[pos++] = (sound_data_size & 0xFF); - - // Sound Data Chunk offset - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - - // Sound Data Chunk blockSize - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - aif->data[pos++] = 0; - - // Sound Data Chunk soundData - for (unsigned int i = 0; i < aif_data->loop_offset; i++) - { - aif->data[pos++] = aif_data->samples[i]; - } - - int j = 0; - for (unsigned int i = aif_data->loop_offset; i < pcm->length; i++) - { - int pcm_index = aif_data->loop_offset + (j++ % (pcm->length - aif_data->loop_offset)); - aif->data[pos++] = aif_data->samples[pcm_index]; - } - - aif->length = pos; - - // Go back and rewrite ckSize - data_size = aif->length - 8; - aif->data[form_size + 0] = ((data_size >> 24) & 0xFF); - aif->data[form_size + 1] = ((data_size >> 16) & 0xFF); - aif->data[form_size + 2] = ((data_size >> 8) & 0xFF); - aif->data[form_size + 3] = (data_size & 0xFF); - - write_bytearray(aif_filename, aif); - - free(aif->data); - free(aif); -} - -void usage(void) -{ - fprintf(stderr, "Usage: aif2pcm bin_file [aif_file]\n"); - fprintf(stderr, " aif2pcm aif_file [bin_file] [--compress]\n"); -} - -int main(int argc, char **argv) -{ - if (argc < 2) - { - usage(); - exit(1); - } - - char *input_file = argv[1]; - char *extension = get_file_extension(input_file); - char *output_file; - bool compressed = false; - - if (argc > 3) - { - for (int i = 3; i < argc; i++) - { - if (strcmp(argv[i], "--compress") == 0) - { - compressed = true; - } - } - } - - if (strcmp(extension, "aif") == 0 || strcmp(extension, "aiff") == 0) - { - if (argc >= 3) - { - output_file = argv[2]; - aif2pcm(input_file, output_file, compressed); - } - else - { - output_file = new_file_extension(input_file, "bin"); - aif2pcm(input_file, output_file, compressed); - free(output_file); - } - } - else if (strcmp(extension, "bin") == 0) - { - if (argc >= 3) - { - output_file = argv[2]; - pcm2aif(input_file, output_file, 60); - } - else - { - output_file = new_file_extension(input_file, "aif"); - pcm2aif(input_file, output_file, 60); - free(output_file); - } - } - else - { - FATAL_ERROR("Input file must be .aif or .bin: '%s'\n", input_file); - } - - return 0; -} diff --git a/tools/compresSmol/Makefile b/tools/compresSmol/Makefile index 768de9d4a..e4799483f 100644 --- a/tools/compresSmol/Makefile +++ b/tools/compresSmol/Makefile @@ -1,14 +1,14 @@ CXX ?= g++ -CXXFLAGS := -Werror -std=c++17 -pthread -O2 -Wunused +CXXFLAGS := -Werror -std=c++17 -O2 INCLUDES := -I . -SRCS := compresSmol.cpp compressAlgo.cpp tANS.cpp fileDispatcher.cpp -TILEMAP_SRCS := mainTiles.cpp compressAlgo.cpp compressSmolTiles.cpp tANS.cpp fileDispatcher.cpp +SRCS := compresSmol.cpp compressAlgo.cpp tANS.cpp +TILEMAP_SRCS := mainTiles.cpp compressSmolTiles.cpp tANS.cpp compressAlgo.cpp -HEADERS := compressAlgo.h tANS.h fileDispatcher.h -TILEMAP_HEADERS := compressAlgo.h compressSmolTiles.h tANS.h fileDispatcher.h +HEADERS := compressAlgo.h tANS.h +TILEMAP_HEADERS := compressSmolTiles.h tANS.h compressAlgo.h ifeq ($(OS),Windows_NT) EXE := .exe diff --git a/tools/compresSmol/compresSmol.cpp b/tools/compresSmol/compresSmol.cpp index 80d7ae9f8..1f7d2b1cb 100644 --- a/tools/compresSmol/compresSmol.cpp +++ b/tools/compresSmol/compresSmol.cpp @@ -18,7 +18,6 @@ bool isNumber(std::string input) } enum Option { - ANALYZE, WRITE, FRAME_WRITE, DECODE, @@ -31,15 +30,12 @@ int main(int argc, char *argv[]) bool printUsage = false; std::string input; std::string output; - int numThreads = 1; InputSettings settings(true, true, true); if (argc > 1) { std::string argument = argv[1]; - if (argument.compare("-a") == 0) - option = ANALYZE; - else if (argument.compare("-w") == 0) + if (argument.compare("-w") == 0) option = WRITE; else if (argument.compare("-fw") == 0) option = FRAME_WRITE; @@ -48,46 +44,12 @@ int main(int argc, char *argv[]) } switch (option) { - case ANALYZE: - if (argc > 2) - input = argv[2]; - else - printUsage = true; - if (argc > 4) - { - std::string arg2 = argv[3]; - std::string arg2arg = argv[4]; - if (arg2.compare("-t") == 0 && isNumber(arg2arg)) - numThreads = std::stoi(arg2arg.c_str()); - } - if (argc > 7) - { - std::string setting1 = argv[5]; - std::string setting2 = argv[6]; - std::string setting3 = argv[7]; - if (setting1.compare("true") == 0) - settings.canEncodeLO = true; - else if (setting1.compare("false") == 0) - settings.canEncodeLO = false; - else - fprintf(stderr, "Unrecognized setting1 \"%s\", defaulting to \"true\"\n", setting1.c_str()); - if (setting2.compare("true") == 0) - settings.canEncodeSyms = true; - else if (setting2.compare("false") == 0) - settings.canEncodeSyms = false; - else - fprintf(stderr, "Unrecognized setting2 \"%s\", defaulting to \"true\"\n", setting2.c_str()); - if (setting3.compare("true") == 0) - settings.canDeltaSyms = true; - else if (setting3.compare("false") == 0) - settings.canDeltaSyms = false; - else - fprintf(stderr, "Unrecognized setting3 \"%s\", defaulting to \"true\"\n", setting3.c_str()); - } - break; case FRAME_WRITE: + // Not implemented yet + fprintf(stderr, "Frame writing isn't implemented yet\n"); settings.useFrames = true; option = WRITE; + return 1; case WRITE: if (argc > 3) { @@ -141,10 +103,6 @@ int main(int argc, char *argv[]) if (printUsage) { printf("Usage:\n\ - %s -a \"path/to/some/directory\"\n\ - Analyses all images currently in .4bpp.lz format and compares with this compression.\n\ - -t can be appended to this mode to specify how many threads to use.\n\ - \n\ %s -w \"path/to/some/file.4bpp\" \"path/to/some/file.4bpp.smol\"\ Compresses the first argument and writes the result to the second argument.\n\ These modes can also be appended with 4 true/false statements that control the following settings of the compression:\n\ @@ -153,64 +111,11 @@ int main(int argc, char *argv[]) - If the compression instructions can be delta encoded.\n\ - If the raw symbols in the compression ca be delta encoded.\n\ %s -d \"path/to/some/file.4bpp.smol\" \"path/to/some/file.4bpp\"\n\ - Decompresses the first argument and writes it to the second argument.", argv[0], argv[0], argv[0]); + Decompresses the first argument and writes it to the second argument.", argv[0], argv[0]); return 0; } - if (option == ANALYZE) - { - std::filesystem::path dirPath = input; - FileDispatcher dispatcher(dirPath); - if (!dispatcher.initFileList()) - { - fprintf(stderr, "Failed to init file list\n"); - return 1; - } - std::mutex dispatchMutex; - std::vector allImages; - std::mutex imageMutex; - - settings.shouldCompare = true; - - std::vector threads; - for (int i = 0; i < numThreads; i++) - { - threads.emplace_back(analyzeImages, &allImages, &imageMutex, - &dispatcher, &dispatchMutex, - settings); - } - - for (int i = 0; i < numThreads; i++) - threads[i].join(); - - size_t lzSizes = 0; - size_t newSizes = 0; - size_t rawSizes = 0; - size_t totalImages = 0; - size_t invalidImages = 0; - for (CompressedImage currImage : allImages) - { - totalImages++; - if (currImage.isValid) - { - lzSizes += currImage.lzSize; - newSizes += currImage.compressedSize; - rawSizes += currImage.rawNumBytes; - } - else - { - fprintf(stderr, "Failed to solve %s\n", currImage.fileName.c_str()); - invalidImages++; - } - } - - fprintf(stderr, "RawSize: %zu\n", rawSizes); - fprintf(stderr, "LZsize: %zu\n", lzSizes); - fprintf(stderr, "SmolSize: %zu\n", newSizes); - fprintf(stderr, "Total Images: %zu\n", totalImages); - fprintf(stderr, "Invalid Images: %zu\n", invalidImages); - } if (option == WRITE) { if (std::filesystem::exists(input)) @@ -240,8 +145,13 @@ int main(int argc, char *argv[]) { if (std::filesystem::exists(input)) { - std::vector inData = readFileAsUInt(input); - std::vector image4bpp = readRawDataVecs(&inData); + std::vector inData; + if (!readFileAsUInt(input, &inData)) + { + return 0; + } + std::vector image4bpp; + readRawDataVecs(&inData, &image4bpp); std::vector charVec(image4bpp.size()*2); for (size_t i = 0; i < image4bpp.size(); i++) { diff --git a/tools/compresSmol/compressAlgo.cpp b/tools/compresSmol/compressAlgo.cpp index 07ac59da8..36d4cbfad 100644 --- a/tools/compresSmol/compressAlgo.cpp +++ b/tools/compresSmol/compressAlgo.cpp @@ -1,18 +1,30 @@ #include "compressAlgo.h" -std::vector getShortCopies(std::vector input, size_t minLength) +ShortCopy::ShortCopy(size_t index, size_t length, size_t offset, unsigned short firstSymbol) { - std::vector copies; - for (size_t startIndex = 1; startIndex < input.size(); startIndex++) + this->index = index; + this->length = length; + this->offset = offset; + this->firstSymbol = firstSymbol; +} + +bool getShortCopies(std::vector *pInput, size_t minLength, std::vector *pShortCopies) +{ + size_t iteration = 0; + std::vector checkVec(pInput->size()); + for (size_t i = 0; i < checkVec.size(); i++) + checkVec[i] = ' '; + for (size_t startIndex = 1; startIndex < pInput->size(); startIndex++) { size_t longestLength = 0; size_t longestOffset; for (size_t searchOffset = 1; searchOffset <= startIndex && searchOffset < 32767; searchOffset++) { size_t currLength = 0; - while (input[startIndex + currLength] == input[startIndex + currLength - searchOffset] - && startIndex + currLength < input.size()) + while (startIndex + currLength < pInput->size() + && (*pInput)[startIndex + currLength] == (*pInput)[startIndex + currLength - searchOffset]) currLength++; + if (currLength > longestLength) { longestLength = currLength; @@ -24,118 +36,118 @@ std::vector getShortCopies(std::vector input, size_t longestLength = 32767; if (longestLength >= minLength) { - std::vector::const_iterator start = input.begin() + startIndex; - std::vector::const_iterator end = input.begin() + startIndex + longestLength; - copies.push_back(ShortCopy(startIndex, longestLength, longestOffset, std::vector(start, end))); - copies[copies.size() - 1].firstSymbol = input[startIndex - 1]; + // Handle non-copies + if (startIndex > 0) + checkVec[startIndex - 1] = 'X'; + + for (size_t i = 0; i < longestLength; i++) + { + checkVec[startIndex + i] = 'X'; + } + for (size_t i = 0; i < startIndex - 1; i++) + { + if (checkVec[i] == ' ') + { + size_t extraIndex = i; + checkVec[extraIndex] = 'O'; + size_t extraLength = 1; + while (checkVec[extraIndex + extraLength] == ' ') + { + checkVec[extraIndex + extraLength] = 'O'; + extraLength++; + } + pShortCopies->push_back(ShortCopy(extraIndex, extraLength, 0, 0)); + break; + } + } + + pShortCopies->push_back(ShortCopy(startIndex, longestLength, longestOffset, (*pInput)[startIndex - 1])); startIndex += longestLength; } } - std::vector checkVec(input.size()); - for (ShortCopy copy : copies) - for (size_t i = 0; i <= copy.length; i++) - checkVec[copy.index + i - 1]++; - size_t currStart = 0; - size_t currLength = 1; - unsigned short prevSym = checkVec[0]; - for (size_t i = 1; i < checkVec.size(); i++) - { - unsigned short currSym = checkVec[i]; - if (currSym == 0 && prevSym == 0) - currLength++; - else if (currSym == 0 && prevSym == 1) - { - currStart = i; - currLength = 1; - } - else if (currSym == 1 && prevSym == 0) - { - std::vector::const_iterator start = input.begin() + currStart; - std::vector::const_iterator end = input.begin() + currStart + currLength; - copies.push_back(ShortCopy(currStart, currLength, 0, std::vector(start, end))); - } - prevSym = currSym; - } - if (prevSym == 0) - { - std::vector::const_iterator start = input.begin() + currStart; - std::vector::const_iterator end = input.begin() + currStart + currLength; - copies.push_back(ShortCopy(currStart, currLength, 0, std::vector(start, end))); - } - return copies; -} - - -bool verifyShortCopies(std::vector *pCopies, std::vector *pImage) -{ - std::vector decodedImage(pImage->size()); - for (ShortCopy copy : (*pCopies)) + // Handle trailing characters + if (checkVec.back() == ' ') { - if (copy.offset == 0) + size_t index = 0; + if (pShortCopies->size() > 0) { - for (size_t i = 0; i < copy.length; i++) - decodedImage[copy.index + i] = copy.usSequence[i]; + index = pShortCopies->back().index + pShortCopies->back().length; + size_t length = checkVec.size() - index; + pShortCopies->push_back(ShortCopy(index, length, 0, 0)); } else { - decodedImage[copy.index - 1] = copy.firstSymbol; - for (size_t i = 0; i < copy.length; i++) - decodedImage[copy.index + i] = (*pImage)[copy.index + i - copy.offset]; + pShortCopies->push_back(ShortCopy(0, checkVec.size(), 0, 0)); } } - for (size_t i = 0; i < decodedImage.size(); i++) - if (decodedImage[i] != (*pImage)[i]) - return false; + + return verifyShortCopies(pShortCopies, pInput); +} + +bool verifyShortCopies(std::vector *pCopies, std::vector *pImage) +{ + size_t totalLength = 0; + for (ShortCopy copy : (*pCopies)) + { + totalLength += copy.length; + if (copy.offset != 0) + { + totalLength++; + } + + if (copy.offset != 0) + { + if (copy.firstSymbol != (*pImage)[copy.index - 1]) + return false; + + for (size_t i = 0; i < copy.length; i++) + { + if ((*pImage)[copy.index + i] != (*pImage)[copy.index + i - copy.offset]) + return false; + } + } + } + if (totalLength != pImage->size()) + return false; + return true; } -SortedShortElement::SortedShortElement() {} - -SortedShortElement::SortedShortElement(size_t index, ShortCopy copy) +bool getShortInstructions(std::vector *pCopies, std::vector *pInstructions, std::vector *pInput) { - this->index = index; - this->copy = copy; - isCopy = true; -} - -std::vector getShortInstructions(std::vector copies, size_t lengthMod) -{ - std::vector instructions; - std::vector unsortedElements; - std::vector sortedElements; - - for (ShortCopy copy : copies) - unsortedElements.push_back(SortedShortElement(copy.index, copy)); - - while (unsortedElements.size() != 0) - { - size_t smallestIndex = 0; - for (size_t i = 0; i < unsortedElements.size(); i++) - { - if (unsortedElements[i].index < unsortedElements[smallestIndex].index) - { - smallestIndex = i; - } - } - sortedElements.push_back(unsortedElements[smallestIndex]); - unsortedElements.erase(unsortedElements.begin() + smallestIndex); - } - for (size_t i = 0; i < sortedElements.size(); i++) + for (ShortCopy copy : (*pCopies)) { ShortCompressionInstruction currInstruction; - currInstruction.index = sortedElements[i].index; - currInstruction.offset = sortedElements[i].copy.offset; - currInstruction.length = sortedElements[i].copy.length; - currInstruction.symbols = sortedElements[i].copy.usSequence; - currInstruction.firstSymbol = sortedElements[i].copy.firstSymbol; - currInstruction.buildBytes(); - instructions.push_back(currInstruction); + currInstruction.index = copy.index; + currInstruction.offset = copy.offset; + currInstruction.length = copy.length; + currInstruction.firstSymbol = copy.firstSymbol; + currInstruction.buildBytes(pInput); + pInstructions->push_back(currInstruction); } - return instructions; + return true; } -void ShortCompressionInstruction::buildBytes() +void getLosFromInstructions(std::vector *pInstructions, std::vector *pOutput) +{ + for (ShortCompressionInstruction instruction : (*pInstructions)) + { + for (unsigned char uc : instruction.loBytes) + pOutput->push_back(uc); + } +} + +void getSymsFromInstructions(std::vector *pInstructions, std::vector *pOutput) +{ + for (ShortCompressionInstruction instruction : (*pInstructions)) + { + for (unsigned short us : instruction.symShorts) + pOutput->push_back(us); + } +} + +void ShortCompressionInstruction::buildBytes(std::vector *pInput) { if (offset != 0) { @@ -173,74 +185,14 @@ void ShortCompressionInstruction::buildBytes() loBytes[loBytes.size() - 1] += LO_CONTINUE_BIT; loBytes.push_back(currLength & BYTE_MASK); } - for (unsigned short currSymbol : symbols) - symShorts.push_back(currSymbol); - } -} - -void CompressionInstruction::buildBytes() -{ - std::vector currBytes; - if (offset != 0) - { - size_t currLength = length; - currBytes.push_back(currLength & 0x7f); - currLength = currLength >> 7; - if (currLength != 0) + for (size_t i = 0; i < length; i++) { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currLength & 0xff); + symShorts.push_back((*pInput)[index + i]); } - size_t currOffset = offset; - currBytes.push_back(currOffset & 0x7f); - currOffset = currOffset >> 7; - if (currOffset != 0) - { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currOffset & 0xff); - } - currBytes.push_back(firstSymbol); } - else - { - currBytes.push_back(0); - size_t currLength = length; - currBytes.push_back(currLength & 0x7f); - currLength = currLength >> 7; - if (currLength != 0) - { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currLength & 0xff); - } - for (unsigned char currSymbol : symbols) - currBytes.push_back(currSymbol); - } - bytes = currBytes; } -std::vector getLosFromInstructions(std::vector instructions) -{ - std::vector loVec; - for (ShortCompressionInstruction instruction : instructions) - { - for (unsigned char uc : instruction.loBytes) - loVec.push_back(uc); - } - return loVec; -} - -std::vector getSymsFromInstructions(std::vector instructions) -{ - std::vector symvec; - for (ShortCompressionInstruction instruction : instructions) - { - for (unsigned short uc : instruction.symShorts) - symvec.push_back(uc); - } - return symvec; -} - -std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec) +bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage) { std::vector decodedImage; size_t loIndex = 0; @@ -279,13 +231,7 @@ std::vector decodeBytesShort(std::vector *pLoVec, } } } - return decodedImage; -} - -bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage) -{ - std::vector shorts = decodeBytesShort(pLoVec, pSymVec); - return compareVectorsShort(&shorts, pImage); + return true; } bool compareVectorsShort(std::vector *pVec1, std::vector *pVec2) @@ -298,407 +244,38 @@ bool compareVectorsShort(std::vector *pVec1, std::vector readFileAsUC(std::string filePath) +bool isModeLoEncoded(CompressionMode mode) { - std::ifstream iStream; - iStream.open(filePath.c_str(), std::ios::binary); - if (!iStream.is_open()) - { - fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); - return std::vector(0); - } - iStream.ignore( std::numeric_limits::max() ); - std::streamsize size = iStream.gcount(); - iStream.clear(); - iStream.seekg( 0, std::ios_base::beg ); - std::vector ucVec(size); - iStream.read((char*)(&ucVec[0]), size); - iStream.close(); - return ucVec; + if (mode == ENCODE_LO + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImage(std::string fileName, InputSettings settings) +bool isModeSymEncoded(CompressionMode mode) { - CompressedImage image; - std::vector input = readFileAsUC(fileName); - if (settings.useFrames) - { - /* - // Determine number of frames - size_t totalPixels = input.size()*2; - // Split input and append - size_t smallFrames = totalPixels/OVERWORLD_16X16; - size_t largeFrames = totalPixels/OVERWORLD_32X32; - */ - } - else - { - image = processImageData(input, settings, fileName); - } - return image; + if (mode == ENCODE_SYMS + || mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImageFrames(std::string fileName, InputSettings settings) +bool isModeSymDelta(CompressionMode mode) { - CompressedImage image; - std::vector input = readFileAsUC(fileName); - std::vector> allInputs(4); - size_t totalSize = input.size(); - size_t partialSize = totalSize/4; - size_t subIndex = 0; - size_t inputIndex = 0; - std::vector frameOffsets; - for (unsigned char currChar : input) - { - frameOffsets.push_back(image.otherBits.size()); - if (subIndex == partialSize) - { - subIndex = 0; - inputIndex++; - } - allInputs[inputIndex].push_back(currChar); - subIndex++; - } - for (size_t i = 0; i < 4; i++) - { - CompressedImage tempImage = processImageData(allInputs[i], settings, fileName); - for (unsigned int currVal : tempImage.writeVec) - image.otherBits.push_back(currVal); - } - unsigned int header = IS_FRAME_CONTAINER; - image.writeVec.push_back(header); - for (size_t i = 0; i < 4; i++) - image.writeVec.push_back((unsigned int)frameOffsets[i]); - for (unsigned int currVal : image.otherBits) - image.writeVec.push_back(currVal); - image.isValid = true; - - return image; + if (mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImageData(std::vector input, InputSettings settings, std::string fileName) -{ - CompressedImage bestImage; - CompressionMode someMode; - bool hasImage = false; - std::vector rawBase = input; - std::vector usBase(rawBase.size()/2); - memcpy(usBase.data(), rawBase.data(), rawBase.size()); - size_t baseLZsize = 0; - bool byteFail = false; - bool copyFail = false; - bool compressionFail = false; - bool uIntConversionFail = false; - if (settings.shouldCompare) - { - std::string lzName = fileName + ".lz"; - baseLZsize = getFileSize(lzName); - } - std::vector bestLO; - std::vector bestSym; - std::vector bestInstructions; - for (size_t minCodeLength = 2; minCodeLength <= 15; minCodeLength++) - { - std::vector shortCopies = getShortCopies(usBase, minCodeLength); - if (!verifyShortCopies(&shortCopies, &usBase)) - { - copyFail = true; - continue; - } - std::vector shortInstructions = getShortInstructions(shortCopies, minCodeLength-1); - std::vector loVec = getLosFromInstructions(shortInstructions); - std::vector symVec = getSymsFromInstructions(shortInstructions); - if (!verifyBytesShort(&loVec, &symVec, &usBase)) - { - byteFail = true; - continue; - } - CompressionMode mode = BASE_ONLY; - //std::vector modesToUse = {ENCODE_SYMS}; - std::vector modesToUse = {BASE_ONLY, ENCODE_SYMS, ENCODE_DELTA_SYMS, ENCODE_LO, ENCODE_BOTH, ENCODE_BOTH_DELTA_SYMS}; - if (fileName.find("test/compression/") != std::string::npos) - { - if (fileName.find("mode_0.4bpp") != std::string::npos) - modesToUse = {BASE_ONLY}; - else if (fileName.find("mode_1.4bpp") != std::string::npos) - modesToUse = {ENCODE_SYMS}; - else if (fileName.find("mode_2.4bpp") != std::string::npos) - modesToUse = {ENCODE_DELTA_SYMS}; - else if (fileName.find("mode_3.4bpp") != std::string::npos) - modesToUse = {ENCODE_LO}; - else if (fileName.find("mode_4.4bpp") != std::string::npos) - modesToUse = {ENCODE_BOTH}; - else if (fileName.find("mode_5.4bpp") != std::string::npos) - modesToUse = {ENCODE_BOTH_DELTA_SYMS}; - else if (fileName.find("test/compression/table_") != std::string::npos) - modesToUse = {ENCODE_SYMS}; - - if (modesToUse.size() == 1) - { - settings.canDeltaSyms = true; - settings.canEncodeLO = true; - settings.canEncodeSyms = true; - } - } - - for (CompressionMode currMode : modesToUse) - { - mode = currMode; - if (!settings.canDeltaSyms - && (mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - if (!settings.canEncodeLO - && (mode == ENCODE_LO - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - if (!settings.canEncodeSyms - && (mode == ENCODE_SYMS - || mode == ENCODE_BOTH - || mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - CompressedImage image = fillCompressVecNew(loVec, symVec, mode,rawBase.size(), fileName); - if (!verifyCompressionShort(&image, &usBase)) - { - compressionFail = true; - continue; - } - std::vector uiVec = getUIntVecFromData(&image); - std::vector decodedImage = readRawDataVecs(&uiVec); - if (!compareVectorsShort(&decodedImage, &usBase)) - { - uIntConversionFail = true; - continue; - } - image.compressedSize = uiVec.size() * 4; - if (!hasImage) - { - bestLO = loVec; - bestSym = symVec; - bestInstructions = shortInstructions; - bestImage = image; - hasImage = true; - bestImage.writeVec = uiVec; - someMode = mode; - } - else if (image.compressedSize < bestImage.compressedSize) - { - bestLO = loVec; - bestSym = symVec; - bestInstructions = shortInstructions; - bestImage = image; - hasImage = true; - bestImage.writeVec = uiVec; - someMode = mode; - } - } - } - bestImage.mode = someMode; - bestImage.fileName = fileName; - bestImage.lzSize = baseLZsize; - bestImage.rawNumBytes = rawBase.size(); - if (hasImage) - bestImage.isValid = true; - else - { - fprintf(stderr, "Failed to compress image %s\nErrors: ", fileName.c_str()); - if (copyFail) - fprintf(stderr, "CopyProcessing "); - if (byteFail) - fprintf(stderr, "ByteConversion "); - if (compressionFail) - fprintf(stderr, "Compression "); - if (uIntConversionFail) - fprintf(stderr, "uIntConversion "); - printf("\n"); - } - return bestImage; -} - -CompressedImage getDataFromUIntVec(std::vector *pInput) -{ - CompressedImage image = readNewHeader(pInput); - - size_t readIndex = 2; - - bool loEncoded = isModeLoEncoded(image.mode); - bool symEncoded = isModeSymEncoded(image.mode); - - if (loEncoded) - { - image.loFreqs[0] = (*pInput)[readIndex]; - image.loFreqs[1] = (*pInput)[readIndex + 1]; - image.loFreqs[2] = (*pInput)[readIndex + 2]; - readIndex += 3; - } - if (symEncoded) - { - image.symFreqs[0] = (*pInput)[readIndex]; - image.symFreqs[1] = (*pInput)[readIndex + 1]; - image.symFreqs[2] = (*pInput)[readIndex + 2]; - readIndex += 3; - } - if (loEncoded || symEncoded) - { - for (size_t i = 0; i < image.bitreamSize; i++) - { - image.tANSbits.push_back((*pInput)[readIndex]); - readIndex++; - } - } - std::vector remainders; - while (readIndex < pInput->size()) - { - unsigned int currInt = (*pInput)[readIndex]; - for (size_t i = 0; i < 4; i++) - remainders.push_back((currInt >> (8*i)) & 0xff); - readIndex++; - } - size_t remIndex = 0; - if (!symEncoded) - { - for (size_t i = 0; i < image.symSize; i++) - { - unsigned short currSym = remainders[remIndex]; - remIndex++; - currSym += remainders[remIndex] << 8; - remIndex++; - image.symVec.push_back(currSym); - } - } - if (!loEncoded) - { - for (size_t i = 0; i < image.loSize; i++) - { - unsigned char currChar = remainders[remIndex]; - remIndex++; - image.loVec.push_back(currChar); - } - } - - return image; -} - -bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage) -{ - std::vector decodedImage = decodeImageShort(pInput); - return compareVectorsShort(&decodedImage, pImage); -} - -std::vector decodeImageShort(CompressedImage *pInput) -{ - DataVecs dataVecs = decodeDataVectorsNew(pInput); - return decodeBytesShort(&dataVecs.loVec, &dataVecs.symVec); -} - -DataVecs decodeDataVectorsNew(CompressedImage *pInput) -{ - CompressedImage headerValues = readNewHeader(&pInput->headers); - size_t loSize = headerValues.loSize; - size_t symSize = headerValues.symSize; - CompressionMode mode = headerValues.mode; - bool loEncoded = isModeLoEncoded(mode); - bool symEncoded = isModeSymEncoded(mode); - bool symDelta = isModeSymDelta(mode); - std::vector loFreqs = unpackFrequencies(&pInput->loFreqs[0]); - std::vector symFreqs = unpackFrequencies(&pInput->symFreqs[0]); - std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - std::vector loDecode; - std::vector symDecode; - - std::vector loVec(loSize); - std::vector symVec(symSize); - std::vector loNibbles(loSize*2); - std::vector symNibbles(symSize*4); - - if (loEncoded) - { - loDecode = createDecodingTable(symbols, loFreqs); - } - if (symEncoded) - { - symDecode = createDecodingTable(symbols, symFreqs); - } - - std::vector allBits(pInput->tANSbits.size()*32); - size_t currIndex = 0; - for (unsigned int ui : pInput->tANSbits) - for (size_t i = 0; i < 32; i++) - { - unsigned int currVal = (ui >> i) & 0x1; - allBits[currIndex] = currVal; - currIndex++; - } - - if (!symEncoded) - for (size_t i = 0; i < symSize; i++) - symVec[i] = pInput->symVec[i]; - if (!loEncoded) - for (size_t i = 0; i < loSize; i++) - loVec[i] = pInput->loVec[i]; - - size_t bitIndex = 0; - int currState = pInput->initialState; - if (loEncoded) - { - bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, loSize*2); - } - if (loEncoded) - for (size_t i = 0; i < loVec.size(); i++) - loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); - - if (symEncoded) - { - bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, symSize*4); - if (symDelta) - deltaDecode(&symNibbles, symNibbles.size()); - } - if (symEncoded) - for (size_t i = 0; i < symVec.size(); i++) - for (size_t j = 0; j < 4; j++) - symVec[i] += (unsigned short)symNibbles[i*4 + j] << (j*4); - - DataVecs returnData; - returnData.loVec = loVec; - returnData.symVec = symVec; - return returnData; -} - -void analyzeImages(std::vector *allImages, std::mutex *imageMutex, FileDispatcher *dispatcher, std::mutex *dispatchMutex, InputSettings settings) -{ - std::string fileName = "Initial Value"; - while (fileName != "") - { - dispatchMutex->lock(); - fileName = dispatcher->requestFileName(); - dispatchMutex->unlock(); - if (fileName == "") - break; - std::string uncompressedFileName = fileName.substr(0, fileName.size()-3); - CompressedImage currImage = processImage(uncompressedFileName, settings); - - imageMutex->lock(); - allImages->push_back(currImage); - imageMutex->unlock(); - } -} - -ShortCopy::ShortCopy() {} -ShortCopy::ShortCopy(size_t index, size_t length, size_t offset, std::vector usSequence) -{ - this->index = index; - this->length = length; - this->offset = offset; - this->usSequence = usSequence; -} - -std::vector getNormalizedCounts(std::vector input) +std::vector getNormalizedCounts(std::vector *input) { std::vector tempVec(16); - for (size_t i = 0; i < input.size(); i++) - tempVec[i] = input[i]; + for (size_t i = 0; i < input->size(); i++) + tempVec[i] = (*input)[i]; tempVec = normalizeCounts(tempVec, TANS_TABLE_SIZE); bool shouldAdjust = false; for (int i = 0; i < 16; i++) @@ -742,499 +319,6 @@ std::vector getFreqWriteInts(std::vector input) return returnVec; } -std::vector unpackFrequencies(unsigned int pInts[3]) -{ - std::vector returnVec; - int freq15 = 0; - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 5; j++) - { - int currVal = (pInts[i] >> (6*j)) & 0x3f; - returnVec.push_back(currVal); - } - freq15 += ((pInts[i] >> 30) & 0x3) << (2*i); - } - returnVec.push_back(freq15); - return returnVec; -} - -int findInitialState(EncodeCol encodeCol, unsigned char firstSymbol) -{ - for (size_t i = 0; i < encodeCol.symbols.size(); i++) - { - if (encodeCol.symbols[i].symbol == firstSymbol) - return encodeCol.symbols[i].nextState; - } - return -1; -} -/* - mode 5 - - state 6 11 - bitstream 12 23 8628 max so far, divide by 4 - loLength 12 23 2922 max so far - symLength 14 37 14k max so far - imageSize 10 47 16384 max so far, divide by 16 - - u32:5 mode+lz - u32:10 image size, divided by 32 - u32:14 symlength, divided by 2 - - u32:6 state - u32:12 bitstream, in ints - u32:12 loLength, in bytes -*/ -std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength) -{ - if (initialState == -1) - initialState = 0; - std::vector returnVec(2); - - returnVec[0] += (unsigned int)mode; // 4 bits - returnVec[0] += (imageSize/IMAGE_SIZE_MODIFIER) << IMAGE_SIZE_OFFSET; // 14 bits - returnVec[0] += (symLength) << SYM_SIZE_OFFSET; // 14 bits - // 32 total - - returnVec[1] += initialState; // 6 bits - returnVec[1] += bitstreamSize << BITSTREAM_SIZE_OFFSET; // 13 bits - returnVec[1] += loLength << LO_SIZE_OFFSET; // 13 bits - - return returnVec; -} - -CompressedImage readNewHeader(std::vector *pInput) -{ - CompressedImage image; - std::vector headers(2); - headers[0] = (*pInput)[0]; - headers[1] = (*pInput)[1]; - image.mode = (CompressionMode)(headers[0] & MODE_MASK); - image.rawNumBytes = ((headers[0] >> IMAGE_SIZE_OFFSET) & IMAGE_SIZE_MASK) * IMAGE_SIZE_MODIFIER; - image.symSize = ((headers[0] >> SYM_SIZE_OFFSET) & SYM_SIZE_MASK); - - image.initialState = headers[1] & INITIAL_STATE_MASK; - image.bitreamSize = (headers[1] >> BITSTREAM_SIZE_OFFSET) & BITSTREAM_SIZE_MASK; - image.loSize = (headers[1] >> LO_SIZE_OFFSET) & LO_SIZE_MASK; - image.headers = headers; - return image; -} - -InputSettings::InputSettings() {} - -InputSettings::InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms) -{ - this->canEncodeLO = canEncodeLO; - this->canEncodeSyms = canEncodeSyms; - this->canDeltaSyms = canDeltaSyms; -} - -std::vector getUIntVecFromData(CompressedImage *pImage) -{ - CompressedImage otherImage = readNewHeader(&pImage->headers); - std::vector returnVec; - returnVec.push_back(pImage->headers[0]); - returnVec.push_back(pImage->headers[1]); - if (isModeLoEncoded(otherImage.mode)) - for (size_t i = 0; i < 3; i++) - returnVec.push_back(pImage->loFreqs[i]); - if (isModeSymEncoded(otherImage.mode)) - for (size_t i = 0; i < 3; i++) - returnVec.push_back(pImage->symFreqs[i]); - if (isModeLoEncoded(otherImage.mode) || isModeSymEncoded(otherImage.mode)) - for (unsigned int ui : pImage->tANSbits) - returnVec.push_back(ui); - unsigned int currInt = 0; - unsigned int currOffset = 0; - if (!isModeSymEncoded(otherImage.mode)) - { - for (size_t i = 0; i < pImage->symVec.size(); i++) - { - currInt += pImage->symVec[i] << (8*(currOffset % 4)); - currOffset += 2; - if (currOffset % 4 == 0) - { - returnVec.push_back(currInt); - currInt = 0; - } - } - } - if (!isModeLoEncoded(otherImage.mode)) - { - for (size_t i = 0; i < pImage->loVec.size(); i++) - { - currInt += 0; - currInt += pImage->loVec[i] << (8*(currOffset % 4)); - currOffset++; - if (currOffset % 4 == 0) - { - returnVec.push_back(currInt); - currInt = 0; - } - } - } - if (currOffset != 0) - returnVec.push_back(currInt); - return returnVec; -} - -bool verifyUIntVecShort(std::vector *pInput, std::vector *pImage) -{ - CompressedImage image = getDataFromUIntVec(pInput); - return verifyCompressionShort(&image, pImage); -} - -std::vector readFileAsUInt(std::string filePath) -{ - std::vector returnVec; - std::vector ucVec; - std::ifstream iStream; - iStream.open(filePath.c_str(), std::ios::binary); - if (!iStream.is_open()) - { - fprintf(stderr, "Error: Couldn't open file %s for reading\n", filePath.c_str()); - return returnVec; - } - iStream.ignore( std::numeric_limits::max() ); - std::streamsize size = iStream.gcount(); - ucVec.resize(size); - iStream.clear(); - iStream.seekg( 0, std::ios_base::beg ); - iStream.read((char*)(&ucVec[0]), size); - iStream.close(); - unsigned int *pUInt = reinterpret_cast(ucVec.data()); - for (size_t i = 0; i < ucVec.size()/4; i++) - returnVec.push_back(pUInt[i]); - return returnVec; -} - -bool isModeLoEncoded(CompressionMode mode) -{ - if (mode == ENCODE_LO - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -bool isModeSymEncoded(CompressionMode mode) -{ - if (mode == ENCODE_SYMS - || mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -bool isModeSymDelta(CompressionMode mode) -{ - if (mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -CompressedImage fillCompressVecNew(std::vector loVec, std::vector symVec, CompressionMode mode, size_t imageBytes, std::string name) -{ - CompressedImage image; - bool loEncoded = isModeLoEncoded(mode); - bool symEncoded = isModeSymEncoded(mode); - bool symDelta = isModeSymDelta(mode); - - std::vector loNibbles(2*loVec.size()); - std::vector symNibbles(4*symVec.size()); - std::vector loDecode; - std::vector symDecode; - std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - if (loEncoded) - { - for (size_t i = 0; i < loVec.size(); i++) - { - loNibbles[2*i] = (loVec)[i] & 0xf; - loNibbles[2*i + 1] = ((loVec)[i] >> 4) & 0xf; - } - std::vector loCounts(16); - for (unsigned char uc : loNibbles) - loCounts[uc]++; - std::vector loFreqs = getNormalizedCounts(loCounts); - std::vector freqs = getFreqWriteInts(loFreqs); - image.loFreqs[0] = freqs[0]; - image.loFreqs[1] = freqs[1]; - image.loFreqs[2] = freqs[2]; - loDecode = createDecodingTable(symbols, loFreqs); - } - if (symEncoded) - { - for (size_t i = 0; i < symVec.size(); i++) - { - for (size_t j = 0; j < 4; j++) - symNibbles[4*i + j] = (symVec[i] >> (4*j)) & 0xf; - } - std::vector symCounts(16); - if (symDelta) - deltaEncode(&symNibbles, symNibbles.size()); - for (unsigned char uc : symNibbles) - symCounts[uc]++; - std::vector symFreqs = getNormalizedCounts(symCounts); - symFreqs = getTestFreqs(symFreqs, name); - std::vector freqs = getFreqWriteInts(symFreqs); - image.symFreqs[0] = freqs[0]; - image.symFreqs[1] = freqs[1]; - image.symFreqs[2] = freqs[2]; - symDecode = createDecodingTable(symbols, symFreqs); - } - int currState = -1; - std::vector bitstream; - if (symEncoded) - { - std::vector reversedVec; - for (size_t i = 0; i < symNibbles.size(); i++) - reversedVec.push_back(symNibbles[symNibbles.size() - i - 1]); - std::vector symEncode = createEncodingTable(symDecode, symbols); - size_t startIndex = 0; - if (currState == -1) - { - currState = findInitialState(symEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; - startIndex = 1; - } - for (size_t i = startIndex; i < reversedVec.size(); i++) - currState = encodeSingleSymbol(symEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; - } - if (loEncoded) - { - std::vector reversedVec; - for (size_t i = 0; i < loNibbles.size(); i++) - reversedVec.push_back(loNibbles[loNibbles.size() - i - 1]); - std::vector loEncode = createEncodingTable(loDecode, symbols); - size_t startIndex = 0; - if (currState == -1) - { - currState = findInitialState(loEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; - startIndex = 1; - } - for (size_t i = startIndex; i < reversedVec.size(); i++) - currState = encodeSingleSymbol(loEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; - } - std::vector reversedBitstream(bitstream.size()); - for (size_t i = 0; i < bitstream.size(); i++) - reversedBitstream[reversedBitstream.size() - 1 - i] = bitstream[i]; - bitstream = reversedBitstream; - std::vector checkBits = bitstream; - int checkState = currState; - - size_t currBitIndex = 0; - if (loEncoded) - { - std::vector checkLoNibbles(loNibbles.size()); - currBitIndex = decodeNibbles(loDecode, &checkBits, &checkState, &checkLoNibbles, currBitIndex, loNibbles.size()); - for (size_t i = 0; i < loNibbles.size(); i++) - if (loNibbles[i] != checkLoNibbles[i]) - { - fprintf(stderr, "LO Mismatch\n"); - break; - } - } - if (symEncoded) - { - std::vector checkSymNibbles(symNibbles.size()); - currBitIndex = decodeNibbles(symDecode, &checkBits, &checkState, &checkSymNibbles, currBitIndex, symNibbles.size()); - for (size_t i = 0; i < symNibbles.size(); i++) - if (symNibbles[i] != checkSymNibbles[i]) - { - fprintf(stderr, "Symbol Mismatch\n"); - break; - } - } - - std::vector tANSbits; - unsigned int currInt = 0; - for (size_t i = 0; i < bitstream.size(); i++) - { - currInt += bitstream[i] << (i%32); - if ((i+1) % 32 == 0) - { - tANSbits.push_back(currInt); - currInt = 0; - } - } - if (bitstream.size() % 32 != 0) - tANSbits.push_back(currInt); - image.headers = getNewHeaders(mode, imageBytes, symVec.size(), currState, tANSbits.size(), loVec.size()); - image.tANSbits = tANSbits; - image.symVec = symVec; - image.loVec = loVec; - image.initialState = currState; - return image; -} - -size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles) -{ - for (size_t i = 0; i < numNibbles; i++) - { - (*nibbleVec)[i] = decodeTable[*currState].symbol; - int currK = decodeTable[*currState].k; - int nextState = decodeTable[*currState].y << currK; - for (size_t j = 0; j < currK; j++) - { - nextState += (*bits)[currBitIndex] << j; - currBitIndex++; - } - *currState = nextState - TANS_TABLE_SIZE; - } - return currBitIndex; -} - -std::vector readRawDataVecs(std::vector *pInput) -{ - std::vector imageVec; - CompressedImage readImage = readNewHeader(pInput); - bool loEncoded = isModeLoEncoded(readImage.mode); - bool symEncoded = isModeSymEncoded(readImage.mode); - bool symDelta = isModeSymDelta(readImage.mode); - size_t readIndex = 2; - std::vector tANSbits; - std::vector allBits; - std::vector symVec(readImage.symSize); - std::vector loVec(readImage.loSize); - std::vector symDecode(TANS_TABLE_SIZE); - std::vector loDecode(TANS_TABLE_SIZE); - int currState = readImage.initialState; - if (loEncoded) - { - for (size_t i = 0; i < 3; i++) - readImage.loFreqs[i] = (*pInput)[readIndex + i]; - readIndex += 3; - std::vector loFreqs = unpackFrequencies(&readImage.loFreqs[0]); - size_t currCol = 0; - for (size_t i = 0; i < 16; i++) - { - for (size_t j = 0; j < loFreqs[i]; j++) - { - loDecode[currCol].state = TANS_TABLE_SIZE + currCol; - loDecode[currCol].symbol = i; - loDecode[currCol].y = loFreqs[i] + j; - int currK = 0; - while ((loDecode[currCol].y << currK) < TANS_TABLE_SIZE) - currK++; - loDecode[currCol].k = currK; - currCol++; - } - } - } - if (symEncoded) - { - for (size_t i = 0; i < 3; i++) - readImage.symFreqs[i] = (*pInput)[readIndex + i]; - readIndex += 3; - std::vector symFreqs = unpackFrequencies(&readImage.symFreqs[0]); - size_t currCol = 0; - for (size_t i = 0; i < 16; i++) - { - for (size_t j = 0; j < symFreqs[i]; j++) - { - symDecode[currCol].state = TANS_TABLE_SIZE + currCol; - symDecode[currCol].symbol = i; - symDecode[currCol].y = symFreqs[i] + j; - int currK = 0; - while ((symDecode[currCol].y << currK) < TANS_TABLE_SIZE) - currK++; - symDecode[currCol].k = currK; - currCol++; - } - } - } - if (loEncoded || symEncoded) - { - tANSbits.resize(readImage.bitreamSize); - for (size_t i = 0; i < readImage.bitreamSize; i++) - tANSbits[i] = (*pInput)[readIndex + i]; - readIndex += readImage.bitreamSize; - allBits.resize(tANSbits.size()*32); - size_t currIndex = 0; - for (unsigned int ui : tANSbits) - { - for (size_t i = 0; i < 32; i++) - { - unsigned int currVal = (ui >> i) & 0x1; - allBits[currIndex] = currVal; - currIndex++; - } - } - } - bool leftOverValues = false; - if (!symEncoded) - { - for (size_t i = 0; i < readImage.symSize; i++) - { - symVec[i] = ((*pInput)[readIndex] >> (16*(i%2))) & 0xffff; - if ((i+1) % 2 == 0) - { - readIndex++; - leftOverValues = false; - } - else - { - leftOverValues = true; - } - } - } - if (!loEncoded) - { - size_t offsetMod = 0; - if (leftOverValues) - offsetMod = 2; - for (size_t i = 0; i < readImage.loSize; i++) - { - loVec[i] = ((*pInput)[readIndex] >> (8*((i%4) + offsetMod))) & 0xff; - if ((i+offsetMod+1) % 4 == 0) - readIndex++; - } - } - size_t bitIndex = 0; - if (loEncoded) - { - std::vector loNibbles(readImage.loSize*2); - bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, readImage.loSize*2); - for (size_t i = 0; i < readImage.loSize; i++) - loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); - } - if (symEncoded) - { - std::vector symNibbles(readImage.symSize*4); - bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, readImage.symSize*4); - if (symDelta) - deltaDecode(&symNibbles, symNibbles.size()); - for (size_t i = 0; i < readImage.symSize; i++) - for (size_t j = 0; j < 4; j++) - symVec[i] += (unsigned short)symNibbles[i*4 + j] << (4*j); - } - imageVec = decodeBytesShort(&loVec, &symVec); - return imageVec; -} - -void deltaEncode(std::vector *buffer, int length) -{ - unsigned char last = 0; - for (int i = 0; i < length; i++) - { - unsigned char current =(*buffer)[i]; - (*buffer)[i] = (current-last) & 0xf; - last = current; - } -} - -void deltaDecode(std::vector *buffer, int length) -{ - unsigned char last = 0; - for (int i = 0; i < length; i++) - { - unsigned char delta =(*buffer)[i]; - (*buffer)[i] = (delta+last) & 0xf; - last = (*buffer)[i]; - } -} - std::vector getTestFreqs(std::vector freqs, std::string name) { if (name.find("test/compression/table_") == std::string::npos) @@ -1465,3 +549,783 @@ std::vector getTestFreqs(std::vector freqs, std::string name) } return freqs; } + +void deltaEncode(std::vector *buffer, int length) +{ + unsigned char last = 0; + for (int i = 0; i < length; i++) + { + unsigned char current =(*buffer)[i]; + (*buffer)[i] = (current-last) & 0xf; + last = current; + } +} + +void deltaDecode(std::vector *buffer, int length) +{ + unsigned char last = 0; + for (int i = 0; i < length; i++) + { + unsigned char delta =(*buffer)[i]; + (*buffer)[i] = (delta+last) & 0xf; + last = (*buffer)[i]; + } +} + +int findInitialState(EncodeCol *encodeCol, unsigned char firstSymbol) +{ + for (size_t i = 0; i < encodeCol->symbols.size(); i++) + { + if (encodeCol->symbols[i].symbol == firstSymbol) + return encodeCol->symbols[i].nextState; + } + return -1; +} + +size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles, bool lastThing) +{ + for (size_t i = 0; i < numNibbles; i++) + { + (*nibbleVec)[i] = decodeTable[*currState].symbol; + + if (i + 1 == nibbleVec->size() && lastThing) + return currBitIndex; + + int currK = decodeTable[*currState].k; + int nextState = decodeTable[*currState].y << currK; + for (size_t j = 0; j < currK; j++) + { + nextState += (*bits)[currBitIndex] << j; + currBitIndex++; + } + *currState = nextState - TANS_TABLE_SIZE; + } + return currBitIndex; +} + +std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength) +{ + if (initialState == -1) + initialState = 0; + std::vector returnVec(2); + + returnVec[0] += (unsigned int)mode; // 4 bits + returnVec[0] += (imageSize/IMAGE_SIZE_MODIFIER) << IMAGE_SIZE_OFFSET; // 14 bits + returnVec[0] += (symLength) << SYM_SIZE_OFFSET; // 14 bits + // 32 total + + returnVec[1] += initialState; // 6 bits + returnVec[1] += bitstreamSize << BITSTREAM_SIZE_OFFSET; // 13 bits + returnVec[1] += loLength << LO_SIZE_OFFSET; // 13 bits + // 32 total + + return returnVec; +} + +bool fillCompressVec(std::vector *pLoVec, std::vector *pSymVec, CompressionMode mode, size_t imageBytes, std::string name, CompressedImage *pOutput) +{ + bool loEncoded = isModeLoEncoded(mode); + bool symEncoded = isModeSymEncoded(mode); + bool symDelta = isModeSymDelta(mode); + + std::vector loNibbles(2 * pLoVec->size()); + std::vector symNibbles(4 * pSymVec->size()); + std::vector loDecode; + std::vector symDecode; + std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + if (loEncoded) + { + for (size_t i = 0; i < pLoVec->size(); i++) + { + loNibbles[2*i] = (*pLoVec)[i] & 0xf; + loNibbles[2*i + 1] = ((*pLoVec)[i] >> 4) & 0xf; + } + std::vector loCounts(16); + for (unsigned char uc : loNibbles) + loCounts[uc]++; + std::vector loFreqs = getNormalizedCounts(&loCounts); + std::vector freqs = getFreqWriteInts(loFreqs); + pOutput->loFreqs[0] = freqs[0]; + pOutput->loFreqs[1] = freqs[1]; + pOutput->loFreqs[2] = freqs[2]; + loDecode = createDecodingTable(symbols, loFreqs); + } + + if (symEncoded) + { + for (size_t i = 0; i < pSymVec->size(); i++) + { + for (size_t j = 0; j < 4; j++) + symNibbles[4*i + j] = ((*pSymVec)[i] >> (4*j)) & 0xf; + } + std::vector symCounts(16); + if (symDelta) + deltaEncode(&symNibbles, symNibbles.size()); + for (unsigned char uc : symNibbles) + symCounts[uc]++; + std::vector symFreqs = getNormalizedCounts(&symCounts); + symFreqs = getTestFreqs(symFreqs, name); + std::vector freqs = getFreqWriteInts(symFreqs); + pOutput->symFreqs[0] = freqs[0]; + pOutput->symFreqs[1] = freqs[1]; + pOutput->symFreqs[2] = freqs[2]; + symDecode = createDecodingTable(symbols, symFreqs); + } + + int currState = -1; + std::vector bitstream; + if (symEncoded) + { + std::vector reversedVec; + for (size_t i = 0; i < symNibbles.size(); i++) + reversedVec.push_back(symNibbles[symNibbles.size() - i - 1]); + std::vector symEncode = createEncodingTable(symDecode, symbols); + size_t startIndex = 0; + if (currState == -1) + { + currState = findInitialState(&symEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; + startIndex = 1; + } + for (size_t i = startIndex; i < reversedVec.size(); i++) + currState = encodeSingleSymbol(symEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; + } + + if (loEncoded) + { + std::vector reversedVec; + for (size_t i = 0; i < loNibbles.size(); i++) + reversedVec.push_back(loNibbles[loNibbles.size() - i - 1]); + std::vector loEncode = createEncodingTable(loDecode, symbols); + size_t startIndex = 0; + if (currState == -1) + { + currState = findInitialState(&loEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; + startIndex = 1; + } + for (size_t i = startIndex; i < reversedVec.size(); i++) + currState = encodeSingleSymbol(loEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; + } + + std::vector reversedBitstream(bitstream.size()); + for (size_t i = 0; i < bitstream.size(); i++) + reversedBitstream[reversedBitstream.size() - 1 - i] = bitstream[i]; + bitstream = reversedBitstream; + std::vector checkBits = bitstream; + int checkState = currState; + size_t currBitIndex = 0; + + if (loEncoded) + { + bool lastThing = !symEncoded; + std::vector checkLoNibbles(loNibbles.size()); + currBitIndex = decodeNibbles(loDecode, &checkBits, &checkState, &checkLoNibbles, currBitIndex, loNibbles.size(), lastThing); + for (size_t i = 0; i < loNibbles.size(); i++) + { + if (loNibbles[i] != checkLoNibbles[i]) + { + fprintf(stderr, "LO Mismatch\n"); + break; + } + } + } + + if (symEncoded) + { + std::vector checkSymNibbles(symNibbles.size()); + currBitIndex = decodeNibbles(symDecode, &checkBits, &checkState, &checkSymNibbles, currBitIndex, symNibbles.size(), true); + for (size_t i = 0; i < symNibbles.size(); i++) + { + if (symNibbles[i] != checkSymNibbles[i]) + { + fprintf(stderr, "Symbol Mismatch\n"); + break; + } + } + } + + std::vector tANSbits; + unsigned int currInt = 0; + for (size_t i = 0; i < bitstream.size(); i++) + { + currInt += bitstream[i] << (i%32); + if ((i+1) % 32 == 0) + { + tANSbits.push_back(currInt); + currInt = 0; + } + } + if (bitstream.size() % 32 != 0) + tANSbits.push_back(currInt); + + pOutput->headers = getNewHeaders(mode, imageBytes, pSymVec->size(), currState, tANSbits.size(), pLoVec->size()); + pOutput->tANSbits = tANSbits; + pOutput->symVec = *pSymVec; + pOutput->loVec = *pLoVec; + pOutput->initialState = currState; + return true; +} + +bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage) +{ + std::vector decodedImage = decodeImageShort(pInput); + return compareVectorsShort(&decodedImage, pImage); +} + +std::vector decodeImageShort(CompressedImage *pInput) +{ + DataVecs dataVecs = decodeDataVectorsNew(pInput); + return decodeBytesShort(&dataVecs.loVec, &dataVecs.symVec); +} + +std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec) +{ + std::vector decodedImage; + size_t loIndex = 0; + size_t symIndex = 0; + while (loIndex < pLoVec->size()) + { + size_t currLength = 0; + size_t currOffset = 0; + currLength += (*pLoVec)[loIndex] & LO_LOW_BITS_MASK; + loIndex++; + if (((*pLoVec)[loIndex-1] & LO_CONTINUE_BIT) == LO_CONTINUE_BIT) + { + currLength += (*pLoVec)[loIndex] << LO_NUM_LOW_BITS; + loIndex++; + } + currOffset += (*pLoVec)[loIndex] & LO_LOW_BITS_MASK; + loIndex++; + if (((*pLoVec)[loIndex-1] & LO_CONTINUE_BIT) == LO_CONTINUE_BIT) + { + currOffset += (*pLoVec)[loIndex] << LO_NUM_LOW_BITS; + loIndex++; + } + if (currLength != 0) + { + decodedImage.push_back((*pSymVec)[symIndex]); + symIndex++; + for (size_t i = 0; i < currLength; i++) + decodedImage.push_back(decodedImage[decodedImage.size() - currOffset]); + } + else + { + for (size_t i = 0; i < currOffset; i++) + { + decodedImage.push_back((*pSymVec)[symIndex]); + symIndex++; + } + } + } + return decodedImage; +} + +DataVecs decodeDataVectorsNew(CompressedImage *pInput) +{ + CompressedImage headerValues; + readNewHeader(&pInput->headers, &headerValues); + size_t loSize = headerValues.loSize; + size_t symSize = headerValues.symSize; + CompressionMode mode = headerValues.mode; + bool loEncoded = isModeLoEncoded(mode); + bool symEncoded = isModeSymEncoded(mode); + bool symDelta = isModeSymDelta(mode); + std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + std::vector loDecode; + std::vector symDecode; + + std::vector loVec(loSize); + std::vector symVec(symSize); + std::vector loNibbles(loSize*2); + std::vector symNibbles(symSize*4); + + if (loEncoded) + { + std::vector loFreqs = unpackFrequencies(&pInput->loFreqs[0]); + loDecode = createDecodingTable(symbols, loFreqs); + } + if (symEncoded) + { + std::vector symFreqs = unpackFrequencies(&pInput->symFreqs[0]); + symDecode = createDecodingTable(symbols, symFreqs); + } + + std::vector allBits(pInput->tANSbits.size()*32); + size_t currIndex = 0; + for (unsigned int ui : pInput->tANSbits) + for (size_t i = 0; i < 32; i++) + { + unsigned int currVal = (ui >> i) & 0x1; + allBits[currIndex] = currVal; + currIndex++; + } + + if (!symEncoded) + for (size_t i = 0; i < symSize; i++) + symVec[i] = pInput->symVec[i]; + if (!loEncoded) + for (size_t i = 0; i < loSize; i++) + loVec[i] = pInput->loVec[i]; + + size_t bitIndex = 0; + int currState = pInput->initialState; + if (loEncoded) + { + bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, loSize*2, !symEncoded); + } + if (loEncoded) + for (size_t i = 0; i < loVec.size(); i++) + loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); + + if (symEncoded) + { + bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, symSize*4, true); + if (symDelta) + deltaDecode(&symNibbles, symNibbles.size()); + } + if (symEncoded) + for (size_t i = 0; i < symVec.size(); i++) + for (size_t j = 0; j < 4; j++) + symVec[i] += (unsigned short)symNibbles[i*4 + j] << (j*4); + + DataVecs returnData; + returnData.loVec = loVec; + returnData.symVec = symVec; + return returnData; +} + +void readNewHeader(std::vector *pInput, CompressedImage *pOutput) +{ + std::vector headers(2); + headers[0] = (*pInput)[0]; + headers[1] = (*pInput)[1]; + pOutput->mode = (CompressionMode)(headers[0] & MODE_MASK); + pOutput->rawNumBytes = ((headers[0] >> IMAGE_SIZE_OFFSET) & IMAGE_SIZE_MASK) * IMAGE_SIZE_MODIFIER; + pOutput->symSize = ((headers[0] >> SYM_SIZE_OFFSET) & SYM_SIZE_MASK); + + pOutput->initialState = headers[1] & INITIAL_STATE_MASK; + pOutput->bitreamSize = (headers[1] >> BITSTREAM_SIZE_OFFSET) & BITSTREAM_SIZE_MASK; + pOutput->loSize = (headers[1] >> LO_SIZE_OFFSET) & LO_SIZE_MASK; + pOutput->headers = headers; +} + +std::vector unpackFrequencies(unsigned int pInts[3]) +{ + std::vector returnVec; + int freq15 = 0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 5; j++) + { + int currVal = (pInts[i] >> (6*j)) & 0x3f; + returnVec.push_back(currVal); + } + freq15 += ((pInts[i] >> 30) & 0x3) << (2*i); + } + returnVec.push_back(freq15); + return returnVec; +} + +void getUIntVecFromData(CompressedImage *pImage, std::vector *pOutput) +{ + CompressedImage otherImage; + readNewHeader(&pImage->headers, &otherImage); + pOutput->push_back(pImage->headers[0]); + pOutput->push_back(pImage->headers[1]); + if (isModeLoEncoded(otherImage.mode)) + for (size_t i = 0; i < 3; i++) + pOutput->push_back(pImage->loFreqs[i]); + if (isModeSymEncoded(otherImage.mode)) + for (size_t i = 0; i < 3; i++) + pOutput->push_back(pImage->symFreqs[i]); + if (isModeLoEncoded(otherImage.mode) || isModeSymEncoded(otherImage.mode)) + for (unsigned int ui : pImage->tANSbits) + pOutput->push_back(ui); + unsigned int currInt = 0; + unsigned int currOffset = 0; + if (!isModeSymEncoded(otherImage.mode)) + { + for (size_t i = 0; i < pImage->symVec.size(); i++) + { + currInt += pImage->symVec[i] << (8*(currOffset % 4)); + currOffset += 2; + if (currOffset % 4 == 0) + { + pOutput->push_back(currInt); + currInt = 0; + } + } + } + if (!isModeLoEncoded(otherImage.mode)) + { + for (size_t i = 0; i < pImage->loVec.size(); i++) + { + currInt += 0; + currInt += pImage->loVec[i] << (8*(currOffset % 4)); + currOffset++; + if (currOffset % 4 == 0) + { + pOutput->push_back(currInt); + currInt = 0; + } + } + } + if (currOffset % 4 != 0) + pOutput->push_back(currInt); +} + +void readRawDataVecs(std::vector *pInput, std::vector *pOutput) +{ + CompressedImage readImage; + readNewHeader(pInput, &readImage); + bool loEncoded = isModeLoEncoded(readImage.mode); + bool symEncoded = isModeSymEncoded(readImage.mode); + bool symDelta = isModeSymDelta(readImage.mode); + size_t readIndex = 2; + std::vector tANSbits; + std::vector allBits; + std::vector symVec(readImage.symSize); + std::vector loVec(readImage.loSize); + std::vector symDecode(TANS_TABLE_SIZE); + std::vector loDecode(TANS_TABLE_SIZE); + int currState = readImage.initialState; + if (loEncoded) + { + for (size_t i = 0; i < 3; i++) + readImage.loFreqs[i] = (*pInput)[readIndex + i]; + readIndex += 3; + std::vector loFreqs = unpackFrequencies(&readImage.loFreqs[0]); + size_t currCol = 0; + for (size_t i = 0; i < 16; i++) + { + for (size_t j = 0; j < loFreqs[i]; j++) + { + loDecode[currCol].state = TANS_TABLE_SIZE + currCol; + loDecode[currCol].symbol = i; + loDecode[currCol].y = loFreqs[i] + j; + int currK = 0; + while ((loDecode[currCol].y << currK) < TANS_TABLE_SIZE) + currK++; + loDecode[currCol].k = currK; + currCol++; + } + } + } + if (symEncoded) + { + for (size_t i = 0; i < 3; i++) + readImage.symFreqs[i] = (*pInput)[readIndex + i]; + readIndex += 3; + std::vector symFreqs = unpackFrequencies(&readImage.symFreqs[0]); + size_t currCol = 0; + for (size_t i = 0; i < 16; i++) + { + for (size_t j = 0; j < symFreqs[i]; j++) + { + symDecode[currCol].state = TANS_TABLE_SIZE + currCol; + symDecode[currCol].symbol = i; + symDecode[currCol].y = symFreqs[i] + j; + int currK = 0; + while ((symDecode[currCol].y << currK) < TANS_TABLE_SIZE) + currK++; + symDecode[currCol].k = currK; + currCol++; + } + } + } + if (loEncoded || symEncoded) + { + tANSbits.resize(readImage.bitreamSize); + for (size_t i = 0; i < readImage.bitreamSize; i++) + tANSbits[i] = (*pInput)[readIndex + i]; + readIndex += readImage.bitreamSize; + allBits.resize(tANSbits.size()*32); + size_t currIndex = 0; + for (unsigned int ui : tANSbits) + { + for (size_t i = 0; i < 32; i++) + { + unsigned int currVal = (ui >> i) & 0x1; + allBits[currIndex] = currVal; + currIndex++; + } + } + } + bool leftOverValues = false; + if (!symEncoded) + { + for (size_t i = 0; i < readImage.symSize; i++) + { + symVec[i] = ((*pInput)[readIndex] >> (16*(i%2))) & 0xffff; + if ((i+1) % 2 == 0) + { + readIndex++; + leftOverValues = false; + } + else + { + leftOverValues = true; + } + } + } + if (!loEncoded) + { + size_t offsetMod = 0; + if (leftOverValues) + offsetMod = 2; + for (size_t i = 0; i < readImage.loSize; i++) + { + loVec[i] = ((*pInput)[readIndex] >> (8*((i%4) + offsetMod))) & 0xff; + if ((i+offsetMod+1) % 4 == 0) + readIndex++; + } + } + size_t bitIndex = 0; + if (loEncoded) + { + std::vector loNibbles(readImage.loSize*2); + bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, readImage.loSize*2, !symEncoded); + for (size_t i = 0; i < readImage.loSize; i++) + loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); + } + if (symEncoded) + { + std::vector symNibbles(readImage.symSize*4); + bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, readImage.symSize*4, true); + if (symDelta) + deltaDecode(&symNibbles, symNibbles.size()); + for (size_t i = 0; i < readImage.symSize; i++) + for (size_t j = 0; j < 4; j++) + symVec[i] += (unsigned short)symNibbles[i*4 + j] << (4*j); + } + *pOutput = decodeBytesShort(&loVec, &symVec); +} + +InputSettings::InputSettings() {} + +InputSettings::InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms) +{ + this->canEncodeLO = canEncodeLO; + this->canEncodeSyms = canEncodeSyms; + this->canDeltaSyms = canDeltaSyms; +} + +bool readFileAsUC(std::string filePath, std::vector *pFileData) +{ + std::ifstream iStream; + iStream.open(filePath.c_str(), std::ios::binary); + if (!iStream.is_open()) + { + fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); + return false; + } + + iStream.ignore( std::numeric_limits::max() ); + std::streamsize size = iStream.gcount(); + iStream.clear(); + iStream.seekg( 0, std::ios_base::beg ); + pFileData->resize(size); + iStream.read((char *)(pFileData->data()), size); + iStream.close(); + return true; +} + +bool readFileAsUInt(std::string filePath, std::vector *pFileData) +{ + std::ifstream iStream; + iStream.open(filePath.c_str(), std::ios::binary); + if (!iStream.is_open()) + { + fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); + return false; + } + + iStream.ignore( std::numeric_limits::max() ); + std::streamsize size = iStream.gcount(); + iStream.clear(); + iStream.seekg( 0, std::ios_base::beg ); + pFileData->resize(size/4); + iStream.read((char *)(pFileData->data()), size); + iStream.close(); + return true; +} + +CompressedImage processImage(std::string fileName, InputSettings settings) +{ + CompressedImage image; + std::vector input; + if (!readFileAsUC(fileName, &input)) + { + fprintf(stderr, "Compression failure\n"); + return image; + } + if (!processImageData(&input, &image, settings, fileName)) + { + fprintf(stderr, "Fail\n"); + } + return image; +} + +// Not implemented yet +CompressedImage processImageFrames(std::string fileName, InputSettings settings) +{ + CompressedImage image; + return image; +} + +bool processImageData(std::vector *pInput, CompressedImage *pImage, InputSettings settings, std::string fileName) +{ + CompressionMode someMode; + bool hasImage = false; + bool byteFail = false; + bool copyFail = false; + bool compressionFail = false; + bool uIntConversionFail = false; + + std::vector usBase(pInput->size() / 2); + memcpy(usBase.data(), pInput->data(), pInput->size()); + + std::vector bestLO; + std::vector bestSym; + std::vector bestInstructions; + + for (size_t minCodeLength = 2; minCodeLength <= 15; minCodeLength++) + { + std::vector shortCopies; + if (!getShortCopies(&usBase, minCodeLength, &shortCopies)) + { + copyFail = true; + printf("ERROR: %zu\n", minCodeLength); + continue; + } + + std::vector shortInstructions; + if (!getShortInstructions(&shortCopies, &shortInstructions, &usBase)) + { + printf("ERROR\n"); + return false; + } + + std::vector loVec; + std::vector symVec; + getLosFromInstructions(&shortInstructions, &loVec); + getSymsFromInstructions(&shortInstructions, &symVec); + if (!verifyBytesShort(&loVec, &symVec, &usBase)) + { + byteFail = true; + printf("Byte veri\n"); + continue; + } + + CompressionMode mode = BASE_ONLY; + std::vector modesToUse = {BASE_ONLY, ENCODE_SYMS, ENCODE_DELTA_SYMS, ENCODE_LO, ENCODE_BOTH, ENCODE_BOTH_DELTA_SYMS}; + if (fileName.find("test/compression/") != std::string::npos) + { + if (fileName.find("mode_0.4bpp") != std::string::npos) + modesToUse = {BASE_ONLY}; + else if (fileName.find("mode_1.4bpp") != std::string::npos) + modesToUse = {ENCODE_SYMS}; + else if (fileName.find("mode_2.4bpp") != std::string::npos) + modesToUse = {ENCODE_DELTA_SYMS}; + else if (fileName.find("mode_3.4bpp") != std::string::npos) + modesToUse = {ENCODE_LO}; + else if (fileName.find("mode_4.4bpp") != std::string::npos) + modesToUse = {ENCODE_BOTH}; + else if (fileName.find("mode_5.4bpp") != std::string::npos) + modesToUse = {ENCODE_BOTH_DELTA_SYMS}; + else if (fileName.find("test/compression/table_") != std::string::npos) + modesToUse = {ENCODE_SYMS}; + + if (modesToUse.size() == 1) + { + settings.canDeltaSyms = true; + settings.canEncodeLO = true; + settings.canEncodeSyms = true; + } + } + for (CompressionMode currMode : modesToUse) + { + CompressedImage currImg; + mode = currMode; + if (!settings.canDeltaSyms + && (mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!settings.canEncodeLO + && (mode == ENCODE_LO + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!settings.canEncodeSyms + && (mode == ENCODE_SYMS + || mode == ENCODE_BOTH + || mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!fillCompressVec(&loVec, &symVec, mode, pInput->size(), fileName, &currImg)) + { + printf("ERROR\n"); + } + + if (!verifyCompressionShort(&currImg, &usBase)) + { + compressionFail = true; + printf("ERROR\n"); + continue; + } + std::vector uiVec; + getUIntVecFromData(&currImg, &uiVec); + std::vector decodedImage; + readRawDataVecs(&uiVec, &decodedImage); + if (!compareVectorsShort(&decodedImage, &usBase)) + { + uIntConversionFail = true; + printf("ERROR\n"); + continue; + } + currImg.compressedSize = uiVec.size() * 4; + if (!hasImage) + { + bestLO = loVec; + bestSym = symVec; + bestInstructions = shortInstructions; + *pImage = currImg; + hasImage = true; + pImage->writeVec = uiVec; + someMode = mode; + } + else if (currImg.compressedSize < pImage->compressedSize) + { + bestLO = loVec; + bestSym = symVec; + bestInstructions = shortInstructions; + *pImage = currImg; + hasImage = true; + pImage->writeVec = uiVec; + someMode = mode; + } + } + } + pImage->mode = someMode; + pImage->fileName = fileName; + pImage->rawNumBytes = pInput->size(); + if (hasImage) + { + pImage->isValid = true; + } + else + { + fprintf(stderr, "Failed to compress image %s\nErrors: ", fileName.c_str()); + if (copyFail) + fprintf(stderr, "CopyProcessing "); + if (byteFail) + fprintf(stderr, "ByteConversion "); + if (compressionFail) + fprintf(stderr, "Compression "); + if (uIntConversionFail) + fprintf(stderr, "uIntConversion "); + printf("\n"); + } + + return true; +} diff --git a/tools/compresSmol/compressAlgo.h b/tools/compresSmol/compressAlgo.h index e27c599ce..6c2ff9373 100644 --- a/tools/compresSmol/compressAlgo.h +++ b/tools/compresSmol/compressAlgo.h @@ -54,20 +54,7 @@ struct ShortCopy { size_t length; size_t offset; unsigned short firstSymbol; - std::vector usSequence; - ShortCopy(); - ShortCopy(size_t index, size_t length, size_t offset, std::vector usSequence); -}; - -struct CompressionInstruction { - size_t length; - size_t offset; - size_t index; - unsigned char firstSymbol; - std::vector symbols; - std::vector bytes; - void buildBytes(); - bool verifyInstruction(); + ShortCopy(size_t index, size_t length, size_t offset, unsigned short firstSymbol); }; struct ShortCompressionInstruction { @@ -75,20 +62,9 @@ struct ShortCompressionInstruction { size_t offset; size_t index; unsigned short firstSymbol; - std::vector symbols; std::vector loBytes; std::vector symShorts; - void buildBytes(); - bool verifyInstruction(); -}; - -struct SortedShortElement { - size_t index; - ShortCopy copy; - bool isRun = false; - bool isCopy = false; - SortedShortElement(); - SortedShortElement(size_t index, ShortCopy copy); + void buildBytes(std::vector *pInput); }; struct CompressedImage { @@ -116,7 +92,6 @@ struct InputSettings { bool canEncodeLO = true; bool canEncodeSyms = true; bool canDeltaSyms = true; - bool shouldCompare = false; bool useFrames = false; InputSettings(); InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms); @@ -127,45 +102,38 @@ struct DataVecs { std::vector symVec; }; -void analyzeImages(std::vector *allImages, std::mutex *imageMutex, FileDispatcher *dispatcher, std::mutex *dispatchMutex, InputSettings settings); - CompressedImage processImage(std::string fileName, InputSettings settings); CompressedImage processImageFrames(std::string fileName, InputSettings settings); -CompressedImage processImageData(std::vector input, InputSettings settings, std::string fileName); +bool processImageData(std::vector *pInput, CompressedImage *pImage, InputSettings settings, std::string fileName); -std::vector readFileAsUInt(std::string filePath); +bool readFileAsUInt(std::string filePath, std::vector *pFileData); -size_t getCompressedSize(CompressedImage *pImage); - -std::vector getShortCopies(std::vector input, size_t minLength); +bool getShortCopies(std::vector *pInput, size_t minLength, std::vector *pShortCopies); bool verifyShortCopies(std::vector *pCopies, std::vector *pImage); std::vector getNormalizedCounts(std::vector input); std::vector getFreqWriteInts(std::vector input); std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength); -int findInitialState(EncodeCol encodeCol, unsigned char firstSymbol); -CompressedImage fillCompressVecNew(std::vector loVec, std::vector symVec, CompressionMode mode, size_t imageBytes, std::string name); -std::vector getShortInstructions(std::vector copies, size_t lengthMod); -std::vector getLosFromInstructions(std::vector instructions); -std::vector getSymsFromInstructions(std::vector instructions); +int findInitialState(EncodeCol *encodeCol, unsigned char firstSymbol); + +bool fillCompressVec(std::vector *pLoVec, std::vector *pSymVec, CompressionMode mode, size_t imageBytes, std::string name, CompressedImage *pOutput); + +bool getShortInstructions(std::vector *pCopies, std::vector *pInstructions, std::vector *pInput); +void getLosFromInstructions(std::vector *pInstructions, std::vector *pOutput); +void getSymsFromInstructions(std::vector *pInstructions, std::vector *pOutput); std::vector unpackFrequencies(unsigned int pInts[3]); -CompressedImage getDataFromUIntVec(std::vector *pInput); -CompressedImage readNewHeader(std::vector *pInput); -std::vector getUIntVecFromData(CompressedImage *pImage); +void readNewHeader(std::vector *pInput, CompressedImage *pOutput); +void getUIntVecFromData(CompressedImage *pImage, std::vector *pOutput); std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec); std::vector decodeImageShort(CompressedImage *pInput); DataVecs decodeDataVectorsNew(CompressedImage *pInput); - -size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles); - bool compareVectorsShort(std::vector *pVec1, std::vector *pVec2); bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage); -bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage); -bool verifyUIntVecShort(std::vector *pInput, std::vector *pImage); -std::vector readRawDataVecs(std::vector *pInput); +bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage); +void readRawDataVecs(std::vector *pInput, std::vector *pOutput); bool isModeLoEncoded(CompressionMode mode); bool isModeSymEncoded(CompressionMode mode); @@ -175,4 +143,5 @@ void deltaEncode(std::vector *buffer, int length); void deltaDecode(std::vector *buffer, int length); std::vector getTestFreqs(std::vector freqs, std::string name); + #endif diff --git a/tools/compresSmol/compressSmolTiles.cpp b/tools/compresSmol/compressSmolTiles.cpp index f8c14e7b3..b4a7ff0fc 100644 --- a/tools/compresSmol/compressSmolTiles.cpp +++ b/tools/compresSmol/compressSmolTiles.cpp @@ -91,15 +91,19 @@ std::vector decompressVector(std::vector *pVec) CompressVectors compressVector(std::vector *pVec) { CompressVectors vecs; - std::vector shortCopies = getShortCopies(*pVec, 2); + std::vector shortCopies; + getShortCopies(pVec, 2, &shortCopies); if (!verifyShortCopies(&shortCopies, pVec)) { fprintf(stderr, "Error getting tile-number compression\n"); return vecs; } - std::vector shortInstructions = getShortInstructions(shortCopies, 0); - std::vector loVec = getLosFromInstructions(shortInstructions); - std::vector symVec = getSymsFromInstructions(shortInstructions); + std::vector shortInstructions; + getShortInstructions(&shortCopies, &shortInstructions, pVec); + std::vector loVec; + getLosFromInstructions(&shortInstructions, &loVec); + std::vector symVec; + getSymsFromInstructions(&shortInstructions, &symVec); if (!verifyBytesShort(&loVec, &symVec, pVec)) { diff --git a/tools/compresSmol/mainTiles.cpp b/tools/compresSmol/mainTiles.cpp index 1fa083446..29626ceae 100644 --- a/tools/compresSmol/mainTiles.cpp +++ b/tools/compresSmol/mainTiles.cpp @@ -19,6 +19,7 @@ int main(int argc, char *argv[]) fileOut.close(); return 0; } + /* else if (argc == 2) { std::filesystem::path filePath = argv[1]; @@ -66,6 +67,7 @@ int main(int argc, char *argv[]) printf("New size: %zu\n", totalSize); return 0; } + */ else { return 0; diff --git a/tools/compresSmol/tANS.cpp b/tools/compresSmol/tANS.cpp index 40abd8a65..36d84caba 100644 --- a/tools/compresSmol/tANS.cpp +++ b/tools/compresSmol/tANS.cpp @@ -1,4 +1,5 @@ #include "tANS.h" +#include std::vector createDecodingTable(std::vector symbols, std::vector frequencies) { @@ -8,6 +9,7 @@ std::vector createDecodingTable(std::vector symbols, s { for (size_t j = 0; j < frequencies[i]; j++) { + //printf("%zu %zu\n", table.size(), currCol); table[currCol].state = TANS_TABLE_SIZE + currCol; table[currCol].symbol = symbols[i]; table[currCol].y = frequencies[i] + j; diff --git a/tools/gbagfx/gfx.c b/tools/gbagfx/gfx.c index 1dfc38e2d..c413e4b0f 100644 --- a/tools/gbagfx/gfx.c +++ b/tools/gbagfx/gfx.c @@ -14,6 +14,7 @@ #define GET_GBA_PAL_BLUE(x) (((x) >> 10) & 0x1F) #define SET_GBA_PAL(r, g, b) (((b) << 10) | ((g) << 5) | (r)) +#define SET_GBA_PAL_RGBA(r, g, b, a) (((a) << 15) | ((b) << 10) | ((g) << 5) | (r)) #define UPCONVERT_BIT_DEPTH(x) (((x) * 255) / 31) @@ -578,8 +579,9 @@ void WriteGbaPalette(char *path, struct Palette *palette) unsigned char red = DOWNCONVERT_BIT_DEPTH(palette->colors[i].red); unsigned char green = DOWNCONVERT_BIT_DEPTH(palette->colors[i].green); unsigned char blue = DOWNCONVERT_BIT_DEPTH(palette->colors[i].blue); + bool alpha = palette->colors[i].alpha; - uint16_t paletteEntry = SET_GBA_PAL(red, green, blue); + uint16_t paletteEntry = SET_GBA_PAL_RGBA(red, green, blue, alpha); fputc(paletteEntry & 0xFF, fp); fputc(paletteEntry >> 8, fp); diff --git a/tools/gbagfx/gfx.h b/tools/gbagfx/gfx.h index 1797d84df..30bbfeb8d 100644 --- a/tools/gbagfx/gfx.h +++ b/tools/gbagfx/gfx.h @@ -10,6 +10,7 @@ struct Color { unsigned char red; unsigned char green; unsigned char blue; + bool alpha; }; struct Palette { diff --git a/tools/gbagfx/jasc_pal.c b/tools/gbagfx/jasc_pal.c index 8d4bb137d..3ae7f8b90 100644 --- a/tools/gbagfx/jasc_pal.c +++ b/tools/gbagfx/jasc_pal.c @@ -22,7 +22,7 @@ // Blue - "0 0 255\r\n" // Brown - "150 75 0\r\n" -#define MAX_LINE_LENGTH 11 +#define MAX_LINE_LENGTH 64 void ReadJascPaletteLine(FILE *fp, char *line) { @@ -156,6 +156,42 @@ void ReadJascPalette(char *path, struct Palette *palette) FATAL_ERROR("Garbage after color data.\n"); fclose(fp); + + // Try to parse alpha/high bit info for colors from auxiliary .pla file + char *dot = strrchr(path, '.'); + if (strcmp(dot, ".pal") != 0) + return; + strcpy(dot, ".pla"); // replace .pal with .pla + fp = fopen(path, "rb"); + + if (fp == NULL) + return; + + int i = 0; + // Keep reading lines until number of colors is reached or we run out + while (i < palette->numColors && fgets(line, MAX_LINE_LENGTH, fp) != NULL) + { + if (line[0] == '#') // comment line; ignore + continue; + + char *s = line; + char *end; + int colorIndex; + + if (!ParseNumber(s, &end, 10, &colorIndex)) + FATAL_ERROR("Failed to parse aux color index.\n"); + + s = end; + + if (colorIndex >= palette->numColors) + FATAL_ERROR("Aux color index %d out of bounds.\n", colorIndex); + + palette->colors[colorIndex].alpha = 1; // set alpha color + // fprintf(stderr, "Color index: %d\n", colorIndex); + i++; + } + + fclose(fp); } void WriteJascPalette(char *path, struct Palette *palette) diff --git a/tools/gbagfx/main.c b/tools/gbagfx/main.c index 01b46d076..6502b87d6 100644 --- a/tools/gbagfx/main.c +++ b/tools/gbagfx/main.c @@ -291,7 +291,7 @@ void HandlePngToGbaCommand(char *inputPath, char *outputPath, int argc, char **a void HandlePngToJascPaletteCommand(char *inputPath, char *outputPath, int argc UNUSED, char **argv UNUSED) { - struct Palette palette = {}; + struct Palette palette = {0}; ReadPngPalette(inputPath, &palette); WriteJascPalette(outputPath, &palette); @@ -299,7 +299,7 @@ void HandlePngToJascPaletteCommand(char *inputPath, char *outputPath, int argc U void HandlePngToGbaPaletteCommand(char *inputPath, char *outputPath, int argc UNUSED, char **argv UNUSED) { - struct Palette palette = {}; + struct Palette palette = {0}; ReadPngPalette(inputPath, &palette); WriteGbaPalette(outputPath, &palette); @@ -307,7 +307,7 @@ void HandlePngToGbaPaletteCommand(char *inputPath, char *outputPath, int argc UN void HandleGbaToJascPaletteCommand(char *inputPath, char *outputPath, int argc UNUSED, char **argv UNUSED) { - struct Palette palette = {}; + struct Palette palette = {0}; ReadGbaPalette(inputPath, &palette); WriteJascPalette(outputPath, &palette); @@ -333,6 +333,9 @@ void HandleJascToGbaPaletteCommand(char *inputPath, char *outputPath, int argc, if (numColors < 1) FATAL_ERROR("Number of colors must be positive.\n"); + + if (numColors > 255) + FATAL_ERROR("Number of colors must be less than 256.\n"); } else { @@ -340,7 +343,7 @@ void HandleJascToGbaPaletteCommand(char *inputPath, char *outputPath, int argc, } } - struct Palette palette = {}; + struct Palette palette = {0}; ReadJascPalette(inputPath, &palette); @@ -425,7 +428,7 @@ void HandlePngToFullwidthJapaneseFontCommand(char *inputPath, char *outputPath, void HandleLZCompressCommand(char *inputPath, char *outputPath, int argc, char **argv) { int overflowSize = 0; - int minDistance = 2; // default, for compatibility with DecompressDataWithHeaderVram() + int minDistance = 2; // default, for compatibility with LZ77UnCompVram() for (int i = 3; i < argc; i++) { diff --git a/tools/mapjson/Makefile b/tools/mapjson/Makefile index 100e809c6..411c96c2d 100644 --- a/tools/mapjson/Makefile +++ b/tools/mapjson/Makefile @@ -1,6 +1,6 @@ CXX ?= g++ -CXXFLAGS := -Wall -std=c++11 -O2 +CXXFLAGS := -Wall -std=c++17 -O2 SRCS := json11.cpp mapjson.cpp diff --git a/tools/mgba-rom-test-hydra/main.c b/tools/mgba-rom-test-hydra/main.c index 8eb66debc..440b4ad5e 100644 --- a/tools/mgba-rom-test-hydra/main.c +++ b/tools/mgba-rom-test-hydra/main.c @@ -11,9 +11,9 @@ * L: Sets the filename to the remainder of the line. * R: Sets the result to the remainder of the line, and flushes any * output buffered since the previous R. - * P/K/F/A: Sets the result to the remaining of the line, flushes any - * output since the previous P/K/F/A and increment the number of - * passes/known fails/assumption fails/fails. + * P/E/K/F/A: Sets the result to the remaining of the line, flushes any + * output since the previous P/E/K/F/A and increment the number of + * passes/expected fails/known fails/assumption fails/fails. */ #include #include @@ -59,6 +59,8 @@ struct Runner int passes; int knownFails; int knownFailsPassing; + int expectedFails; + int expectedFailsPassing; int todos; int assumptionFails; int fails; @@ -67,6 +69,8 @@ struct Runner char failed_TestFilenameLine[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; char knownFailingPassed_TestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; char knownFailingPassed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; + char expectedFailingPassed_TestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; + char expectedFailingPassed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; char assumeFailed_TestNames[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; char assumeFailed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST][MAX_TEST_LIST_BUFFER_LENGTH]; }; @@ -241,6 +245,9 @@ static void handle_read(int i, struct Runner *runner) case 'P': runner->passes++; goto add_to_results; + case 'E': + runner->expectedFails++; + goto add_to_results; case 'K': runner->knownFails++; goto add_to_results; @@ -252,6 +259,14 @@ static void handle_read(int i, struct Runner *runner) } runner->knownFailsPassing++; goto add_to_results; + case 'V': + if (runner->expectedFailsPassing < MAX_SUMMARY_TESTS_TO_LIST) + { + strcpy(runner->expectedFailingPassed_TestNames[runner->expectedFailsPassing], runner->test_name); + strcpy(runner->expectedFailingPassed_FilenameLine[runner->expectedFailsPassing], runner->filename_line); + } + runner->expectedFailsPassing++; + goto add_to_results; case 'T': runner->todos++; goto add_to_results; @@ -737,6 +752,8 @@ int main(int argc, char *argv[]) // Reap test runners and collate exit codes. int exit_code = 0; int passes = 0; + int expectedFails = 0; + int expectedFailsPassing = 0; int knownFails = 0; int knownFailsPassing = 0; int todos = 0; @@ -750,6 +767,9 @@ int main(int argc, char *argv[]) char knownFailingPassed_TestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; char knownFailingPassed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; + char expectedFailingPassed_TestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; + char expectedFailingPassed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; + char assumeFailed_TestNames[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; char assumeFailed_FilenameLine[MAX_SUMMARY_TESTS_TO_LIST * MAX_PROCESSES][MAX_TEST_LIST_BUFFER_LENGTH]; @@ -766,6 +786,7 @@ int main(int argc, char *argv[]) if (WIFEXITED(wstatus) && WEXITSTATUS(wstatus) > exit_code) exit_code = WEXITSTATUS(wstatus); passes += runners[i].passes; + expectedFails += runners[i].expectedFails; knownFails += runners[i].knownFails; for (int j = 0; j < runners[i].knownFailsPassing; j++) { @@ -776,6 +797,15 @@ int main(int argc, char *argv[]) } knownFailsPassing++; } + for (int j = 0; j < runners[i].expectedFailsPassing; j++) + { + if (j < MAX_SUMMARY_TESTS_TO_LIST) + { + strcpy(expectedFailingPassed_TestNames[expectedFailsPassing], runners[i].expectedFailingPassed_TestNames[j]); + strcpy(expectedFailingPassed_FilenameLine[expectedFailsPassing], runners[i].expectedFailingPassed_FilenameLine[j]); + } + expectedFailsPassing++; + } todos += runners[i].todos; for (int j = 0; j < runners[i].assumptionFails; j++) { @@ -855,6 +885,8 @@ int main(int argc, char *argv[]) fprintf(stdout, "\n"); if (fails > 0) fprintf(stdout, "- Tests \e[31mFAILED\e[0m : %d Add TESTS='X' to run tests with the defined prefix.\n", fails); + if (expectedFailsPassing > 0) + fprintf(stdout, "- \e[31mEXPECTED_FAIL_PASSING\e[0m: %d\n", expectedFailsPassing); if (knownFails > 0) fprintf(stdout, "- Tests \e[33mKNOWN_FAILING\e[0m: %d\n", knownFails); if (assumptionFails > 0) @@ -863,7 +895,10 @@ int main(int argc, char *argv[]) fprintf(stdout, "- Tests \e[33mTO_DO\e[0m: %d\n", todos); if (knownFailsPassing > 0) fprintf(stdout, "- \e[32mKNOWN_FAILING_PASSING\e[0m: %d \e[33mPlease remove KNOWN_FAILING if these tests intentionally PASS\e[0m\n", knownFailsPassing); - fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); + if (expectedFails > 0) + fprintf(stdout, "- Tests \e[32mEXPECT_FAILING\e[0m: %d\n", expectedFails); + if (passes > 0) + fprintf(stdout, "- Tests \e[32mPASSED\e[0m: %d\n", passes); fprintf(stdout, "- Tests \e[34mTOTAL\e[0m: %d\n", results); } fprintf(stdout, "\n"); diff --git a/tools/mgba/README.md b/tools/mgba/README.md index 617e6e058..246d681d0 100644 --- a/tools/mgba/README.md +++ b/tools/mgba/README.md @@ -5,3 +5,4 @@ The source code for these specific builds is available from: - Windows: - Linux: + - Mac: diff --git a/tools/mgba/mgba-rom-test-mac b/tools/mgba/mgba-rom-test-mac new file mode 100755 index 000000000..43c8976c0 Binary files /dev/null and b/tools/mgba/mgba-rom-test-mac differ