mirror of
https://github.com/Alcaro/Flips.git
synced 2026-09-07 18:16:06 -05:00
Move these to subfolder
This commit is contained in:
Alcaro
878
patch/libbps-suf.cpp
Normal file
878
patch/libbps-suf.cpp
Normal file
@@ -0,0 +1,878 @@
|
||||
#include "libbps.h"
|
||||
#include "arlib/crc32.h"
|
||||
#include "arlib/file.h"
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
//These two give minor performance penalties and will print some random stuff to stdout.
|
||||
//The former will verify the correctness of the output patch, the latter will print some performance data.
|
||||
//Can be useful for debugging, but should be disabled for release builds.
|
||||
#ifdef BPS_STANDALONE
|
||||
#endif
|
||||
//#define TEST_CORRECT
|
||||
//#define TEST_PERF
|
||||
|
||||
//If the suffix array of [0, 0, 0, 0] is [3, 2, 1, 0], set to true. If it's [0, 1, 2, 3], this is false.
|
||||
//If it's [4, 3, 2, 1, 0] or [0, 1, 2, 3, 4], remove the 4 (easily done with some pointer math), and follow the above.
|
||||
//If it's something else, get a non-broken array calculator.
|
||||
#define EOF_IS_LAST false
|
||||
|
||||
#if defined(TEST_CORRECT) || defined(TEST_PERF)
|
||||
#include <stdio.h>
|
||||
#endif
|
||||
|
||||
//Algorithm description:
|
||||
//
|
||||
//This is heavily built upon suffix sorting; the implementation I use, libdivsufsort, claims
|
||||
// O(n log n) complexity, so I'll believe that. There is also SA-IS, which claims O(n), but if that
|
||||
// is true, its constant factors are ridiculously high.
|
||||
//
|
||||
//The program starts by taking an equal amount of the source file and target file, concatenates that
|
||||
// with target first, and suffix sorts it.
|
||||
//It also calculates a reverse index, such that reverse[sorted[i]]==i.
|
||||
//
|
||||
//To find a match, it goes to reverse[outpos], and scans sorted[] up and down for the closest entry
|
||||
// that either starts before the current output position, or is somewhere in the source file.
|
||||
//As the source file comes last, the end-of-file marker (whose value is outside the range of a byte)
|
||||
// is guaranteed to not be in the way for a better match.
|
||||
//This is called O(n) times, and averages O(1) as at least 50% of sorted[] is in range. However, it
|
||||
// is worst-case O(n) for sorted inputs, giving a total of O(n^2).
|
||||
//
|
||||
//It then checks which of the two candidates are superior, by checking how far they match each
|
||||
// other, and then checking if the upper one has another correct byte.
|
||||
//This is potentially O(n), but for each matched byte, another iteration is removed from the outer
|
||||
// loop, so the sum of all calls is O(n).
|
||||
//
|
||||
//When the program approaches the end of the sorted area, it re-sorts twice as much as last time.
|
||||
// This gives O(log n) calls to the suffix sorter.
|
||||
//Given O(n log n) for one sorting step, the time taken is O(n/1 log n/1 + n/2 log n/2 +
|
||||
// n/4 log n/4 + ...), which is strictly less than O(n/1 log n + n/2 log n + n/4 log n + ...), which
|
||||
// equals O(2n log n), which is O(n log n). (The exact value of that infinite sum is 2n*log(n/2).)
|
||||
//
|
||||
//Many details were omitted from the above, but that's the basic setup.
|
||||
//
|
||||
//Thus, the program is O(max(n log n, n, n) = n log n) average and O(max(n log n, n^2, n) = n^2)
|
||||
// worst case.
|
||||
//
|
||||
//I conclude that the task of finding, understanding and implementing a sub-O(n^2) algorithm for
|
||||
// delta patching is resolved.
|
||||
|
||||
|
||||
//Known cases where this function does not emit the optimal encoding:
|
||||
//If a match in the target file would extend further than target_search_size, it is often skipped.
|
||||
// Penalty: O(log n), with extremely low constants (it'd require a >256B match to be exactly there).
|
||||
// Even for big files, the penalty is very likely to remain zero; even hitting double-digit bytes
|
||||
// would require a file designed exactly for that.
|
||||
//If multiple matches are equally good, it picks one at random, not the one that's cheaper to encode.
|
||||
// Penalty: Likely O(n) or O(n log log n), with low constants. I'd guess ~1.4% for my 48MB test file.
|
||||
//However, due to better heuristics and others' performance optimizations, this one still beats its
|
||||
// competitors.
|
||||
|
||||
|
||||
//Possible optimizations:
|
||||
//divsufsort() takes approximately 2/3 of the total time. create_reverse_index() takes roughly a third of the remainder.
|
||||
//Each iteration takes four times as long as the previous one.
|
||||
//If each iteration takes 4 times as long as the previous one, then the last one takes 3/4 of the total time.
|
||||
//Since divsufsort+create_reverse_index doesn't depend on anything else, the last iteration can be split off to its own thread.
|
||||
//This would split it to
|
||||
//Search, non-final: 2/9 * 1/4 = 2/36
|
||||
//Search, final: 2/9 * 3/4 = 6/36
|
||||
//Sort+rev, non-final: 7/9 * 1/4 = 7/36
|
||||
//Sort+rev, final: 7/9 * 3/4 = 21/36
|
||||
//All non-final must be done sequentially. Both Sort Final and non-final must be done before Search Final can start.
|
||||
//This means the final time, if Sort Final is split off, is
|
||||
//max(7/36+2/36, 21/36) + 6/36 = 27/36 = 3/4
|
||||
//of the original time.
|
||||
//Due to
|
||||
//- the considerable complexity costs (OpenMP doesn't seem able to represent the "insert a wait in
|
||||
// the middle of this while loop" I would need)
|
||||
//- the added memory use, approximately 25% higher - it's already high enough
|
||||
//- libdivsufsort already using threads, which would make the gains lower
|
||||
// and would increase complexity, as I have to ensure the big one remains threaded -
|
||||
// and that the small ones are not, as that'd starve the big one
|
||||
//I deem a possible 25% boost not worthwhile.
|
||||
|
||||
|
||||
//Both sorting algorithms claim O(1) memory use (in addition to the bytes and the output). In
|
||||
// addition to that, this algorithm uses (source.len*target.len)*(sizeof(uint8_t)+2*sizeof(off_t))
|
||||
// bytes of memory, plus the input and output files, plus the patch.
|
||||
//For most hardware, this is 9*(source.len+target.len), or 5*(source+target) for the slim one.
|
||||
|
||||
|
||||
#include "sais.cpp"
|
||||
template<typename sais_index_type>
|
||||
static void sufsort(sais_index_type* SA, const uint8_t* T, sais_index_type n) {
|
||||
if(n <= 1) { if(n == 1) SA[0] = 0; return; }
|
||||
sais_main<sais_index_type>(T, SA, 0, n, 256);
|
||||
}
|
||||
|
||||
//According to <https://code.google.com/p/libdivsufsort/wiki/SACA_Benchmarks>, divsufsort achieves
|
||||
// approximately half the time of SAIS for nearly all files, despite SAIS' promises of linear
|
||||
// performance (divsufsort claims O(n log n)).
|
||||
|
||||
//divsufsort only allocates O(1) for some radix/bucket sorting. SAIS seems constant too.
|
||||
//I'd prefer to let them allocate from an array I give it, but divsuf doesn't allow that, and there
|
||||
// are only half a dozen allocations per call anyways.
|
||||
|
||||
//This ends up in libdivsufsort if available, otherwise lite.
|
||||
#include "divsufsort.h"
|
||||
static void sufsort(int32_t* SA, uint8_t* T, int32_t n)
|
||||
{
|
||||
divsufsort(T, SA, n);
|
||||
}
|
||||
|
||||
#ifdef USE_DIVSUFSORT64
|
||||
#include "divsufsort64.h"
|
||||
|
||||
static void sufsort(int64_t* SA, uint8_t* T, int64_t n)
|
||||
{
|
||||
divsufsort(T, SA, n);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
template<typename T> static T min(T a, T b) { return a<b ? a : b; }
|
||||
template<typename T> static T max(T a, T b) { return a<b ? b : a; }
|
||||
|
||||
|
||||
|
||||
namespace {
|
||||
struct bps_creator {
|
||||
uint8_t* out;
|
||||
size_t outlen;
|
||||
size_t outbuflen;
|
||||
|
||||
void reserve(size_t len)
|
||||
{
|
||||
if (outlen+len > outbuflen)
|
||||
{
|
||||
if (!outbuflen) outbuflen = 128;
|
||||
while (outlen+len > outbuflen) outbuflen *= 2;
|
||||
out = (uint8_t*)realloc(out, outbuflen);
|
||||
}
|
||||
}
|
||||
|
||||
void append(const uint8_t * data, size_t len)
|
||||
{
|
||||
reserve(len);
|
||||
memcpy(out+outlen, data, len);
|
||||
outlen+=len;
|
||||
}
|
||||
|
||||
void appendnum(size_t num)
|
||||
{
|
||||
#ifdef TEST_CORRECT
|
||||
if (num > 1000000000)
|
||||
printf("ERROR: Attempt to write %.8lX\n",(unsigned long)num),abort();
|
||||
#endif
|
||||
reserve(sizeof(size_t)*8/7+1);
|
||||
|
||||
while (num >= 128)
|
||||
{
|
||||
out[outlen++]=(num&0x7F);
|
||||
num>>=7;
|
||||
num--;
|
||||
}
|
||||
out[outlen++]=num|0x80;
|
||||
}
|
||||
|
||||
void appendnum32(uint32_t num)
|
||||
{
|
||||
reserve(4);
|
||||
out[outlen++] = num>>0;
|
||||
out[outlen++] = num>>8;
|
||||
out[outlen++] = num>>16;
|
||||
out[outlen++] = num>>24;
|
||||
}
|
||||
|
||||
static size_t maxsize()
|
||||
{
|
||||
return SIZE_MAX>>2; // can be reduced to SIZE_MAX>>1 by amending append_cmd, but the mallocs overflow at that point anyways.
|
||||
}
|
||||
|
||||
size_t sourcelen;
|
||||
size_t targetlen;
|
||||
const uint8_t* targetmem;
|
||||
|
||||
enum bpscmd { SourceRead, TargetRead, SourceCopy, TargetCopy };
|
||||
|
||||
size_t outpos;
|
||||
|
||||
size_t sourcecopypos;
|
||||
size_t targetcopypos;
|
||||
|
||||
size_t numtargetread;
|
||||
|
||||
bps_creator(file* source, file* target, struct mem metadata)
|
||||
{
|
||||
outlen = 0;
|
||||
outbuflen = 128;
|
||||
out = (uint8_t*)malloc(outbuflen);
|
||||
|
||||
outpos = 0;
|
||||
|
||||
sourcecopypos = 0;
|
||||
targetcopypos = 0;
|
||||
|
||||
numtargetread = 0;
|
||||
|
||||
append((const uint8_t*)"BPS1", 4);
|
||||
appendnum(source->len);
|
||||
appendnum(target->len);
|
||||
appendnum(metadata.len);
|
||||
append(metadata.ptr, metadata.len);
|
||||
|
||||
setProgress(NULL, NULL);
|
||||
}
|
||||
|
||||
|
||||
void move_target(const uint8_t* ptr)
|
||||
{
|
||||
targetmem = ptr;
|
||||
}
|
||||
|
||||
size_t encode_delta(size_t prev, size_t next)
|
||||
{
|
||||
bool negative = (next<prev);
|
||||
size_t offset = negative ? prev-next : next-prev;
|
||||
return (negative?1:0) | (offset<<1);
|
||||
}
|
||||
|
||||
void append_delta(size_t prev, size_t next)
|
||||
{
|
||||
appendnum(encode_delta(prev, next));
|
||||
}
|
||||
|
||||
void append_cmd(bpscmd command, size_t count)
|
||||
{
|
||||
appendnum((count-1)<<2 | command);
|
||||
}
|
||||
|
||||
void flush_target_read()
|
||||
{
|
||||
if (!numtargetread) return;
|
||||
append_cmd(TargetRead, numtargetread);
|
||||
append(targetmem+outpos-numtargetread, numtargetread);
|
||||
numtargetread = 0;
|
||||
}
|
||||
|
||||
size_t emit_source_copy(size_t location, size_t count)
|
||||
{
|
||||
if (location == outpos) return emit_source_read(location, count);
|
||||
flush_target_read();
|
||||
append_cmd(SourceCopy, count);
|
||||
append_delta(sourcecopypos, location);
|
||||
sourcecopypos = location+count;
|
||||
outpos += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
size_t emit_source_read(size_t location, size_t count)
|
||||
{
|
||||
flush_target_read();
|
||||
#ifdef TEST_CORRECT
|
||||
if (location != outpos)
|
||||
puts("ERROR: SourceRead not from source pointer"),abort();
|
||||
#endif
|
||||
append_cmd(SourceRead, count);
|
||||
outpos+=count;
|
||||
return count;
|
||||
}
|
||||
|
||||
size_t emit_target_copy(size_t location, size_t count)
|
||||
{
|
||||
flush_target_read();
|
||||
append_cmd(TargetCopy, count);
|
||||
append_delta(targetcopypos, location);
|
||||
targetcopypos = location+count;
|
||||
outpos += count;
|
||||
return count;
|
||||
}
|
||||
|
||||
size_t emit_target_read()
|
||||
{
|
||||
numtargetread++;
|
||||
outpos++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
size_t abs_diff(size_t a, size_t b)
|
||||
{
|
||||
return (b<a) ? (a-b) : (b-a);
|
||||
}
|
||||
size_t num_cost(size_t num)
|
||||
{
|
||||
if (num<128) return 1;
|
||||
if (num<128*128) return 2; // 32KB
|
||||
if (num<128*128*128) return 3; // 2MB
|
||||
if (num<128*128*128*128) return 4; // 256MB
|
||||
// 128^5 is 32GB, let's just assume the sizes don't go any higher...
|
||||
// worst case, a bad match is used. except a 32GB match is by definition good.
|
||||
return 5;
|
||||
}
|
||||
|
||||
bool use_match(bool hastargetread, size_t cost, size_t len)
|
||||
{
|
||||
//numbers calculated via trial and error; checking for each cost, optimizing 'len' for each, and checking what happens
|
||||
//then a pattern was identified and used
|
||||
//yes, it looks weird
|
||||
return len >= 1+cost+hastargetread+(len==1);
|
||||
}
|
||||
|
||||
|
||||
//Return value is how many bytes were used. If you believe the given one sucks, use TargetRead and return 1.
|
||||
size_t match(bool is_target, size_t pos, size_t len)
|
||||
{
|
||||
if (!use_match(
|
||||
numtargetread,
|
||||
(!is_target && pos==outpos) ? 1 : // SourceRead
|
||||
(num_cost(abs_diff(pos, (is_target ? targetcopypos : sourcecopypos)))+1),
|
||||
len
|
||||
))
|
||||
{
|
||||
return emit_target_read();
|
||||
}
|
||||
|
||||
if (is_target) return emit_target_copy(pos, len);
|
||||
else return emit_source_copy(pos, len);
|
||||
}
|
||||
|
||||
|
||||
bool (*prog_func)(void* userdata, size_t done, size_t total);
|
||||
void* prog_dat;
|
||||
|
||||
static bool prog_func_null(void* userdata, size_t done, size_t total) { return true; }
|
||||
|
||||
void setProgress(bool (*progress)(void* userdata, size_t done, size_t total), void* userdata)
|
||||
{
|
||||
if (!progress) progress = prog_func_null;
|
||||
|
||||
prog_func=progress;
|
||||
prog_dat=userdata;
|
||||
}
|
||||
|
||||
bool progress(size_t done, size_t total)
|
||||
{
|
||||
return prog_func(prog_dat, done, total);
|
||||
}
|
||||
|
||||
|
||||
void finish(const uint8_t* source, const uint8_t* target)
|
||||
{
|
||||
flush_target_read();
|
||||
#ifdef TEST_CORRECT
|
||||
if (outpos != targetlen)
|
||||
puts("ERROR: patch creates wrong ROM size"),abort();
|
||||
#endif
|
||||
|
||||
appendnum32(crc32(source, sourcelen));
|
||||
appendnum32(crc32(target, targetlen));
|
||||
appendnum32(crc32(out, outlen));
|
||||
}
|
||||
|
||||
struct mem getpatch()
|
||||
{
|
||||
struct mem ret = { out, outlen };
|
||||
out = NULL;
|
||||
return ret;
|
||||
}
|
||||
|
||||
~bps_creator() { free(out); }
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
#ifdef TEST_PERF
|
||||
static int match_len_n=0;
|
||||
static int match_len_tot=0;
|
||||
#endif
|
||||
|
||||
template<typename off_t>
|
||||
static off_t match_len(const uint8_t* a, const uint8_t* b, off_t len)
|
||||
{
|
||||
off_t i;
|
||||
for (i=0;i<len && a[i]==b[i];i++) {}
|
||||
#ifdef TEST_PERF
|
||||
match_len_n++;
|
||||
match_len_tot+=i;
|
||||
#endif
|
||||
return i;
|
||||
}
|
||||
|
||||
//This one assumes that the longest common prefix of 'a' and 'b' is shared also by 'search'.
|
||||
//In practice, lexographically, a < search < b, which is a stronger guarantee.
|
||||
template<typename off_t>
|
||||
static off_t pick_best_of_two(const uint8_t* search, off_t searchlen,
|
||||
const uint8_t* data, off_t datalen,
|
||||
off_t a, off_t b,
|
||||
off_t* bestlen)
|
||||
{
|
||||
off_t commonlen = match_len(data+a, data+b, min(datalen-a, datalen-b));
|
||||
if (commonlen>=searchlen)
|
||||
{
|
||||
*bestlen=searchlen;
|
||||
return a;
|
||||
}
|
||||
|
||||
if (a+commonlen<datalen && search[commonlen]==data[a+commonlen])
|
||||
{
|
||||
// a is better
|
||||
*bestlen = commonlen + match_len(search+commonlen, data+a+commonlen, min(searchlen, datalen-a)-commonlen);
|
||||
return a;
|
||||
}
|
||||
else
|
||||
{
|
||||
// b is better, or they're equal
|
||||
*bestlen = commonlen + match_len(search+commonlen, data+b+commonlen, min(searchlen, datalen-b)-commonlen);
|
||||
return b;
|
||||
}
|
||||
}
|
||||
|
||||
//This one takes a match, which is assumed optimal, and looks for the lexographically closest one
|
||||
// that either starts before 'maxstart', or starts at or after 'minstart'.
|
||||
template<typename off_t>
|
||||
static off_t adjust_match(off_t match, const uint8_t* search, off_t searchlen,
|
||||
const uint8_t* data,off_t datalen, off_t maxstart,off_t minstart,
|
||||
const off_t* sorted, off_t sortedlen,
|
||||
off_t* bestlen)
|
||||
{
|
||||
off_t match_up = match;
|
||||
off_t match_dn = match;
|
||||
while (match_up>=0 && sorted[match_up]>=maxstart && sorted[match_up]<minstart) match_up--;
|
||||
while (match_dn<sortedlen && sorted[match_dn]>=maxstart && sorted[match_dn]<minstart) match_dn++;
|
||||
if (match_up<0 || match_dn>=sortedlen)
|
||||
{
|
||||
if (match_up<0 && match_dn>=sortedlen)
|
||||
{
|
||||
*bestlen=0;
|
||||
return 0;
|
||||
}
|
||||
off_t pos = sorted[match_up<0 ? match_dn : match_up];
|
||||
*bestlen = match_len(search, data+pos, min(searchlen, datalen-pos));
|
||||
return pos;
|
||||
}
|
||||
|
||||
return pick_best_of_two(search,searchlen, data,datalen, sorted[match_up],sorted[match_dn], bestlen);
|
||||
}
|
||||
|
||||
|
||||
|
||||
static uint16_t read2_uc(const uint8_t* data)
|
||||
{
|
||||
return data[0]<<8 | data[1];
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
static uint16_t read2(const uint8_t* data, off_t len)
|
||||
{
|
||||
if (len>=2) return read2_uc(data);
|
||||
else
|
||||
{
|
||||
uint16_t out = (EOF_IS_LAST ? 0xFFFF : 0x0000);
|
||||
if (len==1) out = (data[0]<<8) | (out&0x00FF);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
static void create_buckets(const uint8_t* data, off_t* index, off_t len, off_t* buckets)
|
||||
{
|
||||
off_t low = 0;
|
||||
off_t high;
|
||||
|
||||
for (int n=0;n<65536;n++)
|
||||
{
|
||||
//'low' remains from the previous iteration and is a known minimum
|
||||
high = low+(len/131072)+1; // optimal value: slightly above a third of the distance to the next one
|
||||
while (true)
|
||||
{
|
||||
if (high > len-1) break;
|
||||
|
||||
off_t pos = index[high];
|
||||
uint16_t here = read2(data+pos, len-pos);
|
||||
|
||||
if (here >= n) break;
|
||||
else
|
||||
{
|
||||
off_t diff = high-low;
|
||||
low = high;
|
||||
high = high+diff*2;
|
||||
}
|
||||
}
|
||||
if (high > len-1) high = len-1;
|
||||
|
||||
|
||||
while (low < high)
|
||||
{
|
||||
off_t mid = low + (high-low)/2;
|
||||
off_t midpos = index[mid];
|
||||
|
||||
uint16_t here = read2(data+midpos, len-midpos);
|
||||
if (here < n) low = mid+1;
|
||||
else high = mid;
|
||||
}
|
||||
buckets[n] = low;
|
||||
}
|
||||
|
||||
buckets[65536] = len;
|
||||
|
||||
#ifdef TEST_CORRECT
|
||||
if (buckets[0]!=0)
|
||||
{
|
||||
printf("e: buckets suck, [0]=%i\n", buckets[0]);
|
||||
abort();
|
||||
}
|
||||
for (int n=0;n<65536;n++)
|
||||
{
|
||||
off_t low = buckets[n];
|
||||
off_t high = buckets[n+1];
|
||||
for (off_t i=low;i<high;i++)
|
||||
{
|
||||
if (read2(data+index[i], len-index[i])!=n)
|
||||
{
|
||||
printf("e: buckets suck, %i != (%i)[%i]%i [%i-%i]", n, i,index[i],read2(data+index[i],len-index[i]),low,high);
|
||||
abort();
|
||||
}
|
||||
}
|
||||
//printf("%i:[%i]%i\n",n,low,read2(data+index[low],len-low));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
static off_t find_index(off_t pos, const uint8_t* data, off_t datalen, const off_t* index, const off_t* reverse, off_t* buckets)
|
||||
{
|
||||
if (reverse) return reverse[pos];
|
||||
|
||||
//if (datalen<2) return 0;
|
||||
uint16_t bucket = read2(data+pos, datalen-pos);
|
||||
//printf("p=%i b=%i\n",pos,bucket);
|
||||
|
||||
//TODO
|
||||
//off_t low = 0;
|
||||
//off_t high = datalen-1;
|
||||
off_t low = buckets[bucket];
|
||||
off_t high = buckets[bucket+1]-1;
|
||||
|
||||
off_t lowmatch = 2;
|
||||
off_t highmatch = 2;
|
||||
|
||||
//printf("b=%i r=%i(%i)-%i(%i)\n",bucket,low,read2(data+index[low],datalen-index[low]),high,read2(data+index[high],datalen-index[high]));
|
||||
//fflush(stdout);
|
||||
while (true)
|
||||
{
|
||||
off_t mid = low + (high-low)/2;
|
||||
off_t midpos = index[mid];
|
||||
if (midpos == pos) return mid;
|
||||
//printf("r=[%i]%i-%i \n",high-low,low,high,);
|
||||
//fflush(stdout);
|
||||
#ifdef TEST_CORRECT
|
||||
if (low >= high)
|
||||
{
|
||||
printf("E: [%i](%i): stuck at %i(%i)-%i(%i)\n", pos, read2_uc(data+pos),
|
||||
low, read2_uc(data+index[low]), high, read2_uc(data+index[high]));
|
||||
int n=0;
|
||||
while (index[n]!=pos) n++;
|
||||
printf("correct one is %i(%i)\n",n, read2_uc(data+index[n]));
|
||||
abort();
|
||||
}
|
||||
#endif
|
||||
|
||||
off_t matchlenstart = min(lowmatch, highmatch);
|
||||
|
||||
off_t len = datalen - max(pos, midpos) - matchlenstart;
|
||||
|
||||
const uint8_t* search = data+pos+matchlenstart;
|
||||
const uint8_t* here = data+midpos+matchlenstart;
|
||||
|
||||
while (len>0 && *search==*here)
|
||||
{
|
||||
search++;
|
||||
here++;
|
||||
len--;
|
||||
}
|
||||
|
||||
off_t matchlen = search-data-pos;
|
||||
|
||||
bool less;
|
||||
if (len > 0) less = (*here<*search);
|
||||
else less = (here > search) ^ EOF_IS_LAST;
|
||||
|
||||
if (less)
|
||||
{
|
||||
low = mid+1;
|
||||
lowmatch = matchlen;
|
||||
}
|
||||
else
|
||||
{
|
||||
high = mid-1;
|
||||
highmatch = matchlen;
|
||||
}
|
||||
|
||||
if (low+256 > high)
|
||||
{
|
||||
off_t i=low;
|
||||
while (true)
|
||||
{
|
||||
if (index[i]==pos) return i;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<typename off_t>
|
||||
static void create_reverse_index(off_t* index, off_t* reverse, off_t len)
|
||||
{
|
||||
//testcase: linux 3.18.14 -> 4.0.4 .xz
|
||||
//without: real23.544 user32.930
|
||||
//with: real22.636 user40.168
|
||||
//'user' jumps up quite a lot, while 'real' only moves a short bit
|
||||
//I'm not sure why the tradeoff is so bad (do the cachelines bounce THAT badly?), but I deem it not worth it.
|
||||
//#pragma omp parallel for
|
||||
for (off_t i=0;i<len;i++) reverse[index[i]]=i;
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
static off_t nextsize(off_t outpos, off_t sortedsize, off_t targetlen)
|
||||
{
|
||||
while (outpos >= sortedsize-256 && sortedsize < targetlen)
|
||||
sortedsize = min(sortedsize*4+3, targetlen);
|
||||
return sortedsize;
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
off_t lerp(off_t x, off_t y, float frac)
|
||||
{
|
||||
return x + (y-x)*frac;
|
||||
}
|
||||
|
||||
template<typename off_t>
|
||||
static bpserror bps_create_suf_core(file* source, file* target, bool moremem, struct bps_creator * out)
|
||||
{
|
||||
#define error(which) do { err = which; goto error; } while(0)
|
||||
bpserror err;
|
||||
|
||||
size_t realsourcelen = source->len;
|
||||
size_t realtargetlen = target->len;
|
||||
|
||||
size_t overflowtest = realsourcelen + realtargetlen;
|
||||
|
||||
//source+target length is bigger than size_t
|
||||
if (overflowtest < realsourcelen) return bps_too_big;
|
||||
|
||||
//source+target doesn't fit in unsigned off_t
|
||||
if ((size_t)(off_t)overflowtest != overflowtest) return bps_too_big;
|
||||
|
||||
//source+target doesn't fit in signed off_t
|
||||
if ((off_t)overflowtest < 0) return bps_too_big;
|
||||
|
||||
//the mallocs would overflow
|
||||
if (realsourcelen+realtargetlen >= SIZE_MAX/sizeof(off_t)) return bps_too_big;
|
||||
|
||||
if (realsourcelen+realtargetlen >= out->maxsize()) return bps_too_big;
|
||||
|
||||
|
||||
off_t sourcelen = realsourcelen;
|
||||
off_t targetlen = realtargetlen;
|
||||
|
||||
uint8_t* mem_joined = (uint8_t*)malloc(sizeof(uint8_t)*(realsourcelen+realtargetlen));
|
||||
|
||||
off_t* sorted = (off_t*)malloc(sizeof(off_t)*(realsourcelen+realtargetlen));
|
||||
|
||||
off_t* sorted_inverse = NULL;
|
||||
if (moremem) sorted_inverse = (off_t*)malloc(sizeof(off_t)*(realsourcelen+realtargetlen));
|
||||
|
||||
off_t* buckets = NULL;
|
||||
if (!sorted_inverse) buckets = (off_t*)malloc(sizeof(off_t)*65537);
|
||||
|
||||
if (!sorted || !mem_joined || (!sorted_inverse && !buckets))
|
||||
{
|
||||
free(mem_joined);
|
||||
free(sorted);
|
||||
free(sorted_inverse);
|
||||
free(buckets);
|
||||
return bps_out_of_mem;
|
||||
}
|
||||
|
||||
//sortedsize is how much of the target file is sorted
|
||||
off_t sortedsize = targetlen;
|
||||
//divide by 4 for each iteration, to avoid sorting 50% of the file (the sorter is slow)
|
||||
while (sortedsize/4 > sourcelen && sortedsize > 1024) sortedsize >>= 2;
|
||||
|
||||
off_t prevsortedsize = 0;
|
||||
off_t outpos = 0;
|
||||
|
||||
goto reindex; // jump into the middle so I won't need a special case to enter it
|
||||
|
||||
while (outpos < targetlen)
|
||||
{
|
||||
if (outpos >= sortedsize-256 && sortedsize < targetlen)
|
||||
{
|
||||
sortedsize = nextsize(outpos, sortedsize, targetlen);
|
||||
|
||||
reindex:
|
||||
|
||||
//this isn't an exact science
|
||||
const float percSort = sorted_inverse ? 0.67 : 0.50;
|
||||
const float percInv = sorted_inverse ? 0.11 : 0.10;
|
||||
//const float percFind = sorted_inverse ? 0.22 : 0.40; // unused
|
||||
|
||||
const size_t progPreSort = lerp(prevsortedsize, sortedsize, 0);
|
||||
const size_t progPreInv = lerp(prevsortedsize, sortedsize, percSort);
|
||||
const size_t progPreFind = lerp(prevsortedsize, sortedsize, percSort+percInv);
|
||||
|
||||
prevsortedsize = sortedsize;
|
||||
|
||||
if (!out->progress(progPreSort, targetlen)) error(bps_canceled);
|
||||
|
||||
if (target->read(mem_joined, 0, sortedsize) < (size_t)sortedsize) error(bps_io);
|
||||
if (source->read(mem_joined+sortedsize, 0, sourcelen) < (size_t)sourcelen) error(bps_io);
|
||||
out->move_target(mem_joined);
|
||||
sufsort(sorted, mem_joined, sortedsize+sourcelen);
|
||||
|
||||
if (!out->progress(progPreInv, targetlen)) error(bps_canceled);
|
||||
|
||||
if (sorted_inverse)
|
||||
create_reverse_index(sorted, sorted_inverse, sortedsize+sourcelen);
|
||||
else
|
||||
create_buckets(mem_joined, sorted, sortedsize+sourcelen, buckets);
|
||||
|
||||
if (!out->progress(progPreFind, targetlen)) error(bps_canceled);
|
||||
}
|
||||
|
||||
off_t matchlen = 0;
|
||||
off_t matchpos = adjust_match(find_index(outpos, mem_joined, sortedsize+sourcelen, sorted, sorted_inverse, buckets),
|
||||
mem_joined+outpos, sortedsize-outpos,
|
||||
mem_joined,sortedsize+sourcelen, outpos,sortedsize,
|
||||
sorted, sortedsize+sourcelen,
|
||||
&matchlen);
|
||||
|
||||
#ifdef TEST_CORRECT
|
||||
if (matchlen && matchpos >= outpos && matchpos < sortedsize) puts("ERROR: found match in invalid location"),abort();
|
||||
if (memcmp(mem_joined+matchpos, mem_joined+outpos, matchlen)) puts("ERROR: found match doesn't match"),abort();
|
||||
#endif
|
||||
|
||||
off_t taken;
|
||||
if (matchpos >= sortedsize) taken = out->match(false, matchpos-sortedsize, matchlen);
|
||||
else taken = out->match(true, matchpos, matchlen);
|
||||
#ifdef TEST_CORRECT
|
||||
if (taken < 0) puts("ERROR: match() returned negative"),abort();
|
||||
if (matchlen >= 7 && taken < matchlen) printf("ERROR: match() took %i bytes, offered %i\n", taken, matchlen),abort();
|
||||
#endif
|
||||
outpos += taken;
|
||||
}
|
||||
|
||||
out->finish(mem_joined+sortedsize, mem_joined);
|
||||
|
||||
err = bps_ok;
|
||||
|
||||
error:
|
||||
free(buckets);
|
||||
free(sorted_inverse);
|
||||
free(sorted);
|
||||
free(mem_joined);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
|
||||
template<typename T> static bpserror bps_create_suf_pick(file* source, file* target, bool moremem, struct bps_creator * bps);
|
||||
template<> bpserror bps_create_suf_pick<uint32_t>(file* source, file* target, bool moremem, struct bps_creator * bps)
|
||||
{
|
||||
return bps_create_suf_core<int32_t>(source, target, moremem, bps);
|
||||
}
|
||||
template<> bpserror bps_create_suf_pick<uint64_t>(file* source, file* target, bool moremem, struct bps_creator * bps)
|
||||
{
|
||||
bpserror err = bps_create_suf_core<int32_t>(source, target, moremem, bps);
|
||||
if (err==bps_too_big) err = bps_create_suf_core<int64_t>(source, target, moremem, bps);
|
||||
return err;
|
||||
}
|
||||
|
||||
//This one picks a function based on 32-bit integers if that fits. This halves memory use for common inputs.
|
||||
//It also handles some stuff related to the BPS headers and footers.
|
||||
extern "C"
|
||||
bpserror bps_create_delta(file* source, file* target, struct mem metadata, struct mem * patchmem,
|
||||
bool (*progress)(void* userdata, size_t done, size_t total), void* userdata, bool moremem)
|
||||
{
|
||||
bps_creator bps(source, target, metadata);
|
||||
bps.setProgress(progress, userdata);
|
||||
|
||||
size_t maindata = bps.outlen;
|
||||
|
||||
//off_t must be signed
|
||||
bpserror err = bps_create_suf_pick<size_t>(source, target, moremem, &bps);
|
||||
if (err!=bps_ok) return err;
|
||||
|
||||
*patchmem = bps.getpatch();
|
||||
|
||||
while ((patchmem->ptr[maindata]&0x80) == 0x00) maindata++;
|
||||
if (maindata==patchmem->len-12-1) return bps_identical;
|
||||
return bps_ok;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
#ifdef BPS_STANDALONE
|
||||
#include <stdio.h>
|
||||
static struct mem ReadWholeFile(const char * filename)
|
||||
{
|
||||
struct mem null = {NULL, 0};
|
||||
|
||||
FILE * file=fopen(filename, "rb");
|
||||
if (!file) return null;
|
||||
fseek(file, 0, SEEK_END);
|
||||
size_t len=ftell(file);
|
||||
fseek(file, 0, SEEK_SET);
|
||||
unsigned char * data=(unsigned char*)malloc(len);
|
||||
size_t truelen=fread(data, 1,len, file);
|
||||
fclose(file);
|
||||
if (len!=truelen)
|
||||
{
|
||||
free(data);
|
||||
return null;
|
||||
}
|
||||
|
||||
struct mem ret = { (unsigned char*)data, len };
|
||||
return ret;
|
||||
}
|
||||
static bool WriteWholeFile(const char * filename, struct mem data)
|
||||
{
|
||||
FILE * file=fopen(filename, "wb");
|
||||
if (!file) return false;
|
||||
unsigned int truelen=fwrite(data.ptr, 1,data.len, file);
|
||||
fclose(file);
|
||||
return (truelen==data.len);
|
||||
}
|
||||
int main(int argc, char * argv[])
|
||||
{
|
||||
//struct mem out = ReadWholeFile(argv[2]);
|
||||
//printf("check=%.8X\n",crc32(out.ptr, out.len));
|
||||
|
||||
struct mem in = ReadWholeFile(argv[1]);
|
||||
struct mem out = ReadWholeFile(argv[2]);
|
||||
struct mem null = {NULL, 0};
|
||||
struct mem p={NULL,0};
|
||||
//int n=50;
|
||||
//for(int i=0;i<n;i++)
|
||||
//printf("%i/%i\n",i,n),
|
||||
bps_create_delta(in,out,null,&p, NULL,NULL);
|
||||
printf("len=%lu \n",p.len);
|
||||
printf("check=%.8X\n",*(uint32_t*)(p.ptr+p.len-4));
|
||||
WriteWholeFile(argv[3], p);
|
||||
free(in.ptr);
|
||||
free(out.ptr);
|
||||
free(p.ptr);
|
||||
|
||||
#ifdef TEST_PERF
|
||||
printf("%i/%i=%f\n",match_len_tot,match_len_n,(float)match_len_tot/match_len_n);
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
614
patch/libbps.cpp
Normal file
614
patch/libbps.cpp
Normal file
@@ -0,0 +1,614 @@
|
||||
#include "libbps.h"
|
||||
|
||||
#include <stdlib.h>//malloc, realloc, free
|
||||
#include <string.h>//memcpy, memset
|
||||
#include <stdint.h>//uint8_t, uint32_t
|
||||
#include "arlib/crc32.h"//crc32
|
||||
#include "arlib/file.h"//file
|
||||
|
||||
static uint32_t read32(uint8_t * ptr)
|
||||
{
|
||||
uint32_t out;
|
||||
out =ptr[0];
|
||||
out|=ptr[1]<<8;
|
||||
out|=ptr[2]<<16;
|
||||
out|=ptr[3]<<24;
|
||||
return out;
|
||||
}
|
||||
|
||||
enum { SourceRead, TargetRead, SourceCopy, TargetCopy };
|
||||
|
||||
static bool try_add(size_t& a, size_t b)
|
||||
{
|
||||
if (SIZE_MAX-a < b) return false;
|
||||
a+=b;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool try_shift(size_t& a, size_t b)
|
||||
{
|
||||
if (SIZE_MAX>>b < a) return false;
|
||||
a<<=b;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool decodenum(const uint8_t*& ptr, size_t& out)
|
||||
{
|
||||
out=0;
|
||||
unsigned int shift=0;
|
||||
while (true)
|
||||
{
|
||||
uint8_t next=*ptr++;
|
||||
size_t addthis=(next&0x7F);
|
||||
if (shift) addthis++;
|
||||
if (!try_shift(addthis, shift)) return false;
|
||||
// unchecked because if it was shifted, the lowest bit is zero, and if not, it's <=0x7F.
|
||||
if (!try_add(out, addthis)) return false;
|
||||
if (next&0x80) return true;
|
||||
shift+=7;
|
||||
}
|
||||
}
|
||||
|
||||
#define error(which) do { error=which; goto exit; } while(0)
|
||||
#define assert_sum(a,b) do { if (SIZE_MAX-(a)<(b)) error(bps_too_big); } while(0)
|
||||
#define assert_shift(a,b) do { if (SIZE_MAX>>(b)<(a)) error(bps_too_big); } while(0)
|
||||
enum bpserror bps_apply(struct mem patch, struct mem in, struct mem * out, struct mem * metadata, bool accept_wrong_input)
|
||||
{
|
||||
enum bpserror error = bps_ok;
|
||||
out->len=0;
|
||||
out->ptr=NULL;
|
||||
if (metadata)
|
||||
{
|
||||
metadata->len=0;
|
||||
metadata->ptr=NULL;
|
||||
}
|
||||
if (patch.len<4+3+12) return bps_broken;
|
||||
|
||||
if (true)
|
||||
{
|
||||
#define read8() (*(patchat++))
|
||||
#define decodeto(var) \
|
||||
do { \
|
||||
if (!decodenum(patchat, var)) error(bps_too_big); \
|
||||
} while(false)
|
||||
#define write8(byte) (*(outat++)=byte)
|
||||
|
||||
const uint8_t * patchat=patch.ptr;
|
||||
const uint8_t * patchend=patch.ptr+patch.len-12;
|
||||
|
||||
if (read8()!='B') error(bps_broken);
|
||||
if (read8()!='P') error(bps_broken);
|
||||
if (read8()!='S') error(bps_broken);
|
||||
if (read8()!='1') error(bps_broken);
|
||||
|
||||
uint32_t crc_in_e = read32(patch.ptr+patch.len-12);
|
||||
uint32_t crc_out_e = read32(patch.ptr+patch.len-8);
|
||||
uint32_t crc_patch_e = read32(patch.ptr+patch.len-4);
|
||||
|
||||
uint32_t crc_in_a = crc32(in.ptr, in.len);
|
||||
uint32_t crc_patch_a = crc32(patch.ptr, patch.len-4);
|
||||
|
||||
if (crc_patch_a != crc_patch_e) error(bps_broken);
|
||||
|
||||
size_t inlen;
|
||||
decodeto(inlen);
|
||||
|
||||
size_t outlen;
|
||||
decodeto(outlen);
|
||||
|
||||
if (inlen!=in.len || crc_in_a!=crc_in_e)
|
||||
{
|
||||
if (in.len==outlen && crc_in_a==crc_out_e) error=bps_to_output;
|
||||
else error=bps_not_this;
|
||||
if (!accept_wrong_input) goto exit;
|
||||
}
|
||||
|
||||
out->len=outlen;
|
||||
out->ptr=(uint8_t*)malloc(outlen);
|
||||
|
||||
const uint8_t * instart=in.ptr;
|
||||
const uint8_t * inreadat=in.ptr;
|
||||
const uint8_t * inend=in.ptr+in.len;
|
||||
|
||||
uint8_t * outstart=out->ptr;
|
||||
uint8_t * outreadat=out->ptr;
|
||||
uint8_t * outat=out->ptr;
|
||||
uint8_t * outend=out->ptr+out->len;
|
||||
|
||||
size_t metadatalen;
|
||||
decodeto(metadatalen);
|
||||
|
||||
if (metadata && metadatalen)
|
||||
{
|
||||
metadata->len=metadatalen;
|
||||
metadata->ptr=(uint8_t*)malloc(metadatalen+1);
|
||||
for (size_t i=0;i<metadatalen;i++) metadata->ptr[i]=read8();
|
||||
metadata->ptr[metadatalen]='\0';//just to be on the safe side - that metadata is assumed to be text, might as well terminate it
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i=0;i<metadatalen;i++) (void)read8();
|
||||
}
|
||||
|
||||
while (patchat<patchend)
|
||||
{
|
||||
size_t thisinstr;
|
||||
decodeto(thisinstr);
|
||||
size_t length=(thisinstr>>2)+1;
|
||||
int action=(thisinstr&3);
|
||||
if (outat+length>outend) error(bps_broken);
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case SourceRead:
|
||||
{
|
||||
if (outat-outstart+length > in.len) error(bps_broken);
|
||||
for (size_t i=0;i<length;i++)
|
||||
{
|
||||
size_t pos = outat-outstart; // don't inline, write8 changes outat
|
||||
write8(instart[pos]);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case TargetRead:
|
||||
{
|
||||
if (patchat+length>patchend) error(bps_broken);
|
||||
for (size_t i=0;i<length;i++) write8(read8());
|
||||
}
|
||||
break;
|
||||
case SourceCopy:
|
||||
{
|
||||
size_t encodeddistance;
|
||||
decodeto(encodeddistance);
|
||||
size_t distance=encodeddistance>>1;
|
||||
if ((encodeddistance&1)==0) inreadat+=distance;
|
||||
else inreadat-=distance;
|
||||
|
||||
if (inreadat<instart || inreadat+length>inend) error(bps_broken);
|
||||
for (size_t i=0;i<length;i++) write8(*inreadat++);
|
||||
}
|
||||
break;
|
||||
case TargetCopy:
|
||||
{
|
||||
size_t encodeddistance;
|
||||
decodeto(encodeddistance);
|
||||
size_t distance=encodeddistance>>1;
|
||||
if ((encodeddistance&1)==0) outreadat+=distance;
|
||||
else outreadat-=distance;
|
||||
|
||||
if (outreadat<outstart || outreadat>=outat || outreadat+length>outend) error(bps_broken);
|
||||
for (size_t i=0;i<length;i++) write8(*outreadat++);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (patchat!=patchend) error(bps_broken);
|
||||
if (outat!=outend) error(bps_broken);
|
||||
|
||||
uint32_t crc_out_a = crc32(out->ptr, out->len);
|
||||
|
||||
if (crc_out_a!=crc_out_e)
|
||||
{
|
||||
error=bps_not_this;
|
||||
if (!accept_wrong_input) goto exit;
|
||||
}
|
||||
return error;
|
||||
#undef read8
|
||||
#undef decodeto
|
||||
#undef write8
|
||||
}
|
||||
|
||||
exit:
|
||||
free(out->ptr);
|
||||
out->len=0;
|
||||
out->ptr=NULL;
|
||||
if (metadata)
|
||||
{
|
||||
free(metadata->ptr);
|
||||
metadata->len=0;
|
||||
metadata->ptr=NULL;
|
||||
}
|
||||
return error;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#define write(val) \
|
||||
do { \
|
||||
out[outlen++]=(val); \
|
||||
if (outlen==outbuflen) \
|
||||
{ \
|
||||
outbuflen*=2; \
|
||||
out=(uint8_t*)realloc(out, outbuflen); \
|
||||
} \
|
||||
} while(0)
|
||||
#define write32(val) \
|
||||
do { \
|
||||
uint32_t tmp=(val); \
|
||||
write(tmp); \
|
||||
write(tmp>>8); \
|
||||
write(tmp>>16); \
|
||||
write(tmp>>24); \
|
||||
} while(0)
|
||||
#define writenum(val) \
|
||||
do { \
|
||||
size_t tmpval=(val); \
|
||||
while (true) \
|
||||
{ \
|
||||
uint8_t tmpbyte=(tmpval&0x7F); \
|
||||
tmpval>>=7; \
|
||||
if (!tmpval) \
|
||||
{ \
|
||||
write(tmpbyte|0x80); \
|
||||
break; \
|
||||
} \
|
||||
write(tmpbyte); \
|
||||
tmpval--; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
enum bpserror bps_create_linear(struct mem sourcemem, struct mem targetmem, struct mem metadata, struct mem * patchmem)
|
||||
{
|
||||
if (sourcemem.len>=(SIZE_MAX>>2) - 16) return bps_too_big;//the 16 is just to be on the safe side, I don't think it's needed.
|
||||
if (targetmem.len>=(SIZE_MAX>>2) - 16) return bps_too_big;
|
||||
|
||||
const uint8_t * source=sourcemem.ptr;
|
||||
const uint8_t * sourceend=sourcemem.ptr+sourcemem.len;
|
||||
if (sourcemem.len>targetmem.len) sourceend=sourcemem.ptr+targetmem.len;
|
||||
const uint8_t * targetbegin=targetmem.ptr;
|
||||
const uint8_t * target=targetmem.ptr;
|
||||
const uint8_t * targetend=targetmem.ptr+targetmem.len;
|
||||
|
||||
const uint8_t * targetcopypos=targetbegin;
|
||||
|
||||
size_t outbuflen=4096;
|
||||
uint8_t * out=(uint8_t*)malloc(outbuflen);
|
||||
size_t outlen=0;
|
||||
write('B');
|
||||
write('P');
|
||||
write('S');
|
||||
write('1');
|
||||
writenum(sourcemem.len);
|
||||
writenum(targetmem.len);
|
||||
writenum(metadata.len);
|
||||
for (size_t i=0;i<metadata.len;i++) write(metadata.ptr[i]);
|
||||
|
||||
size_t mainContentPos=outlen;
|
||||
|
||||
const uint8_t * lastknownchange=targetbegin;
|
||||
while (target<targetend)
|
||||
{
|
||||
size_t numunchanged=0;
|
||||
while (source+numunchanged<sourceend && source[numunchanged]==target[numunchanged]) numunchanged++;
|
||||
if (numunchanged>1)
|
||||
{
|
||||
//assert_shift((numunchanged-1), 2);
|
||||
writenum((numunchanged-1)<<2 | 0);//SourceRead
|
||||
source+=numunchanged;
|
||||
target+=numunchanged;
|
||||
}
|
||||
|
||||
size_t numchanged=0;
|
||||
if (lastknownchange>target) numchanged=lastknownchange-target;
|
||||
while ((source+numchanged>=sourceend ||
|
||||
source[numchanged]!=target[numchanged] ||
|
||||
source[numchanged+1]!=target[numchanged+1] ||
|
||||
source[numchanged+2]!=target[numchanged+2]) &&
|
||||
target+numchanged<targetend)
|
||||
{
|
||||
numchanged++;
|
||||
if (source+numchanged>=sourceend) numchanged=targetend-target;
|
||||
}
|
||||
lastknownchange=target+numchanged;
|
||||
if (numchanged)
|
||||
{
|
||||
//assert_shift((numchanged-1), 2);
|
||||
size_t rle1start=(target==targetbegin);
|
||||
while (true)
|
||||
{
|
||||
if (
|
||||
target[rle1start-1]==target[rle1start+0] &&
|
||||
target[rle1start+0]==target[rle1start+1] &&
|
||||
target[rle1start+1]==target[rle1start+2] &&
|
||||
target[rle1start+2]==target[rle1start+3])
|
||||
{
|
||||
numchanged=rle1start;
|
||||
break;
|
||||
}
|
||||
if (
|
||||
target[rle1start-2]==target[rle1start+0] &&
|
||||
target[rle1start-1]==target[rle1start+1] &&
|
||||
target[rle1start+0]==target[rle1start+2] &&
|
||||
target[rle1start+1]==target[rle1start+3] &&
|
||||
target[rle1start+2]==target[rle1start+4])
|
||||
{
|
||||
numchanged=rle1start;
|
||||
break;
|
||||
}
|
||||
if (rle1start+3>=numchanged) break;
|
||||
rle1start++;
|
||||
}
|
||||
if (numchanged)
|
||||
{
|
||||
writenum((numchanged-1)<<2 | TargetRead);
|
||||
for (size_t i=0;i<numchanged;i++)
|
||||
{
|
||||
write(target[i]);
|
||||
}
|
||||
source+=numchanged;
|
||||
target+=numchanged;
|
||||
}
|
||||
if (target[-2]==target[0] && target[-1]==target[1] && target[0]==target[2])
|
||||
{
|
||||
//two-byte RLE
|
||||
size_t rlelen=0;
|
||||
while (target+rlelen<targetend && target[0]==target[rlelen+0] && target[1]==target[rlelen+1]) rlelen+=2;
|
||||
writenum((rlelen-1)<<2 | TargetCopy);
|
||||
writenum((target-targetcopypos-2)<<1);
|
||||
source+=rlelen;
|
||||
target+=rlelen;
|
||||
targetcopypos=target-2;
|
||||
}
|
||||
else if (target[-1]==target[0] && target[0]==target[1])
|
||||
{
|
||||
//one-byte RLE
|
||||
size_t rlelen=0;
|
||||
while (target+rlelen<targetend && target[0]==target[rlelen]) rlelen++;
|
||||
writenum((rlelen-1)<<2 | TargetCopy);
|
||||
writenum((target-targetcopypos-1)<<1);
|
||||
source+=rlelen;
|
||||
target+=rlelen;
|
||||
targetcopypos=target-1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
write32(crc32(sourcemem.ptr, sourcemem.len));
|
||||
write32(crc32(targetmem.ptr, targetmem.len));
|
||||
write32(crc32(out, outlen));
|
||||
|
||||
patchmem->ptr=out;
|
||||
patchmem->len=outlen;
|
||||
|
||||
//while this may look like it can be fooled by a patch containing one of any other command, it
|
||||
// can't, because the ones that aren't SourceRead requires an argument.
|
||||
size_t i;
|
||||
for (i=mainContentPos;(out[i]&0x80)==0x00;i++) {}
|
||||
if (i==outlen-12-1) return bps_identical;
|
||||
|
||||
return bps_ok;
|
||||
}
|
||||
|
||||
#undef write_nocrc
|
||||
#undef write
|
||||
#undef writenum
|
||||
|
||||
void bps_free(struct mem mem)
|
||||
{
|
||||
free(mem.ptr);
|
||||
}
|
||||
#undef error
|
||||
|
||||
|
||||
|
||||
struct bpsinfo bps_get_info(file* patch, bool changefrac)
|
||||
{
|
||||
#define error(why) do { ret.error=why; return ret; } while(0)
|
||||
struct bpsinfo ret;
|
||||
size_t len = patch->len;
|
||||
if (len<4+3+12) error(bps_broken);
|
||||
|
||||
uint8_t top[256];
|
||||
size_t toplen = len>256 ? 256 : len;
|
||||
if (patch->read(top, 0, toplen) < toplen) error(bps_io);
|
||||
if (memcmp(top, "BPS1", 4)) error(bps_broken);
|
||||
|
||||
const uint8_t* patchdat=top+4;
|
||||
if (!decodenum(patchdat, ret.size_in)) error(bps_too_big);
|
||||
if (!decodenum(patchdat, ret.size_out)) error(bps_too_big);
|
||||
|
||||
uint8_t checksums[12];
|
||||
if (patch->read(checksums, len-12, 12) < 12) error(bps_io);
|
||||
ret.crc_in = read32(checksums+0);
|
||||
ret.crc_out = read32(checksums+4);
|
||||
ret.crc_patch=read32(checksums+8);
|
||||
|
||||
if (changefrac && ret.size_in>0)
|
||||
{
|
||||
//algorithm: each command adds its length to the numerator, unless it's above 32, in which case
|
||||
// it adds 32; or if it's SourceRead, in which case it adds 0
|
||||
//denominator is just input length
|
||||
uint8_t* patchbin=(uint8_t*)malloc(len);
|
||||
if (patch->read(patchbin, 0, len) < len) error(bps_io);
|
||||
size_t outpos=0; // position in the output file
|
||||
size_t changeamt=0; // change score
|
||||
const uint8_t* patchat=patchbin+(patchdat-top);
|
||||
|
||||
size_t metasize;
|
||||
if (!decodenum(patchat, metasize)) error(bps_too_big);
|
||||
patchat+=metasize;
|
||||
|
||||
const uint8_t* patchend=patchbin+len-12;
|
||||
|
||||
while (patchat<patchend && outpos<ret.size_in)
|
||||
{
|
||||
size_t thisinstr;
|
||||
decodenum(patchat, thisinstr);
|
||||
size_t length=(thisinstr>>2)+1;
|
||||
int action=(thisinstr&3);
|
||||
int min_len_32 = (length<32 ? length : 32);
|
||||
|
||||
switch (action)
|
||||
{
|
||||
case SourceRead:
|
||||
{
|
||||
changeamt+=0;
|
||||
}
|
||||
break;
|
||||
case TargetRead:
|
||||
{
|
||||
changeamt+=min_len_32;
|
||||
patchat+=length;
|
||||
}
|
||||
break;
|
||||
case SourceCopy:
|
||||
case TargetCopy:
|
||||
{
|
||||
changeamt+=min_len_32;
|
||||
size_t ignore;
|
||||
decodenum(patchat, ignore);
|
||||
}
|
||||
break;
|
||||
}
|
||||
outpos+=length;
|
||||
}
|
||||
if (patchat>patchend || outpos>ret.size_out) error(bps_broken);
|
||||
ret.change_num = (changeamt<ret.size_in ? changeamt : ret.size_in);
|
||||
ret.change_denom = ret.size_in;
|
||||
|
||||
free(patchbin);
|
||||
}
|
||||
else
|
||||
{
|
||||
//this also happens if change fraction is not requested, but it's undefined behaviour anyways.
|
||||
ret.change_num=1;
|
||||
ret.change_denom=1;
|
||||
}
|
||||
|
||||
ret.error=bps_ok;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
|
||||
#if 0
|
||||
#warning Disable this in release versions.
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
//Congratulations, you found the undocumented feature! It compares two equivalent BPS patches and
|
||||
// tells where each one is more compact. (It crashes or gives bogus answers on invalid or
|
||||
// non-equivalent patches.) Have fun.
|
||||
void bps_compare(struct mem patch1mem, struct mem patch2mem)
|
||||
{
|
||||
const uint8_t * patch[2]={patch1mem.ptr, patch2mem.ptr};
|
||||
size_t patchpos[2]={0,0};
|
||||
size_t patchlen[2]={patch1mem.len-12, patch2mem.len-12};
|
||||
size_t patchoutpos[2]={0,0};
|
||||
|
||||
size_t patchcopypos[2][4]={0,0};//[0] and [1] are unused, but this is just debug code, it doesn't need to be neat.
|
||||
|
||||
#define read8(id) (patch[id][patchpos[id]++])
|
||||
#define decodeto(id, var) \
|
||||
do { \
|
||||
var=0; \
|
||||
int shift=0; \
|
||||
while (true) \
|
||||
{ \
|
||||
uint8_t next=read8(id); \
|
||||
size_t addthis=(next&0x7F)<<shift; \
|
||||
var+=addthis; \
|
||||
if (next&0x80) break; \
|
||||
shift+=7; \
|
||||
var+=1<<shift; \
|
||||
} \
|
||||
} while(false)
|
||||
|
||||
size_t lastmatch=0;
|
||||
size_t patchposatmatch[2]={0,0};
|
||||
|
||||
size_t outlen;
|
||||
patch[0]+=4; patch[1]+=4;//BPS1
|
||||
size_t tempuint;
|
||||
decodeto(0, tempuint); decodeto(1, tempuint);//source-size
|
||||
decodeto(0, outlen); decodeto(1, outlen);//target-size
|
||||
decodeto(0, tempuint); patch[0]+=tempuint;//metadata
|
||||
decodeto(1, tempuint); patch[1]+=tempuint;//metadata
|
||||
|
||||
bool show=false;
|
||||
while (patchpos[0]<patchlen[0] && patchpos[1]<patchlen[1])
|
||||
{
|
||||
bool step[2]={(patchoutpos[0]<=patchoutpos[1]), (patchoutpos[0]>=patchoutpos[1])};
|
||||
char describe[2][256];
|
||||
for (int i=0;i<2;i++)
|
||||
{
|
||||
if (step[i])
|
||||
{
|
||||
size_t patchposstart=patchpos[i];
|
||||
decodeto(i, tempuint);
|
||||
size_t len=(tempuint>>2)+1;
|
||||
patchoutpos[i]+=len;
|
||||
int action=(tempuint&3);
|
||||
//enum { SourceRead, TargetRead, SourceCopy, TargetCopy };
|
||||
const char * actionnames[]={"SourceRead", "TargetRead", "SourceCopy", "TargetCopy"};
|
||||
if (action==TargetRead) patchpos[i]+=len;
|
||||
if (action==SourceCopy || action==TargetCopy)
|
||||
{
|
||||
decodeto(i, tempuint);
|
||||
int delta = tempuint>>1;
|
||||
if (tempuint&1) delta=-delta;
|
||||
patchcopypos[i][action]+=delta;
|
||||
sprintf(describe[i], "%s from %i (%+i) for %i in %i", actionnames[action], patchcopypos[i][action], delta, len, patchpos[i]-patchposstart);
|
||||
patchcopypos[i][action]+=len;
|
||||
}
|
||||
else sprintf(describe[i], "%s from %i for %i in %i", actionnames[action], patchoutpos[i], len, patchpos[i]-patchposstart);
|
||||
if (!step[i^1])
|
||||
{
|
||||
printf("%i: %s\n", i+1, describe[i]);
|
||||
show=true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (step[0] && step[1])
|
||||
{
|
||||
if (!strcmp(describe[0], describe[1])) /*printf("3: %s\n", describe[0])*/;
|
||||
else
|
||||
{
|
||||
printf("1: %s\n2: %s\n", describe[0], describe[1]);
|
||||
show=true;
|
||||
}
|
||||
}
|
||||
if (patchoutpos[0]==patchoutpos[1])
|
||||
{
|
||||
size_t used[2]={patchpos[0]-patchposatmatch[0], patchpos[1]-patchposatmatch[1]};
|
||||
char which='=';
|
||||
if (used[0]<used[1]) which='+';
|
||||
if (used[0]>used[1]) which='-';
|
||||
if (show)
|
||||
{
|
||||
printf("%c: %i,%i bytes since last match (%i)\n", which, used[0], used[1], patchoutpos[0]);
|
||||
show=false;
|
||||
}
|
||||
patchposatmatch[0]=patchpos[0];
|
||||
patchposatmatch[1]=patchpos[1];
|
||||
lastmatch=patchoutpos[0];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static struct mem ReadWholeFile(const char * filename)
|
||||
{
|
||||
struct mem null = {NULL, 0};
|
||||
|
||||
FILE * file=fopen(filename, "rb");
|
||||
if (!file) return null;
|
||||
fseek(file, 0, SEEK_END);
|
||||
size_t len=ftell(file);
|
||||
fseek(file, 0, SEEK_SET);
|
||||
unsigned char * data=(unsigned char*)malloc(len);
|
||||
size_t truelen=fread(data, 1,len, file);
|
||||
fclose(file);
|
||||
if (len!=truelen)
|
||||
{
|
||||
free(data);
|
||||
return null;
|
||||
}
|
||||
|
||||
struct mem ret = { (unsigned char*)data, len };
|
||||
return ret;
|
||||
}
|
||||
int main(int argc,char**argv)
|
||||
{
|
||||
bps_compare(ReadWholeFile(argv[1]),ReadWholeFile(argv[2]));
|
||||
}
|
||||
#endif
|
||||
81
patch/libbps.h
Normal file
81
patch/libbps.h
Normal file
@@ -0,0 +1,81 @@
|
||||
#include "global.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
struct file; // For C, you can pass this around, but you can't really use it. But you can still use the other functions.
|
||||
|
||||
enum bpserror {
|
||||
bps_ok,//Patch applied or created successfully.
|
||||
|
||||
bps_to_output,//You attempted to apply a patch to its output.
|
||||
bps_not_this, //This is not the intended input file for this patch.
|
||||
bps_broken, //This is not a BPS patch, or it's malformed somehow.
|
||||
bps_io, //The patch could not be read.
|
||||
|
||||
bps_identical, //The input files are identical.
|
||||
bps_too_big, //Somehow, you're asking for something a size_t can't represent.
|
||||
bps_out_of_mem,//Memory allocation failure.
|
||||
bps_canceled, //The callback returned false.
|
||||
|
||||
bps_shut_up_gcc//This one isn't used, it's just to kill a stray comma warning.
|
||||
};
|
||||
|
||||
//Applies the given BPS patch to the given ROM and puts it in 'out'. Metadata, if present and
|
||||
// requested ('metadata'!=NULL), is also returned. Send both to bps_free when you're done with them.
|
||||
//If accept_wrong_input is true, it may return bps_to_output or bps_not_this, while putting non-NULL in out/metadata.
|
||||
enum bpserror bps_apply(struct mem patch, struct mem in, struct mem * out, struct mem * metadata, bool accept_wrong_input);
|
||||
|
||||
//Creates a BPS patch that converts source to target and stores it to patch. It is safe to give
|
||||
// {NULL,0} as metadata.
|
||||
enum bpserror bps_create_linear(struct mem source, struct mem target, struct mem metadata, struct mem * patch);
|
||||
|
||||
//Very similar to bps_create_linear; the difference is that this one takes longer to run, but
|
||||
// generates smaller patches.
|
||||
//Because it can take much longer, a progress meter is supplied; total is guaranteed to be constant
|
||||
// between every call until this function returns, done is guaranteed to increase between each
|
||||
// call, and done/total is an approximate percentage counter. Anything else is undefined; for
|
||||
// example, progress may or may not be called for done=0, progress may or may not be called for
|
||||
// done=total, done may or may not increase by the same amount between each call, and the duration
|
||||
// between each call may or may not be constant.
|
||||
//To cancel the patch creation, return false from the callback.
|
||||
//It is safe to pass in NULL for the progress indicator if you're not interested. If the callback is
|
||||
// NULL, it can obviously not be canceled that way (though if it's a CLI program, you can always
|
||||
// Ctrl-C it).
|
||||
//The 'moremem' flag makes it use about twice as much memory (9*(source+target) instead of 5*), but is usually slightly faster.
|
||||
enum bpserror bps_create_delta(file* source, file* target, struct mem metadata, struct mem * patch,
|
||||
bool (*progress)(void* userdata, size_t done, size_t total), void* userdata,
|
||||
bool moremem);
|
||||
|
||||
//Frees the memory returned in the output parameters of the above. Do not call it twice on the same
|
||||
// input, nor on anything you got from anywhere else. bps_free is guaranteed to be equivalent to
|
||||
// calling stdlib.h's free() on mem.ptr.
|
||||
void bps_free(struct mem mem);
|
||||
|
||||
struct bpsinfo {
|
||||
enum bpserror error; // If this is not bps_ok, all other values are undefined.
|
||||
|
||||
size_t size_in;
|
||||
size_t size_out;
|
||||
|
||||
uint32_t crc_in;
|
||||
uint32_t crc_out;
|
||||
uint32_t crc_patch;
|
||||
|
||||
//Tells approximately how much of the input ROM is changed compared to the output ROM.
|
||||
//It's quite heuristic. The algorithm may change with or without notice.
|
||||
//As of writing, I believe this is accurate to 2 significant digits in base 10.
|
||||
//It's also more expensive to calculate than the other data, so it's optional.
|
||||
//If you don't want it, their values are undefined.
|
||||
//The denominator is always guaranteed nonzero, even if something else says it's undefined.
|
||||
//Note that this can return success for invalid patches.
|
||||
size_t change_num;
|
||||
size_t change_denom;
|
||||
};
|
||||
struct bpsinfo bps_get_info(file* patch, bool changefrac);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
388
patch/libips.cpp
Normal file
388
patch/libips.cpp
Normal file
@@ -0,0 +1,388 @@
|
||||
#include <stdlib.h>//malloc, realloc, free
|
||||
#include <string.h>//memcpy, memset
|
||||
|
||||
#include "libips.h"
|
||||
|
||||
#define min(a,b) ((a)<(b)?(a):(b))
|
||||
#define max(a,b) ((a)>(b)?(a):(b))
|
||||
#define clamp(a,b,c) max(a,min(b,c))
|
||||
|
||||
struct ipsstudy {
|
||||
enum ipserror error;
|
||||
unsigned int outlen_min;
|
||||
unsigned int outlen_max;
|
||||
unsigned int outlen_min_mem;
|
||||
};
|
||||
|
||||
enum ipserror ips_study(struct mem patch, struct ipsstudy * study)
|
||||
{
|
||||
study->error=ips_invalid;
|
||||
if (patch.len<8) return ips_invalid;
|
||||
const unsigned char * patchat=patch.ptr;
|
||||
const unsigned char * patchend=patchat+patch.len;
|
||||
#define read8() ((patchat<patchend)?(*patchat++):0)
|
||||
#define read16() ((patchat+1<patchend)?(patchat+=2,((patchat[-2]<<8)|patchat[-1])):0)
|
||||
#define read24() ((patchat+2<patchend)?(patchat+=3,((patchat[-3]<<16)|(patchat[-2]<<8)|patchat[-1])):0)
|
||||
if (read8()!='P' ||
|
||||
read8()!='A' ||
|
||||
read8()!='T' ||
|
||||
read8()!='C' ||
|
||||
read8()!='H')
|
||||
{
|
||||
return ips_invalid;
|
||||
}
|
||||
unsigned int offset=read24();
|
||||
unsigned int outlen=0;
|
||||
unsigned int thisout=0;
|
||||
unsigned int lastoffset=0;
|
||||
bool w_scrambled=false;
|
||||
while (offset!=0x454F46)//454F46=EOF
|
||||
{
|
||||
unsigned int size=read16();
|
||||
if (size==0)
|
||||
{
|
||||
size=read16();
|
||||
if (!size) w_scrambled=true;
|
||||
thisout=offset+size;
|
||||
read8();
|
||||
}
|
||||
else
|
||||
{
|
||||
thisout=offset+size;
|
||||
patchat+=size;
|
||||
}
|
||||
if (offset<lastoffset) w_scrambled=true;
|
||||
lastoffset=offset;
|
||||
if (thisout>outlen) outlen=thisout;
|
||||
if (patchat>=patchend) return ips_invalid;
|
||||
offset=read24();
|
||||
}
|
||||
study->outlen_min_mem=outlen;
|
||||
study->outlen_max=0xFFFFFFFF;
|
||||
if (patchat+3==patchend)
|
||||
{
|
||||
unsigned int truncate=read24();
|
||||
study->outlen_max=truncate;
|
||||
if (outlen>truncate)
|
||||
{
|
||||
outlen=truncate;
|
||||
w_scrambled=true;
|
||||
}
|
||||
}
|
||||
if (patchat!=patchend) return ips_invalid;
|
||||
study->outlen_min=outlen;
|
||||
#undef read8
|
||||
#undef read16
|
||||
#undef read24
|
||||
study->error=ips_ok;
|
||||
if (w_scrambled) study->error=ips_scrambled;
|
||||
return study->error;
|
||||
}
|
||||
|
||||
enum ipserror ips_apply_study(struct mem patch, struct ipsstudy * study, struct mem in, struct mem * out)
|
||||
{
|
||||
out->ptr=NULL;
|
||||
out->len=0;
|
||||
if (study->error==ips_invalid) return study->error;
|
||||
#define read8() (*patchat++)//guaranteed to not overflow at this point, we already checked the patch
|
||||
#define read16() (patchat+=2,((patchat[-2]<<8)|patchat[-1]))
|
||||
#define read24() (patchat+=3,((patchat[-3]<<16)|(patchat[-2]<<8)|patchat[-1]))
|
||||
unsigned int outlen=clamp(study->outlen_min, in.len, study->outlen_max);
|
||||
out->ptr=(uint8_t*)malloc(max(outlen, study->outlen_min_mem));
|
||||
out->len=outlen;
|
||||
|
||||
bool anychanges=false;
|
||||
if (outlen!=in.len) anychanges=true;
|
||||
|
||||
if (out->len>in.len)
|
||||
{
|
||||
memcpy(out->ptr, in.ptr, in.len);
|
||||
memset(out->ptr+in.len, 0, out->len-in.len);
|
||||
}
|
||||
else memcpy(out->ptr, in.ptr, outlen);
|
||||
const unsigned char * patchat=patch.ptr+5;
|
||||
unsigned int offset=read24();
|
||||
while (offset!=0x454F46)
|
||||
{
|
||||
unsigned int size=read16();
|
||||
if (size==0)
|
||||
{
|
||||
size=read16();
|
||||
if (!size) {}//no clue (fix the change detector if changing this)
|
||||
unsigned char b=read8();
|
||||
|
||||
if (size && (out->ptr[offset]!=b || memcmp(out->ptr+offset, out->ptr+offset, size-1))) anychanges=true;
|
||||
|
||||
memset(out->ptr+offset, b, size);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (memcmp(out->ptr+offset, patchat, size)) anychanges=true;
|
||||
|
||||
memcpy(out->ptr+offset, patchat, size);
|
||||
patchat+=size;
|
||||
}
|
||||
offset=read24();
|
||||
}
|
||||
#undef read8
|
||||
#undef read16
|
||||
#undef read24
|
||||
|
||||
if (study->outlen_max!=0xFFFFFFFF && in.len<=study->outlen_max) study->error=ips_notthis;//truncate data without this being needed is a poor idea
|
||||
if (!anychanges) study->error=ips_thisout;
|
||||
return study->error;
|
||||
}
|
||||
|
||||
enum ipserror ips_apply(struct mem patch, struct mem in, struct mem * out)
|
||||
{
|
||||
struct ipsstudy study;
|
||||
ips_study(patch, &study);
|
||||
return ips_apply_study(patch, &study, in, out);
|
||||
}
|
||||
|
||||
//Known situations where this function does not generate an optimal patch:
|
||||
//In: 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80 80
|
||||
//Out: FF FF FF FF FF FF FF FF 00 01 02 03 04 05 06 07 FF FF FF FF FF FF FF FF
|
||||
//IPS: [ RLE ] [ Copy ] [ RLE ]
|
||||
//Possible improvement: RLE across the entire file, copy on top of that.
|
||||
//Rationale: It would be a huge pain to create such a multi-pass tool if it should support writing a byte
|
||||
// more than twice, and I don't like half-assing stuff. It's also unlikely to apply to anything.
|
||||
|
||||
|
||||
//Known improvements over LIPS:
|
||||
//In: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F
|
||||
//Out: FF 01 02 03 04 05 FF FF FF FF FF FF FF FF FF FF
|
||||
//LIPS:[ Copy ] [ RLE ]
|
||||
//Mine:[] [ Unchanged ] [ RLE ]
|
||||
//Rationale: While LIPS can break early if it finds something RLEable in the middle of a block, it's not
|
||||
// smart enough to back off if there's something unchanged between the changed area and the RLEable spot.
|
||||
|
||||
//In: FF FF FF FF FF FF FF
|
||||
//Out: 00 00 00 00 01 02 03
|
||||
//LIPS:[ RLE ] [ Copy ]
|
||||
//Mine:[ Copy ]
|
||||
//Rationale: Mistuned heuristics in LIPS.
|
||||
|
||||
//It is also known that I win in some other situations. I didn't bother checking which, though.
|
||||
|
||||
//There are no known cases where LIPS wins over libips.
|
||||
|
||||
enum ipserror ips_create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem)
|
||||
{
|
||||
unsigned int sourcelen=sourcemem.len;
|
||||
unsigned int targetlen=targetmem.len;
|
||||
const unsigned char * source=sourcemem.ptr;
|
||||
const unsigned char * target=targetmem.ptr;
|
||||
|
||||
patchmem->ptr=NULL;
|
||||
patchmem->len=0;
|
||||
|
||||
if (targetlen>=16777216) return ips_16MB;
|
||||
|
||||
unsigned int offset=0;
|
||||
unsigned int outbuflen=4096;
|
||||
unsigned char * out=(uint8_t*)malloc(outbuflen);
|
||||
unsigned int outlen=0;
|
||||
#define write8(val) do { out[outlen++]=(val); if (outlen==outbuflen) { outbuflen*=2; out=(uint8_t*)realloc(out, outbuflen); } } while(0)
|
||||
#define write16(val) do { write8((val)>>8); write8((val)); } while(0)
|
||||
#define write24(val) do { write8((val)>>16); write8((val)>>8); write8((val)); } while(0)
|
||||
write8('P');
|
||||
write8('A');
|
||||
write8('T');
|
||||
write8('C');
|
||||
write8('H');
|
||||
int lastknownchange=0;
|
||||
//int forcewrite=(targetlen>sourcelen?1:0);
|
||||
while (offset<targetlen)
|
||||
{
|
||||
while (offset<sourcelen && (offset<sourcelen?source[offset]:0)==target[offset]) offset++;
|
||||
//check how much we need to edit until it starts getting similar
|
||||
int thislen=0;
|
||||
int consecutiveunchanged=0;
|
||||
thislen=lastknownchange-offset;
|
||||
if (thislen<0) thislen=0;
|
||||
while (true)
|
||||
{
|
||||
unsigned int thisbyte=offset+thislen+consecutiveunchanged;
|
||||
if (thisbyte<sourcelen && (thisbyte<sourcelen?source[thisbyte]:0)==target[thisbyte]) consecutiveunchanged++;
|
||||
else
|
||||
{
|
||||
thislen+=consecutiveunchanged+1;
|
||||
consecutiveunchanged=0;
|
||||
}
|
||||
if (consecutiveunchanged>=6 || thislen>=65536) break;
|
||||
}
|
||||
|
||||
//avoid premature EOF
|
||||
if (offset==0x454F46)
|
||||
{
|
||||
offset--;
|
||||
thislen++;
|
||||
}
|
||||
|
||||
lastknownchange=offset+thislen;
|
||||
if (thislen>65535) thislen=65535;
|
||||
if (offset+thislen>targetlen) thislen=targetlen-offset;
|
||||
if (offset==targetlen) continue;
|
||||
|
||||
//check if RLE here is worthwhile
|
||||
int byteshere;
|
||||
for (byteshere=0;byteshere<thislen && target[offset]==target[offset+byteshere];byteshere++) {}
|
||||
if (byteshere==thislen)
|
||||
{
|
||||
int thisbyte=target[offset];
|
||||
int i=0;
|
||||
while (true)
|
||||
{
|
||||
unsigned int pos=offset+byteshere+i-1;
|
||||
if (pos>=targetlen || target[pos]!=thisbyte || byteshere+i>65535) break;
|
||||
if (pos>=sourcelen || (pos<sourcelen?source[pos]:0)!=thisbyte)
|
||||
{
|
||||
byteshere+=i;
|
||||
thislen+=i;
|
||||
i=0;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
if ((byteshere>8-5 && byteshere==thislen) || byteshere>8)
|
||||
{
|
||||
write24(offset);
|
||||
write16(0);
|
||||
write16(byteshere);
|
||||
write8(target[offset]);
|
||||
offset+=byteshere;
|
||||
}
|
||||
else
|
||||
{
|
||||
//check if we'd gain anything from ending the block early and switching to RLE
|
||||
int byteshere=0;
|
||||
int stopat=0;
|
||||
while (stopat+byteshere<thislen)
|
||||
{
|
||||
if (target[offset+stopat]==target[offset+stopat+byteshere]) byteshere++;
|
||||
else
|
||||
{
|
||||
stopat+=byteshere;
|
||||
byteshere=0;
|
||||
}
|
||||
if (byteshere>8+5 || //rle-worthy despite two ips headers
|
||||
(byteshere>8 && stopat+byteshere==thislen) || //rle-worthy at end of data
|
||||
(byteshere>8 && !memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy
|
||||
{
|
||||
if (stopat) thislen=stopat;
|
||||
break;//we don't scan the entire block if we know we'll want to RLE, that'd gain nothing.
|
||||
}
|
||||
}
|
||||
//don't write unchanged bytes at the end of a block if we want to RLE the next couple of bytes
|
||||
if (offset+thislen!=targetlen)
|
||||
{
|
||||
while (offset+thislen-1<sourcelen && target[offset+thislen-1]==(offset+thislen-1<sourcelen?source[offset+thislen-1]:0))
|
||||
{
|
||||
thislen--;
|
||||
}
|
||||
}
|
||||
|
||||
if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//still worth it?
|
||||
{
|
||||
write24(offset);
|
||||
write16(0);
|
||||
write16(thislen);
|
||||
write8(target[offset]);
|
||||
}
|
||||
else
|
||||
{
|
||||
write24(offset);
|
||||
write16(thislen);
|
||||
int i;
|
||||
for (i=0;i<thislen;i++)
|
||||
{
|
||||
write8(target[offset+i]);
|
||||
}
|
||||
}
|
||||
offset+=thislen;
|
||||
}
|
||||
}
|
||||
write8('E');
|
||||
write8('O');
|
||||
write8('F');
|
||||
if (sourcelen>targetlen) write24(targetlen);
|
||||
#undef write
|
||||
patchmem->ptr=out;
|
||||
patchmem->len=outlen;
|
||||
if (outlen==8) return ips_identical;
|
||||
return ips_ok;
|
||||
}
|
||||
|
||||
void ips_free(struct mem mem)
|
||||
{
|
||||
free(mem.ptr);
|
||||
}
|
||||
|
||||
#if 0
|
||||
#warning Disable this in release versions.
|
||||
#include <stdio.h>
|
||||
|
||||
//Congratulations, you found the undocumented feature! I don't think it's useful for anything except debugging libips, though.
|
||||
void ips_dump(struct mem patch)
|
||||
{
|
||||
if (patch.len<8)
|
||||
{
|
||||
puts("Invalid");
|
||||
return;
|
||||
}
|
||||
const unsigned char * patchat=patch.ptr;
|
||||
const unsigned char * patchend=patchat+patch.len;
|
||||
#define read8() ((patchat<patchend)?(*patchat++):0)
|
||||
#define read16() ((patchat+1<patchend)?(patchat+=2,((patchat[-2]<<8)|patchat[-1])):0)
|
||||
#define read24() ((patchat+2<patchend)?(patchat+=3,((patchat[-3]<<16)|(patchat[-2]<<8)|patchat[-1])):0)
|
||||
if (read8()!='P' ||
|
||||
read8()!='A' ||
|
||||
read8()!='T' ||
|
||||
read8()!='C' ||
|
||||
read8()!='H')
|
||||
{
|
||||
puts("Invalid");
|
||||
return;
|
||||
}
|
||||
int blockstart=patchat-patch.ptr;
|
||||
int offset=read24();
|
||||
int outlen=0;
|
||||
int thisout=0;
|
||||
while (offset!=0x454F46)//454F46=EOF
|
||||
{
|
||||
int size=read16();
|
||||
if (size==0)
|
||||
{
|
||||
int rlelen=read16();
|
||||
thisout=offset+rlelen;
|
||||
printf("[%X] %X: %i (RLE)\n", blockstart, offset, rlelen);
|
||||
read8();
|
||||
}
|
||||
else
|
||||
{
|
||||
thisout=offset+size;
|
||||
printf("[%X] %X: %i\n", blockstart, offset, size);
|
||||
patchat+=size;
|
||||
}
|
||||
if (thisout>outlen) outlen=thisout;
|
||||
if (patchat>=patchend)
|
||||
{
|
||||
puts("Invalid");
|
||||
return;
|
||||
}
|
||||
blockstart=patchat-patch.ptr;
|
||||
offset=read24();
|
||||
}
|
||||
printf("Expand to 0x%X\n", outlen);
|
||||
if (patchat+3==patchend)
|
||||
{
|
||||
int truncate=read24();
|
||||
printf("Truncate to 0x%X\n", truncate);
|
||||
}
|
||||
if (patchat!=patchend) puts("Invalid");
|
||||
#undef read8
|
||||
#undef read16
|
||||
#undef read24
|
||||
}
|
||||
#endif
|
||||
36
patch/libips.h
Normal file
36
patch/libips.h
Normal file
@@ -0,0 +1,36 @@
|
||||
#include "global.h"
|
||||
|
||||
enum ipserror {
|
||||
ips_ok,//Patch applied or created successfully.
|
||||
|
||||
ips_notthis,//The patch is most likely not intended for this ROM.
|
||||
ips_thisout,//You most likely applied the patch on the output ROM.
|
||||
ips_scrambled,//The patch is technically valid, but seems scrambled or malformed.
|
||||
ips_invalid,//The patch is invalid.
|
||||
|
||||
ips_16MB,//One or both files is bigger than 16MB. The IPS format doesn't support that. The created
|
||||
//patch contains only the differences to that point.
|
||||
ips_identical,//The input buffers are identical.
|
||||
|
||||
ips_shut_up_gcc//This one isn't used, it's just to kill a stray comma warning.
|
||||
};
|
||||
|
||||
//Applies the IPS patch in [patch, patchlen] to [in, inlen] and stores it to [out, outlen]. Send the
|
||||
// return value in out to ips_free when you're done with it.
|
||||
enum ipserror ips_apply(struct mem patch, struct mem in, struct mem * out);
|
||||
|
||||
//Creates an IPS patch that converts source to target and stores it to patch.
|
||||
enum ipserror ips_create(struct mem source, struct mem target, struct mem * patch);
|
||||
|
||||
//Frees the memory returned in the output parameters of the above. Do not call it twice on the same
|
||||
// input, nor on anything you got from anywhere else. ips_free is guaranteed to be equivalent to
|
||||
// calling stdlib.h's free() on mem.ptr.
|
||||
void ips_free(struct mem mem);
|
||||
|
||||
//ips_study allows you to detect most patching errors without applying it to a ROM, or even a ROM to
|
||||
// apply it to. ips_apply calls ips_study and ips_apply_study, so if you call ips_study yourself,
|
||||
// it's recommended to call ips_apply_study to not redo the calculation. ips_free is still
|
||||
// required.
|
||||
struct ipsstudy;
|
||||
enum ipserror ips_study(struct mem patch, struct ipsstudy * study);
|
||||
enum ipserror ips_apply_study(struct mem patch, struct ipsstudy * study, struct mem in, struct mem * out);
|
||||
167
patch/libups.cpp
Normal file
167
patch/libups.cpp
Normal file
@@ -0,0 +1,167 @@
|
||||
#include "libups.h"
|
||||
|
||||
#include <stdint.h>//uint8_t, uint32_t
|
||||
#include <stdlib.h>//malloc, realloc, free
|
||||
#include <string.h>//memcpy, memset
|
||||
#include "arlib/crc32.h"
|
||||
|
||||
static uint32_t read32(uint8_t * ptr)
|
||||
{
|
||||
uint32_t out;
|
||||
out =ptr[0];
|
||||
out|=ptr[1]<<8;
|
||||
out|=ptr[2]<<16;
|
||||
out|=ptr[3]<<24;
|
||||
return out;
|
||||
}
|
||||
|
||||
#define error(which) do { error=which; goto exit; } while(0)
|
||||
#define assert_sum(a,b) do { if (SIZE_MAX-(a)<(b)) error(ups_too_big); } while(0)
|
||||
#define assert_shift(a,b) do { if (SIZE_MAX>>(b)<(a)) error(ups_too_big); } while(0)
|
||||
enum upserror ups_apply(struct mem patch, struct mem in, struct mem * out)
|
||||
{
|
||||
enum upserror error;
|
||||
out->len=0;
|
||||
out->ptr=NULL;
|
||||
if (patch.len<4+2+12) return ups_broken;
|
||||
|
||||
if (true)
|
||||
{
|
||||
#define readpatch8() (*(patchat++))
|
||||
#define readin8() (*(inat++))
|
||||
#define writeout8(byte) (*(outat++)=byte)
|
||||
|
||||
#define decodeto(var) \
|
||||
do { \
|
||||
var=0; \
|
||||
unsigned int shift=0; \
|
||||
while (true) \
|
||||
{ \
|
||||
uint8_t next=readpatch8(); \
|
||||
assert_shift(next&0x7F, shift); \
|
||||
size_t addthis=(next&0x7F)<<shift; \
|
||||
assert_sum(var, addthis); \
|
||||
var+=addthis; \
|
||||
if (next&0x80) break; \
|
||||
shift+=7; \
|
||||
assert_sum(var, 1U<<shift); \
|
||||
var+=1<<shift; \
|
||||
} \
|
||||
} while(false)
|
||||
|
||||
bool backwards=false;
|
||||
|
||||
uint8_t * patchat=patch.ptr;
|
||||
uint8_t * patchend=patch.ptr+patch.len-12;
|
||||
|
||||
if (readpatch8()!='U') error(ups_broken);
|
||||
if (readpatch8()!='P') error(ups_broken);
|
||||
if (readpatch8()!='S') error(ups_broken);
|
||||
if (readpatch8()!='1') error(ups_broken);
|
||||
|
||||
size_t inlen;
|
||||
size_t outlen;
|
||||
decodeto(inlen);
|
||||
decodeto(outlen);
|
||||
if (inlen!=in.len)
|
||||
{
|
||||
size_t tmp=inlen;
|
||||
inlen=outlen;
|
||||
outlen=tmp;
|
||||
backwards=true;
|
||||
}
|
||||
if (inlen!=in.len) error(ups_not_this);
|
||||
|
||||
out->len=outlen;
|
||||
out->ptr=(uint8_t*)malloc(outlen);
|
||||
memset(out->ptr, 0, outlen);
|
||||
|
||||
//uint8_t * instart=in.ptr;
|
||||
uint8_t * inat=in.ptr;
|
||||
uint8_t * inend=in.ptr+in.len;
|
||||
|
||||
//uint8_t * outstart=out->ptr;
|
||||
uint8_t * outat=out->ptr;
|
||||
uint8_t * outend=out->ptr+out->len;
|
||||
|
||||
while (patchat<patchend)
|
||||
{
|
||||
size_t skip;
|
||||
decodeto(skip);
|
||||
while (skip>0)
|
||||
{
|
||||
uint8_t out;
|
||||
if (inat>=inend) out=0;
|
||||
else out=readin8();
|
||||
if (outat<outend) writeout8(out);
|
||||
skip--;
|
||||
}
|
||||
uint8_t tmp;
|
||||
do
|
||||
{
|
||||
tmp=readpatch8();
|
||||
uint8_t out;
|
||||
if (inat>=inend) out=0;
|
||||
else out=readin8();
|
||||
if (outat<outend) writeout8(out^tmp);
|
||||
}
|
||||
while (tmp);
|
||||
}
|
||||
if (patchat!=patchend) error(ups_broken);
|
||||
while (outat<outend) writeout8(0);
|
||||
while (inat<inend) (void)readin8();
|
||||
|
||||
uint32_t crc_in_expected=read32(patchat);
|
||||
uint32_t crc_out_expected=read32(patchat+4);
|
||||
uint32_t crc_patch_expected=read32(patchat+8);
|
||||
|
||||
uint32_t crc_in=crc32(in.ptr, in.len);
|
||||
uint32_t crc_out=crc32(out->ptr, out->len);
|
||||
uint32_t crc_patch=crc32(patch.ptr, patch.len-4);
|
||||
|
||||
if (inlen==outlen)
|
||||
{
|
||||
if ((crc_in!=crc_in_expected || crc_out!=crc_out_expected) && (crc_in!=crc_out_expected || crc_out!=crc_in_expected)) error(ups_not_this);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!backwards)
|
||||
{
|
||||
if (crc_in!=crc_in_expected) error(ups_not_this);
|
||||
if (crc_out!=crc_out_expected) error(ups_not_this);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (crc_in!=crc_out_expected) error(ups_not_this);
|
||||
if (crc_out!=crc_in_expected) error(ups_not_this);
|
||||
}
|
||||
}
|
||||
if (crc_patch!=crc_patch_expected) error(ups_broken);
|
||||
return ups_ok;
|
||||
#undef read8
|
||||
#undef decodeto
|
||||
#undef write8
|
||||
}
|
||||
|
||||
exit:
|
||||
free(out->ptr);
|
||||
out->len=0;
|
||||
out->ptr=NULL;
|
||||
return error;
|
||||
}
|
||||
|
||||
enum upserror ups_create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem)
|
||||
{
|
||||
patchmem->ptr=NULL;
|
||||
patchmem->len=0;
|
||||
return ups_broken;//unimplemented, just pick a random error
|
||||
}
|
||||
|
||||
void ups_free(struct mem mem)
|
||||
{
|
||||
free(mem.ptr);
|
||||
}
|
||||
|
||||
#if 0
|
||||
//Sorry, no undocumented features here. The only thing that can change an UPS patch is swapping the two sizes and checksums, and I don't create anyways.
|
||||
#endif
|
||||
30
patch/libups.h
Normal file
30
patch/libups.h
Normal file
@@ -0,0 +1,30 @@
|
||||
#include "global.h"
|
||||
|
||||
//Several of those are unused, but remain there so the remaining ones match bpserror.
|
||||
enum upserror {
|
||||
ups_ok,//Patch applied or created successfully.
|
||||
|
||||
ups_unused1, //bps_to_output
|
||||
ups_not_this,//This is not the intended input file for this patch.
|
||||
ups_broken, //This is not a UPS patch, or it's malformed somehow.
|
||||
ups_unused2, //bps_io
|
||||
|
||||
ups_identical,//The input files are identical.
|
||||
ups_too_big, //Somehow, you're asking for something a size_t can't represent.
|
||||
ups_unused3, //bps_out_of_mem
|
||||
ups_unused4, //bps_canceled
|
||||
|
||||
ups_shut_up_gcc//This one isn't used, it's just to kill a stray comma warning.
|
||||
};
|
||||
|
||||
//Applies the UPS patch in [patch, patchlen] to [in, inlen] and stores it to [out, outlen]. Send the
|
||||
// return value in out to ups_free when you're done with it.
|
||||
enum upserror ups_apply(struct mem patch, struct mem in, struct mem * out);
|
||||
|
||||
//Creates an UPS patch that converts source to target and stores it to patch. (Not implemented.)
|
||||
enum upserror ups_create(struct mem source, struct mem target, struct mem * patch);
|
||||
|
||||
//Frees the memory returned in the output parameters of the above. Do not call it twice on the same
|
||||
// input, nor on anything you got from anywhere else. ups_free is guaranteed to be equivalent to
|
||||
// calling stdlib.h's free() on mem.ptr.
|
||||
void ups_free(struct mem mem);
|
||||
Reference in New Issue
Block a user