Leahnaya/Flips
1
0
Fork 0
mirror of https://github.com/Alcaro/Flips.git synced 2026-09-07 10:06:09 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add test suite, fix about five IPS bugs found

Browse Source
This commit is contained in:
Alcaro
2016-12-23 13:13:21 +01:00
parent 81a35ee437
commit 3f3bf6c3a5
10 changed files with 381 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
patch/ips.cpp
View File

@@ -181,8 +181,8 @@ result apply(const file& patch, const file& source, file& target)
static result create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem)
{
unsigned int sourcelen=sourcemem.len;
unsigned int targetlen=targetmem.len;
int sourcelen=sourcemem.len;
int targetlen=targetmem.len;
const unsigned char * source=sourcemem.ptr;
const unsigned char * target=targetmem.ptr;
@@ -191,10 +191,10 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
if (targetlen>=16777216) return e_too_big;
unsigned int offset=0;
unsigned int outbuflen=4096;
int offset=0;
int outbuflen=4096;
unsigned char * out=(uint8_t*)malloc(outbuflen);
unsigned int outlen=0;
int outlen=0;
#define write8(val) do { out[outlen++]=(val); if (outlen==outbuflen) { outbuflen*=2; out=(uint8_t*)realloc(out, outbuflen); } } while(0)
#define write16(val) do { write8((val)>>8); write8((val)); } while(0)
#define write24(val) do { write8((val)>>16); write8((val)>>8); write8((val)); } while(0)
@@ -204,10 +204,11 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
write8('C');
write8('H');
int lastknownchange=0;
int lastwritten=0;
//int forcewrite=(targetlen>sourcelen?1:0);
while (offset<targetlen)
{
while (offset<sourcelen && (offset<sourcelen?source[offset]:0)==target[offset]) offset++;
while (offset<targetlen && (offset<sourcelen?source[offset]:0)==target[offset]) offset++;
//check how much we need to edit until it starts getting similar
int thislen=0;
int consecutiveunchanged=0;
@@ -215,14 +216,14 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
if (thislen<0) thislen=0;
while (true)
{
unsigned int thisbyte=offset+thislen+consecutiveunchanged;
if (thisbyte<sourcelen && (thisbyte<sourcelen?source[thisbyte]:0)==target[thisbyte]) consecutiveunchanged++;
int thisbyte=offset+thislen+consecutiveunchanged;
if (thisbyte<targetlen && (thisbyte<sourcelen?source[thisbyte]:0)==target[thisbyte]) consecutiveunchanged++;
else
{
thislen+=consecutiveunchanged+1;
consecutiveunchanged=0;
}
if (consecutiveunchanged>=6 || thislen>=65536) break;
if (consecutiveunchanged>=6 || thislen>65535) break;
}
//avoid premature EOF
@@ -246,7 +247,7 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
int i=0;
while (true)
{
unsigned int pos=offset+byteshere+i-1;
int pos=offset+byteshere+i-1;
if (pos>=targetlen || target[pos]!=thisbyte || byteshere+i>65535) break;
if (pos>=sourcelen || (pos<sourcelen?source[pos]:0)!=thisbyte)
{
@@ -264,6 +265,7 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
write16(byteshere);
write8(target[offset]);
offset+=byteshere;
lastwritten=offset;
}
else
{
@@ -280,7 +282,8 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
}
if (byteshere>8+5 || //rle-worthy despite two ips headers
(byteshere>8 && stopat+byteshere==thislen) || //rle-worthy at end of data
(byteshere>8 && !memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy
(byteshere>8 && offset+stopat+byteshere+8 <= thislen &&
!memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy
{
if (stopat) thislen=stopat;
break;//we don't scan the entire block if we know we'll want to RLE, that'd gain nothing.
@@ -295,6 +298,12 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
}
}
//avoid infinite loops if an RLEable block starts at 'EOF'
if (offset+thislen == 0x454F46)
{
if (thislen==0xFFFF) thislen--;
else thislen++;
}
if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//still worth it?
{
write24(offset);
@@ -306,13 +315,29 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
{
write24(offset);
write16(thislen);
int i;
for (i=0;i<thislen;i++)
for (int i=0;i<thislen;i++)
{
write8(target[offset+i]);
}
}
offset+=thislen;
lastwritten=offset;
}
}
if (sourcelen<targetlen && lastwritten!=targetlen)
{
if (targetlen-1==0x454F46)
{
write24(targetlen-2);
write16(2);
write8(target[targetlen-2]);
write8(target[targetlen-1]);
}
else
{
write24(targetlen-1);
write16(1);
write8(target[targetlen-1]);
}
}
write8('E');

124
patch/patch.h
View File

@@ -1,3 +1,4 @@
#pragma once
#include "../arlib.h"
namespace patch {
@@ -27,17 +28,24 @@ enum result {
namespace ips {
result apply(const file& patch, const file& source, file& target);
static inline result apply(const file& patch, const file& source, file&& target) { return apply(patch, source, (file&)target); }
result create(const file& source, const file& target, file& patch);
static inline result create(const file& source, const file& target, file&& patch) { return create(source, target, (file&)patch); }
}
namespace ups {
result apply(const file& patch, const file& source, file& target);
static inline result apply(const file& patch, const file& source, file&& target) { return apply(patch, source, (file&)target); }
//ups is worthless
//result create(const file& source, const file& target, file& patch);
}
namespace bps {
result apply(const file& patch, const file& source, file& target, bool accept_wrong_input);
result apply(const file& patch, const file& source, file& target, bool accept_wrong_input = false);
static inline result apply(const file& patch, const file& source, file&& target, bool accept_wrong_input = false)
{
return apply(patch, source, (file&)target, accept_wrong_input);
}
//Because this one can take quite a long time, a progress meter is supplied. total is guaranteed to
// be constant between every call until this function returns, done is guaranteed to increase
// between each call, and done/total is an approximate percentage counter. Anything else is
@@ -47,6 +55,21 @@ result apply(const file& patch, const file& source, file& target, bool accept_wr
//To cancel patch creation, return true from the callback. It's safe to pass in NULL if you're not interested.
result create(const file& source, const file& target, const file& metadata, file& patch,
function<bool(size_t done, size_t total)> progress);
static inline result create(const file& source, const file& target, const file& metadata, file&& patch,
function<bool(size_t done, size_t total)> progress)
{
return create(source, target, metadata, (file&)patch, progress);
}
static inline result create(const file& source, const file& target, file& patch,
function<bool(size_t done, size_t total)> progress)
{
return create(source, target, file::mem(NULL), (file&)patch, progress);
}
static inline result create(const file& source, const file& target, file&& patch,
function<bool(size_t done, size_t total)> progress)
{
return create(source, target, (file&)patch, progress);
}
struct info {
result parse(const file& patch, bool changefrac = false);
@@ -71,6 +94,105 @@ struct info {
};
}
//Used for patch application.
class filebufreader {
file& f;
size_t fpos;
array<byte> buf;
size_t bufpos;
uint32_t crc;
public:
filebufreader(file& f) : f(f), fpos(0), bufpos(0), crc(0) {}
arrayview<byte> peek(size_t bytes)
{
if (buf.size()-bufpos < bytes)
{
buf = buf.slice(bufpos, buf.size()-bufpos);
bufpos = 0;
size_t bytehave = buf.size();
size_t byteread = bytes + 4096;
buf.resize(bytehave + byteread);
byteread = f.read(buf.slice(bytehave, byteread), fpos);
fpos += byteread;
buf.resize(bytehave + byteread);
}
return buf.slice(bufpos, min(buf.size()-bufpos, bytes));
}
arrayview<byte> read(size_t bytes)
{
arrayview<byte> ret = peek(bytes);
if (ret.size() != bytes) return NULL;
bufpos += bytes;
crc = crc32_update(ret, crc); // TODO: perhaps it's faster if this one is calculated in large batches
return ret;
}
byte read() { return read(1)[0]; }
size_t remaining() { return buf.size()-bufpos + f.size()-fpos; }
uint32_t crc32() { return crc; }
};
class streamreader {
filebufreader f;
public:
streamreader(file& f) : f(f) {}
arrayview<byte> bytes(size_t n) { return f.read(n); }
uint8_t u8()
{
return f.read(1)[0];
}
uint16_t u16()
{
arrayview<byte> b = f.read(2);
return b[0] | b[1]<<8;
}
uint32_t u24()
{
arrayview<byte> b = f.read(3);
return b[0] | b[1]<<8 | b[2]<<16;
}
uint32_t u32()
{
arrayview<byte> b = f.read(4);
return b[0] | b[1]<<8 | b[2]<<16 | b[3]<<24;
}
// size_t bpsnum() // close to uleb128, but uleb lacks the +1 that ensures there's only one way to encode an integer
// {
// size_t ret = 0;
// size_t shift = 0;
// while (true)
// {
// uint8_t next = f.read();
// if (SIZE_MAX>>shift < (next&0x7F)) return (size_t)-1;
// size_t shifted = (next&0x7F)<<shift;
//
//#define assert_sum(a,b) do { if (SIZE_MAX-(a)<(b)) error(e_too_big); } while(0)
//#define assert_shift(a,b) do { if (SIZE_MAX>>(b)<(a)) error(e_too_big); } while(0)
//
// }
//#define decodeto(var) \
// do { \
// var=0; \
// unsigned int shift=0; \
// while (true) \
// { \
// uint8_t next=readpatch8(); \
// assert_shift(next&0x7F, shift); \
// size_t addthis=(next&0x7F)<<shift; \
// assert_sum(var, addthis); \
// var+=addthis; \
// if (next&0x80) break; \
// shift+=7; \
// assert_sum(var, 1U<<shift); \
// var+=1<<shift; \
// } \
// } while(false)
//
// arrayview<byte> b = f.peek(16);
// }
};
//Deprecated
struct mem {
mem() : ptr(NULL), len(0) {}

184
patch/test.cpp Normal file
View File

@@ -0,0 +1,184 @@
#include "patch.h"
namespace patch {
test("filebufreader")
{
array<byte> bytes;
for (int i=0;i<65536;i++)
{
bytes[i] = i ^ i>>8;
}
file f = file::mem(bytes);
assert_eq(f.size(), 65536);
filebufreader br = f;
size_t pos = 0;
#define EXPECT(n) \
do { \
assert_eq(br.remaining(), 65536-pos); \
arrayview<byte> var = br.read(n); \
for (size_t i=0;i<n;i++) \
assert_eq(var[i], bytes[pos+i]); \
pos += n; \
assert_eq(br.remaining(), 65536-pos); \
assert_eq(br.crc32(), crc32(bytes.slice(0, pos))); \
} while(0)
EXPECT(1);
EXPECT(6);
EXPECT(14);
EXPECT(4000);
EXPECT(4000); // cross the buffers
assert_eq(br.read(), bytes[pos++]); // single-byte reader
assert_eq(br.read(), bytes[pos++]);
assert_eq(br.read(), bytes[pos++]);
assert_eq(br.read(), bytes[pos++]);
EXPECT(16000);
assert(br.read(65536).ptr() == NULL);
EXPECT(4000);
}
test("streamreader")
{
array<byte> bytes;
for (int i=0;i<65536;i++)
{
bytes[i] = i ^ i>>8;
}
file f = file::mem(bytes);
assert_eq(f.size(), 65536);
streamreader r = f;
test_skip("not yet");
}
static bool testips;
static bool testbps;
static void createtest(arrayview<byte> a, arrayview<byte> b, size_t ipssize, size_t bpssize)
{
if (testips && b.size()<=16777216)
{
array<byte> patch;
ips::create(file::mem(a), file::mem(b), file::mem(patch)); // don't worry about return value
array<byte> b2;
result r = ips::apply(file::mem(patch), file::mem(a), file::mem(b2));
if (r!=e_to_output) assert_eq(r, e_ok);
assert_eq(b2.size(), b.size());
for (size_t i=0;i<b.size();i++) assert_eq(b[i], b2[i]);
//ensure no accidental creation size regressions - or improving it without moving the goalposts
if (patch.size()!=ipssize)
{
for(byte g:patch)printf("%.2X ",g);
}
assert_eq(patch.size(), ipssize);
}
if (testbps)
{
array<byte> patch;
bps::create(file::mem(a), file::mem(b), file::mem(patch), NULL); // don't worry about return value
array<byte> b2;
bps::apply(file::mem(patch), file::mem(a), file::mem(b2));
assert_eq(b2.size(), b.size());
for (size_t i=0;i<b.size();i++) assert_eq(b[i], b2[i]);
//if (patch.size()!=bpssize)
//{
//for(byte g:patch)printf("%.2X ",g);
//}
assert_eq(patch.size(), bpssize);
}
}
static void simpletests()
{
array<byte> empty;
array<byte> one0; one0[0] = 0;
array<byte> one1; one1[0] = 1;
array<byte> seq128; for (int i=0;i<128;i++) seq128[i]=i;
array<byte> seq256; for (int i=0;i<256;i++) seq256[i]=i;
array<byte> seq256nul4; for (int i=0;i<256;i++) seq256nul4[i]=i; seq256nul4[255+4]=0;
array<byte> seq256nul5; for (int i=0;i<256;i++) seq256nul5[i]=i; seq256nul5[255+5]=0;
array<byte> seq256nul6; for (int i=0;i<256;i++) seq256nul6[i]=i; seq256nul6[255+6]=0;
array<byte> seq256nul7; for (int i=0;i<256;i++) seq256nul7[i]=i; seq256nul7[255+7]=0;
array<byte> seq256b4; for (int i=0;i<256;i++) seq256b4[i]=i; seq256b4[255+4]=1;
array<byte> seq256b5; for (int i=0;i<256;i++) seq256b5[i]=i; seq256b5[255+5]=1;
array<byte> seq256b6; for (int i=0;i<256;i++) seq256b6[i]=i; seq256b6[255+6]=1;
array<byte> seq256b7; for (int i=0;i<256;i++) seq256b7[i]=i; seq256b7[255+7]=1;
array<byte> eof; eof[0x454F46] = 1;
array<byte> eof2; for (int i=0;i<16;i++) eof2[0x454F46+i] = 1;
array<byte> eof3; eof3[0x454F46] = 0;
array<byte> eof4; eof4[0x454F45] = 2; eof4[0x454F46] = 1;
array<byte> eof5; eof5[0x454F45] = 2; for (int i=0;i<16;i++) eof5[0x454F46+i] = 1;
array<byte> eof6; eof6[0x454F45] = 2; eof6[0x454F46] = 0;
int base = 5+3; // PATCHEOF
int record = 3+2; // offset, len
int rle = 3+2+2+1; // offset, len=0, rlelen, byte
int trunc = 3;
testcall(createtest(empty, empty, base, 0));
testcall(createtest(empty, one1, base+record+1, 0));
testcall(createtest(one1, empty, base+trunc, 0));
testcall(createtest(one0, one1, base+record+1, 0));
testcall(createtest(seq256, seq128, base+trunc, 0));
testcall(createtest(seq128, seq256, base+record+128, 0));
testcall(createtest(empty, seq256nul4, base+record+255+4, 0));
testcall(createtest(empty, seq256nul5, base+record+255+5, 0));
testcall(createtest(empty, seq256nul6, base+record+255+6, 0));
testcall(createtest(empty, seq256nul7, base+record+255+record+1, 0));
testcall(createtest(empty, seq256b4, base+record+255+4, 0));
testcall(createtest(empty, seq256b5, base+record+255+5, 0));
testcall(createtest(empty, seq256b6, base+record+255+6, 0));
testcall(createtest(empty, seq256b7, base+record+255+record+1, 0));
testcall(createtest(empty, eof, base+record+2, 0));
testcall(createtest(empty, eof2, base+record+2+rle, 0));
testcall(createtest(empty, eof3, base+record+2, 0));
}
test("IPS")
{
testips=true;
testbps=false;
simpletests();
}
test("BPS")
{
//test_skip("fix ips first");
testips=false;
testbps=true;
simpletests();
}
test("the big ones")
{
testips=true;
testbps=true;
test_skip("enable this whenever ips/bps tests are done (also this is sufficient ups test)");
array<byte> smw = file::read("patch/test/smw.sfc");
array<byte> smw_bps = file::read("patch/test/smwcp.bps");
array<byte> dl = file::read("patch/test/langrisser.sfc");
array<byte> dl_ups = file::read("patch/test/dl.ups");
array<byte> sm64 = file::read("patch/test/sm64.z64");
array<byte> sm64_bps = file::read("patch/test/star.bps");
if (!smw || !smw_bps || !dl || !dl_ups || !sm64 || !sm64_bps) test_skip("test files not present; see patch/test/readme.txt");
array<byte> smwhack;
bps::apply(file::mem(smw_bps), file::mem(smw), file::mem(smwhack));
testcall(createtest(smw, smwhack, 3328022, 2077386));
array<byte> sm64hack;
bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack));
testcall(createtest(sm64, sm64hack, -1, 0));
array<byte> dlhack;
ups::apply(file::mem(dl_ups), file::mem(dl), file::mem(dlhack));
testcall(createtest(dl, dlhack, 0, 0));
}
}

9
patch/test/readme.txt Normal file
View File

@@ -0,0 +1,9 @@
These files are copyrighted, large, and not very relevant to Floating IPS. You'll have to fill them in yourself.
MD5 checksums and sources:
8134163af1e658a3f1af130afd66d17a dl.ups http://superfamicom.org/translations/info/der-langrisser-english
91d62c4cb790fc2fb38b10b68616e228 langrisser.sfc
20b854b239203baf6c961b850a4a51a2 sm64.z64
b38466ee8ae3130f3cf1c94f9c7655c0 smwcp.bps https://smwc.me/s/5420
cdd3c8c37322978ca8669b34bc89c804 smw.sfc
461ae37a8c9d3a4e1990c9b8907b716f star.bps https://smwc.me/s/13353