From 3f3bf6c3a53b82b7efffcb179b35e547e1ce4de0 Mon Sep 17 00:00:00 2001 From: Alcaro Date: Fri, 23 Dec 2016 13:13:21 +0100 Subject: [PATCH] Add test suite, fix about five IPS bugs found --- Makefile | 1 - arlib/Makefile | 4 +- arlib/arlib.h | 2 +- arlib/test.cpp | 24 ++++-- arlib/test.h | 8 +- flips.cpp | 1 + patch/ips.cpp | 51 +++++++++--- patch/patch.h | 124 +++++++++++++++++++++++++++- patch/test.cpp | 184 ++++++++++++++++++++++++++++++++++++++++++ patch/test/readme.txt | 9 +++ 10 files changed, 381 insertions(+), 27 deletions(-) create mode 100644 patch/test.cpp create mode 100644 patch/test/readme.txt diff --git a/Makefile b/Makefile index 69b204a..06e14be 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ PROGRAM = flips ARGUI = 1 ARWUTF = 1 -EXTRAOBJ += obj/divsufsort-c$(OBJSUFFIX).o SOURCES += patch/*.cpp DOMAINS += LDSS diff --git a/arlib/Makefile b/arlib/Makefile index 4934b5d..a600055 100644 --- a/arlib/Makefile +++ b/arlib/Makefile @@ -41,7 +41,7 @@ CXXFLAGS = $(CFLAGS) LD = g++ LFLAGS = OBJNAME = -CCXXFLAGS = -fvisibility=hidden -Wall +CCXXFLAGS = -fvisibility=hidden -Wall -Og ifneq ($(EXCEPTIONS),1) CCXXFLAGS += -fno-exceptions endif @@ -137,8 +137,8 @@ ifneq ($(SELFTEST),) ifeq ($(ARLIB_MAIN),) CONF_CFLAGS += -DARLIB_TEST -Dmain=not_quite_main CFLAGS_ARLIB += -UARLIB_TEST -DARLIB_TESTRUNNER - DOMAINS := else + DOMAINS := SOURCES := CFLAGS_ARLIB += -DARLIB_TEST -DARLIB_TESTRUNNER -DARLIB_TEST_ARLIB endif diff --git a/arlib/arlib.h b/arlib/arlib.h index 8386c98..90dd3c9 100644 --- a/arlib/arlib.h +++ b/arlib/arlib.h @@ -26,7 +26,7 @@ #include "test.h" #include "zip.h" -//not in #ifdef, it contains some dummy implementations if threads are disabled +//no #ifdef, it contains some dummy implementations if threads are disabled #include "thread/thread.h" #if !defined(ARGUI_NONE) && !defined(ARGUI_WINDOWS) && !defined(ARGUI_GTK3) diff --git a/arlib/test.cpp b/arlib/test.cpp index e96e08c..8d5ed1c 100644 --- a/arlib/test.cpp +++ b/arlib/test.cpp @@ -25,7 +25,7 @@ _testdecl::_testdecl(void(*func)(), const char * loc, const char * name) g_testlist = next; } -static bool thisfail; +int _test_result; static array callstack; void _teststack_push(int line) { callstack.append(line); } @@ -44,8 +44,8 @@ static string stack(int top) static void _testfail(cstring why) { - if (!thisfail) puts(why); // discard multiple failures from same test, they're probably caused by same thing - thisfail = true; + if (!_test_result) puts(why); // discard multiple failures from same test, they're probably caused by same thing + _test_result = 1; } void _testfail(cstring why, int line) @@ -65,6 +65,12 @@ void _testeqfail(cstring name, int line, cstring expected, cstring actual) } } +void _test_skip(cstring why) +{ + if (!_test_result) puts("skipped: "+why); + _test_result = 2; +} + #undef main // the real main is #define'd to something stupid on test runs int main(int argc, char* argv[]) { @@ -74,7 +80,7 @@ int main(int argc, char* argv[]) _window_init_file(); #endif - int count[2]={0,0}; + int count[3]={0,0,0}; //flip list backwards //order of static initializers is implementation defined, but this makes output better under gcc @@ -95,15 +101,17 @@ int main(int argc, char* argv[]) if (test->name) printf("Testing %s (%s)...", test->name, test->loc); else printf("Testing %s...", test->loc); fflush(stdout); - thisfail = false; + _test_result = 0; callstack.reset(); test->func(); - count[thisfail]++; - if (!thisfail) puts(" pass"); + count[_test_result]++; + if (!_test_result) puts(" pass"); free(test); test = next; } - printf("Passed %i, failed %i\n", count[0], count[1]); + printf("Passed %i, failed %i", count[0], count[1]); + if (count[2]) printf(", skipped %i", count[2]); + puts(""); return 0; } diff --git a/arlib/test.h b/arlib/test.h index 756fb02..ff00a0c 100644 --- a/arlib/test.h +++ b/arlib/test.h @@ -19,12 +19,16 @@ public: _testdecl(void(*func)(), const char * loc, const char * name); }; +extern int _test_result; + void _testfail(cstring why, int line); void _testeqfail(cstring why, int line, cstring expected, cstring actual); void _teststack_push(int line); void _teststack_pop(); +void _test_skip(cstring why); + #define TESTFUNCNAME JOIN(_testfunc, __LINE__) #define test(...) \ static void TESTFUNCNAME(); \ @@ -41,7 +45,8 @@ void _teststack_pop(); return; \ } \ } while(0) -#define testcall(x) _teststack_push(__LINE__),x,_teststack_pop() +#define testcall(x) do { _teststack_push(__LINE__); x; _teststack_pop(); if (_test_result) return; } while(0) +#define test_skip(x) do { _test_skip(x); return; } while(0) #else @@ -49,5 +54,6 @@ void _teststack_pop(); #define assert(x) #define assert_eq(x,y) #define testcall(x) x +#define test_skip(x) return #endif diff --git a/flips.cpp b/flips.cpp index d2d3513..edba54d 100644 --- a/flips.cpp +++ b/flips.cpp @@ -145,4 +145,5 @@ int main(int argc, char* argv[]) wnd->set_title("Flips v" FLIPSVER); wnd->set_visible(true); while (wnd->is_visible()) window_run_wait(); + return 0; } diff --git a/patch/ips.cpp b/patch/ips.cpp index f542404..bbe2ecb 100644 --- a/patch/ips.cpp +++ b/patch/ips.cpp @@ -181,8 +181,8 @@ result apply(const file& patch, const file& source, file& target) static result create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem) { - unsigned int sourcelen=sourcemem.len; - unsigned int targetlen=targetmem.len; + int sourcelen=sourcemem.len; + int targetlen=targetmem.len; const unsigned char * source=sourcemem.ptr; const unsigned char * target=targetmem.ptr; @@ -191,10 +191,10 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa if (targetlen>=16777216) return e_too_big; - unsigned int offset=0; - unsigned int outbuflen=4096; + int offset=0; + int outbuflen=4096; unsigned char * out=(uint8_t*)malloc(outbuflen); - unsigned int outlen=0; + int outlen=0; #define write8(val) do { out[outlen++]=(val); if (outlen==outbuflen) { outbuflen*=2; out=(uint8_t*)realloc(out, outbuflen); } } while(0) #define write16(val) do { write8((val)>>8); write8((val)); } while(0) #define write24(val) do { write8((val)>>16); write8((val)>>8); write8((val)); } while(0) @@ -204,10 +204,11 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa write8('C'); write8('H'); int lastknownchange=0; + int lastwritten=0; //int forcewrite=(targetlen>sourcelen?1:0); while (offset=6 || thislen>=65536) break; + if (consecutiveunchanged>=6 || thislen>65535) break; } //avoid premature EOF @@ -246,7 +247,7 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa int i=0; while (true) { - unsigned int pos=offset+byteshere+i-1; + int pos=offset+byteshere+i-1; if (pos>=targetlen || target[pos]!=thisbyte || byteshere+i>65535) break; if (pos>=sourcelen || (pos8+5 || //rle-worthy despite two ips headers (byteshere>8 && stopat+byteshere==thislen) || //rle-worthy at end of data - (byteshere>8 && !memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy + (byteshere>8 && offset+stopat+byteshere+8 <= thislen && + !memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy { if (stopat) thislen=stopat; break;//we don't scan the entire block if we know we'll want to RLE, that'd gain nothing. @@ -295,6 +298,12 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa } } + //avoid infinite loops if an RLEable block starts at 'EOF' + if (offset+thislen == 0x454F46) + { + if (thislen==0xFFFF) thislen--; + else thislen++; + } if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//still worth it? { write24(offset); @@ -306,13 +315,29 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa { write24(offset); write16(thislen); - int i; - for (i=0;i progress); +static inline result create(const file& source, const file& target, const file& metadata, file&& patch, + function progress) +{ + return create(source, target, metadata, (file&)patch, progress); +} +static inline result create(const file& source, const file& target, file& patch, + function progress) +{ + return create(source, target, file::mem(NULL), (file&)patch, progress); +} +static inline result create(const file& source, const file& target, file&& patch, + function progress) +{ + return create(source, target, (file&)patch, progress); +} struct info { result parse(const file& patch, bool changefrac = false); @@ -71,6 +94,105 @@ struct info { }; } +//Used for patch application. +class filebufreader { + file& f; + size_t fpos; + + array buf; + size_t bufpos; + + uint32_t crc; + +public: + filebufreader(file& f) : f(f), fpos(0), bufpos(0), crc(0) {} + arrayview peek(size_t bytes) + { + if (buf.size()-bufpos < bytes) + { + buf = buf.slice(bufpos, buf.size()-bufpos); + bufpos = 0; + size_t bytehave = buf.size(); + size_t byteread = bytes + 4096; + buf.resize(bytehave + byteread); + byteread = f.read(buf.slice(bytehave, byteread), fpos); + fpos += byteread; + buf.resize(bytehave + byteread); + } + return buf.slice(bufpos, min(buf.size()-bufpos, bytes)); + } + arrayview read(size_t bytes) + { + arrayview ret = peek(bytes); + if (ret.size() != bytes) return NULL; + bufpos += bytes; + crc = crc32_update(ret, crc); // TODO: perhaps it's faster if this one is calculated in large batches + return ret; + } + byte read() { return read(1)[0]; } + size_t remaining() { return buf.size()-bufpos + f.size()-fpos; } + uint32_t crc32() { return crc; } +}; +class streamreader { + filebufreader f; +public: + streamreader(file& f) : f(f) {} + arrayview bytes(size_t n) { return f.read(n); } + uint8_t u8() + { + return f.read(1)[0]; + } + uint16_t u16() + { + arrayview b = f.read(2); + return b[0] | b[1]<<8; + } + uint32_t u24() + { + arrayview b = f.read(3); + return b[0] | b[1]<<8 | b[2]<<16; + } + uint32_t u32() + { + arrayview b = f.read(4); + return b[0] | b[1]<<8 | b[2]<<16 | b[3]<<24; + } +// size_t bpsnum() // close to uleb128, but uleb lacks the +1 that ensures there's only one way to encode an integer +// { +// size_t ret = 0; +// size_t shift = 0; +// while (true) +// { +// uint8_t next = f.read(); +// if (SIZE_MAX>>shift < (next&0x7F)) return (size_t)-1; +// size_t shifted = (next&0x7F)<>(b)<(a)) error(e_too_big); } while(0) +// +// } +//#define decodeto(var) \ +// do { \ +// var=0; \ +// unsigned int shift=0; \ +// while (true) \ +// { \ +// uint8_t next=readpatch8(); \ +// assert_shift(next&0x7F, shift); \ +// size_t addthis=(next&0x7F)< b = f.peek(16); +// } +}; + //Deprecated struct mem { mem() : ptr(NULL), len(0) {} diff --git a/patch/test.cpp b/patch/test.cpp new file mode 100644 index 0000000..239c0d5 --- /dev/null +++ b/patch/test.cpp @@ -0,0 +1,184 @@ +#include "patch.h" + +namespace patch { +test("filebufreader") +{ + array bytes; + for (int i=0;i<65536;i++) + { + bytes[i] = i ^ i>>8; + } + file f = file::mem(bytes); + assert_eq(f.size(), 65536); + + filebufreader br = f; + + size_t pos = 0; +#define EXPECT(n) \ + do { \ + assert_eq(br.remaining(), 65536-pos); \ + arrayview var = br.read(n); \ + for (size_t i=0;i bytes; + for (int i=0;i<65536;i++) + { + bytes[i] = i ^ i>>8; + } + file f = file::mem(bytes); + assert_eq(f.size(), 65536); + + streamreader r = f; + test_skip("not yet"); +} + +static bool testips; +static bool testbps; +static void createtest(arrayview a, arrayview b, size_t ipssize, size_t bpssize) +{ + if (testips && b.size()<=16777216) + { + array patch; + ips::create(file::mem(a), file::mem(b), file::mem(patch)); // don't worry about return value + array b2; + result r = ips::apply(file::mem(patch), file::mem(a), file::mem(b2)); + if (r!=e_to_output) assert_eq(r, e_ok); + assert_eq(b2.size(), b.size()); + for (size_t i=0;i patch; + bps::create(file::mem(a), file::mem(b), file::mem(patch), NULL); // don't worry about return value + array b2; + bps::apply(file::mem(patch), file::mem(a), file::mem(b2)); + assert_eq(b2.size(), b.size()); + for (size_t i=0;i empty; + array one0; one0[0] = 0; + array one1; one1[0] = 1; + array seq128; for (int i=0;i<128;i++) seq128[i]=i; + array seq256; for (int i=0;i<256;i++) seq256[i]=i; + array seq256nul4; for (int i=0;i<256;i++) seq256nul4[i]=i; seq256nul4[255+4]=0; + array seq256nul5; for (int i=0;i<256;i++) seq256nul5[i]=i; seq256nul5[255+5]=0; + array seq256nul6; for (int i=0;i<256;i++) seq256nul6[i]=i; seq256nul6[255+6]=0; + array seq256nul7; for (int i=0;i<256;i++) seq256nul7[i]=i; seq256nul7[255+7]=0; + array seq256b4; for (int i=0;i<256;i++) seq256b4[i]=i; seq256b4[255+4]=1; + array seq256b5; for (int i=0;i<256;i++) seq256b5[i]=i; seq256b5[255+5]=1; + array seq256b6; for (int i=0;i<256;i++) seq256b6[i]=i; seq256b6[255+6]=1; + array seq256b7; for (int i=0;i<256;i++) seq256b7[i]=i; seq256b7[255+7]=1; + array eof; eof[0x454F46] = 1; + array eof2; for (int i=0;i<16;i++) eof2[0x454F46+i] = 1; + array eof3; eof3[0x454F46] = 0; + array eof4; eof4[0x454F45] = 2; eof4[0x454F46] = 1; + array eof5; eof5[0x454F45] = 2; for (int i=0;i<16;i++) eof5[0x454F46+i] = 1; + array eof6; eof6[0x454F45] = 2; eof6[0x454F46] = 0; + + int base = 5+3; // PATCHEOF + int record = 3+2; // offset, len + int rle = 3+2+2+1; // offset, len=0, rlelen, byte + int trunc = 3; + testcall(createtest(empty, empty, base, 0)); + testcall(createtest(empty, one1, base+record+1, 0)); + testcall(createtest(one1, empty, base+trunc, 0)); + testcall(createtest(one0, one1, base+record+1, 0)); + testcall(createtest(seq256, seq128, base+trunc, 0)); + testcall(createtest(seq128, seq256, base+record+128, 0)); + testcall(createtest(empty, seq256nul4, base+record+255+4, 0)); + testcall(createtest(empty, seq256nul5, base+record+255+5, 0)); + testcall(createtest(empty, seq256nul6, base+record+255+6, 0)); + testcall(createtest(empty, seq256nul7, base+record+255+record+1, 0)); + testcall(createtest(empty, seq256b4, base+record+255+4, 0)); + testcall(createtest(empty, seq256b5, base+record+255+5, 0)); + testcall(createtest(empty, seq256b6, base+record+255+6, 0)); + testcall(createtest(empty, seq256b7, base+record+255+record+1, 0)); + testcall(createtest(empty, eof, base+record+2, 0)); + testcall(createtest(empty, eof2, base+record+2+rle, 0)); + testcall(createtest(empty, eof3, base+record+2, 0)); +} + +test("IPS") +{ + testips=true; + testbps=false; + + simpletests(); +} + +test("BPS") +{ + //test_skip("fix ips first"); + + testips=false; + testbps=true; + + simpletests(); +} + +test("the big ones") +{ + testips=true; + testbps=true; + + test_skip("enable this whenever ips/bps tests are done (also this is sufficient ups test)"); + + array smw = file::read("patch/test/smw.sfc"); + array smw_bps = file::read("patch/test/smwcp.bps"); + array dl = file::read("patch/test/langrisser.sfc"); + array dl_ups = file::read("patch/test/dl.ups"); + array sm64 = file::read("patch/test/sm64.z64"); + array sm64_bps = file::read("patch/test/star.bps"); + if (!smw || !smw_bps || !dl || !dl_ups || !sm64 || !sm64_bps) test_skip("test files not present; see patch/test/readme.txt"); + + array smwhack; + bps::apply(file::mem(smw_bps), file::mem(smw), file::mem(smwhack)); + testcall(createtest(smw, smwhack, 3328022, 2077386)); + + array sm64hack; + bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack)); + testcall(createtest(sm64, sm64hack, -1, 0)); + + array dlhack; + ups::apply(file::mem(dl_ups), file::mem(dl), file::mem(dlhack)); + testcall(createtest(dl, dlhack, 0, 0)); +} +} diff --git a/patch/test/readme.txt b/patch/test/readme.txt new file mode 100644 index 0000000..4ba0e95 --- /dev/null +++ b/patch/test/readme.txt @@ -0,0 +1,9 @@ +These files are copyrighted, large, and not very relevant to Floating IPS. You'll have to fill them in yourself. + +MD5 checksums and sources: +8134163af1e658a3f1af130afd66d17a dl.ups http://superfamicom.org/translations/info/der-langrisser-english +91d62c4cb790fc2fb38b10b68616e228 langrisser.sfc +20b854b239203baf6c961b850a4a51a2 sm64.z64 +b38466ee8ae3130f3cf1c94f9c7655c0 smwcp.bps https://smwc.me/s/5420 +cdd3c8c37322978ca8669b34bc89c804 smw.sfc +461ae37a8c9d3a4e1990c9b8907b716f star.bps https://smwc.me/s/13353