From d8e4ec5fb7bcd5cd0576e3629da5a4db5c544aca Mon Sep 17 00:00:00 2001 From: Alcaro Date: Sun, 25 Dec 2016 18:34:49 +0100 Subject: [PATCH] Rewrite and optimize IPS creator (it's smaller too!) --- arlib/Makefile | 11 +-- arlib/array.h | 61 ++++++++++------ patch/bps.cpp | 37 ++++++---- patch/ips.cpp | 187 +++++++++++++++++++++++-------------------------- patch/patch.h | 14 ++-- patch/test.cpp | 19 +++-- 6 files changed, 175 insertions(+), 154 deletions(-) diff --git a/arlib/Makefile b/arlib/Makefile index 64a4bff..429ef61 100644 --- a/arlib/Makefile +++ b/arlib/Makefile @@ -41,7 +41,7 @@ CXXFLAGS = $(CFLAGS) LD = g++ LFLAGS = OBJNAME = -CCXXFLAGS = -fvisibility=hidden -Wall -Og +CCXXFLAGS = -fvisibility=hidden -Wall ifneq ($(EXCEPTIONS),1) CCXXFLAGS += -fno-exceptions endif @@ -91,10 +91,13 @@ endif OPTFLAGS := -Os -fomit-frame-pointer -fmerge-all-constants -fvisibility=hidden OPTFLAGS += -fno-unwind-tables -fno-asynchronous-unwind-tables OPTFLAGS += -ffunction-sections -fdata-sections -OPTFLAGS += -Werror +OPTFLAGS += -Werror -DNDEBUG -ifeq ($(OPT),1) - CFLAGS += $(OPTFLAGS) +ifneq ($(OPT),0) + CONF_CFLAGS += $(OPTFLAGS) + ifeq ($(OPT),spd) + CONF_CFLAGS += -O3 + endif LFLAGS += -Wl,--gc-sections -s DEBUG = 0 OBJNAME += -opt diff --git a/arlib/array.h b/arlib/array.h index 703af8c..a709fa2 100644 --- a/arlib/array.h +++ b/arlib/array.h @@ -339,6 +339,44 @@ public: return *this; } + array& operator+=(arrayview other) + { + size_t prevcount = this->count; + size_t othercount = other.size(); + + const T* src; + T* dst; + + if (other.ptr() >= this->ptr() && other.ptr() < this->ptr()+this->size()) + { + size_t start = other.ptr()-this->ptr(); + + resize_grow_noinit(prevcount + othercount); + src = this->items+start; + dst = this->items+prevcount; + } + else + { + resize_grow_noinit(prevcount + othercount); + src = other.ptr(); + dst = this->items+prevcount; + } + + if (this->trivial_copy) + { + memcpy(dst, src, sizeof(T)*othercount); + } + else + { + for (size_t i=0;icount;i++) this->items[i].~T(); @@ -354,29 +392,6 @@ public: ret.resize_grow_noinit(count); return ret; } - - array& operator+=(arrayview other) - { - //TODO: x+=x doesn't work - size_t prevcount = this->count; - size_t othercount = other.size(); - - resize_grow_noinit(prevcount + othercount); - - if (this->trivial_copy) - { - memcpy(this->items+prevcount, other.ptr(), sizeof(T)*othercount); - } - else - { - for (size_t i=0;iitems[prevcount + i]) T(other[i]); - } - } - - return *this; - } }; diff --git a/patch/bps.cpp b/patch/bps.cpp index 497ce43..c1e1292 100644 --- a/patch/bps.cpp +++ b/patch/bps.cpp @@ -19,7 +19,7 @@ namespace patch { namespace bps { enum { SourceRead, TargetRead, SourceCopy, TargetCopy }; -result apply(arrayview patchmem, arrayview in, array& out, bool accept_wrong_input) +result apply(arrayview patchmem, arrayview inmem, array& outmem, bool accept_wrong_input) { if (patchmem.size()<4+3+12) return e_broken; @@ -44,7 +44,7 @@ result apply(arrayview patchmem, arrayview in, array& out, boo uint32_t crc_out_e = checks.u32(); uint32_t crc_patch_e = checks.u32(); - uint32_t crc_in_a = crc32(in); + uint32_t crc_in_a = crc32(inmem); uint32_t crc_patch_a = crc32(patchmem.slice(0, patchmem.size()-4)); if (crc_patch_a != crc_patch_e) error(e_broken); @@ -55,14 +55,13 @@ result apply(arrayview patchmem, arrayview in, array& out, boo size_t outlen; decodeto(outlen); - if (inlen!=in.size() || (crc_in_a!=crc_in_e && !accept_wrong_input)) + if (inlen!=inmem.size() || (crc_in_a!=crc_in_e && !accept_wrong_input)) { - if (in.size()==outlen && crc_in_a==crc_out_e) error=e_to_output; + if (inmem.size()==outlen && crc_in_a==crc_out_e) error=e_to_output; else error=e_not_this; - if (inlen==in.size() && !accept_wrong_input) goto exit; + if (inlen==inmem.size() && !accept_wrong_input) goto exit; } - array outmem; outmem.reserve_noinit(outlen); membufwriter out = outmem; @@ -85,8 +84,8 @@ result apply(arrayview patchmem, arrayview in, array& out, boo { case SourceRead: { - if (out.pos()+length > in.size()) error(e_broken); - out.write(in.slice(out.pos(), length)); + if (out.pos()+length > inmem.size()) error(e_broken); + out.write(inmem.slice(out.pos(), length)); } break; case TargetRead: @@ -102,8 +101,8 @@ result apply(arrayview patchmem, arrayview in, array& out, boo size_t distance=encodeddistance>>1; if ((encodeddistance&1)==0) { - if (inreadat+length > in.size()) error(e_broken); inreadat+=distance; + if (inreadat+length > inmem.size()) error(e_broken); } else { @@ -111,7 +110,7 @@ result apply(arrayview patchmem, arrayview in, array& out, boo inreadat-=distance; } - out.write(in.slice(inreadat, length)); + out.write(inmem.slice(inreadat, length)); inreadat+=length; } break; @@ -122,9 +121,9 @@ result apply(arrayview patchmem, arrayview in, array& out, boo size_t distance=encodeddistance>>1; if ((encodeddistance&1)==0) { - if (distance+outreadat > out.pos()) error(e_broken); - if (outreadat+length > out.size()) error(e_broken); outreadat+=distance; + if (outreadat+length > out.size()) error(e_broken); + if (outreadat >= out.pos()) error(e_broken); } else { @@ -132,8 +131,16 @@ result apply(arrayview patchmem, arrayview in, array& out, boo outreadat-=distance; } - out.write(outmem.slice(outreadat, length)); - outreadat+=length; + size_t outreadstart = outreadat; + outreadat += length; + + while (outreadstart+length > out.pos()) + { + size_t chunk = out.pos()-outreadstart; + out.write(outmem.slice(outreadstart, chunk)); + length -= chunk; + } + out.write(outmem.slice(outreadstart, length)); } break; } @@ -154,7 +161,7 @@ result apply(arrayview patchmem, arrayview in, array& out, boo } exit: - out.resize(0); + outmem.resize(0); return error; } diff --git a/patch/ips.cpp b/patch/ips.cpp index 09f9be6..7253fb8 100644 --- a/patch/ips.cpp +++ b/patch/ips.cpp @@ -29,7 +29,7 @@ result apply(arrayview patchmem, const file& in, array& out) if (patch.remaining() < 2+1+3) error(e_broken); size = patch.u16be(); uint8_t b = patch.u8(); - if (!size) error(e_broken); // is this defined? + if (!size) error(e_broken); // don't know if this is defined, probably isn't out.reserve(offset+size); if (!anychanges && @@ -52,8 +52,8 @@ result apply(arrayview patchmem, const file& in, array& out) } if (patch.remaining()==3) { - uint32_t newsize = patch.u24(); - if (newsize <= out.size() && !error) error = e_not_this; + uint32_t newsize = patch.u24be(); + if (newsize >= in.size() && !error) error = e_not_this; out.resize(newsize); } if (patch.remaining()!=0) error = e_damaged; @@ -92,52 +92,55 @@ exit: //There are no known cases where LIPS wins over this. -static result create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem) +result create(array source, arrayview target, array& patchmem) { - int sourcelen=sourcemem.len; - int targetlen=targetmem.len; - const unsigned char * source=sourcemem.ptr; - const unsigned char * target=targetmem.ptr; - - patchmem->ptr=NULL; - patchmem->len=0; + int truesourcelen=source.size(); + int targetlen=target.size(); + source.resize(target.size()); + //const unsigned char * source=sourcemem.ptr(); + //const unsigned char * target=targetmem.ptr(); if (targetlen>=16777216) return e_too_big; int offset=0; - int outbuflen=4096; - unsigned char * out=(uint8_t*)malloc(outbuflen); - int outlen=0; -#define write8(val) do { out[outlen++]=(val); if (outlen==outbuflen) { outbuflen*=2; out=(uint8_t*)realloc(out, outbuflen); } } while(0) -#define write16(val) do { write8((val)>>8); write8((val)); } while(0) -#define write24(val) do { write8((val)>>16); write8((val)>>8); write8((val)); } while(0) - write8('P'); - write8('A'); - write8('T'); - write8('C'); - write8('H'); + + array out; + out.append('P'); + out.append('A'); + out.append('T'); + out.append('C'); + out.append('H'); int lastknownchange=0; int lastwritten=0; - //int forcewrite=(targetlen>sourcelen?1:0); +#define ENC24(n) (byte)((n)>>16), (byte)((n)>>8), (byte)((n)>>0) +#define ENC16(n) (byte)((n)>>8), (byte)((n)>>0) +#define ENC8(n) (byte)((n)>>0) while (offset=6 || thislen>65535) break; + else unchangedtimer=6; } + thislen = searchat-offset-6+unchangedtimer; //avoid premature EOF if (offset==0x454F46) @@ -147,11 +150,11 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa } lastknownchange=offset+thislen; - if (thislen>65535) thislen=65535; + if (offset+thislen>targetlen) thislen=targetlen-offset; if (offset==targetlen) continue; - //check if RLE here is worthwhile + //check if starting a RLE here is worthwhile int byteshere; for (byteshere=0;byteshere=targetlen || target[pos]!=thisbyte || byteshere+i>65535) break; - if (pos>=sourcelen || (pos=truesourcelen || source[pos]!=thisbyte) { byteshere+=i; thislen+=i; @@ -173,39 +176,46 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa } if ((byteshere>8-5 && byteshere==thislen) || byteshere>8) { - write24(offset); - write16(0); - write16(byteshere); - write8(target[offset]); + byte bytes[] = { ENC24(offset), ENC16(0), ENC16(byteshere), target[offset] }; + out += arrayview(bytes); offset+=byteshere; lastwritten=offset; } else { //check if we'd gain anything from ending the block early and switching to RLE - int byteshere=0; - int stopat=0; - while (stopat+byteshere= 12) { - stopat+=byteshere; - byteshere=0; - } - if (byteshere>8+5 || //rle-worthy despite two ips headers - (byteshere>8 && stopat+byteshere==thislen) || //rle-worthy at end of data - (byteshere>8 && offset+stopat+byteshere+8 <= thislen && - !memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy - { - if (stopat) thislen=stopat; - break;//we don't scan the entire block if we know we'll want to RLE, that'd gain nothing. + thislen=rlestart; + break; } } - //don't write unchanged bytes at the end of a block if we want to RLE the next couple of bytes + if (thislen-rlestart >= 8) + { + thislen=rlestart; + } + + //don't copy unchanged bytes at the end of a block if (offset+thislen!=targetlen) { - while (offset+thislen-13 && !memcmp(&target[offset], &target[offset+1], thislen-1))//still worth it? + + if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//can we switch to RLE for these few bytes? { - write24(offset); - write16(0); - write16(thislen); - write8(target[offset]); + byte bytes[] = { ENC24(offset), ENC16(0), ENC16(thislen), target[offset] }; + out += arrayview(bytes); } else { - write24(offset); - write16(thislen); - for (int i=0;i(bytes); + out += target.slice(offset, thislen); } offset+=thislen; lastwritten=offset; } } - if (sourcelen(bytes); } else { - write24(targetlen-1); - write16(1); - write8(target[targetlen-1]); + byte bytes[] = { ENC24(targetlen-1), ENC16(1), target[targetlen-1] }; + out += arrayview(bytes); } } - write8('E'); - write8('O'); - write8('F'); - if (sourcelen>targetlen) write24(targetlen); -#undef write - patchmem->ptr=out; - patchmem->len=outlen; - if (outlen==8) return e_identical; + byte bytes[] = { 'E', 'O', 'F', ENC24(targetlen) }; + if (truesourcelen>targetlen) out+=arrayview(bytes, 6); + else out+=arrayview(bytes, 3); +#undef ENC8 +#undef ENC16 +#undef ENC24 + patchmem = std::move(out); + if (patchmem.size()==8) return e_identical; return e_ok; } -result create(const file& source, const file& target, file& patch) -{ - struct mem sourcemem = source.mmap(); - struct mem targetmem = target.mmap(); - struct mem patchmem; - result r = create(sourcemem, targetmem, &patchmem); - source.unmap(sourcemem.v()); - target.unmap(targetmem.v()); - patch.write(patchmem.v()); - free(patchmem.ptr); - return r; -} - #if 0 #warning Disable this in release versions. #include diff --git a/patch/patch.h b/patch/patch.h index 11ceb8a..d51d949 100644 --- a/patch/patch.h +++ b/patch/patch.h @@ -26,15 +26,15 @@ enum result { e_canceled //Patch creation callback said cancel. }; +//All of these functions can be called with arrayview inputs and array& outputs, but +// they give lower memory use and/or better performance if you follow the listed types. +//For example, IPS and UPS application start with copying the input file to the output; +// if you give them a file object directly, they'll read it straight from disk to the target buffer. + namespace ips { result apply(arrayview patch, const file& in, array& out); -static inline result apply(arrayview patch, arrayview in, array& out) -{ - file inf = file::mem(in); - return apply(patch, inf, out); -} -result create(const file& source, const file& target, file& patch); -static inline result create(const file& source, const file& target, file&& patch) { return create(source, target, (file&)patch); } +static inline result apply(arrayview patch, arrayview in, array& out) { return apply(patch, file::mem(in), out); } +result create(array source, arrayview target, array& patch); } namespace ups { diff --git a/patch/test.cpp b/patch/test.cpp index 3592b4a..69e4c08 100644 --- a/patch/test.cpp +++ b/patch/test.cpp @@ -62,7 +62,7 @@ static void createtest(arrayview a, arrayview b, size_t ipssize, siz if (testips && b.size()<=16777216) { array patch; - result r = ips::create(file::mem(a), file::mem(b), file::mem(patch)); + result r = ips::create(a, b, patch); if (r!=e_identical) assert_eq(r, e_ok); array b2; r = ips::apply(patch, a, b2); @@ -98,6 +98,7 @@ static void createtest(arrayview a, arrayview b, size_t ipssize, siz } } +MAYBE_UNUSED static void simpletests() { array empty; @@ -168,7 +169,8 @@ test("BPS") test("the big ones") { testips=true; - testbps=true; + //testbps=true; + testbps=false; array smw = file::read("patch/test/smw.sfc"); array smw_bps = file::read("patch/test/smwcp.bps"); @@ -179,20 +181,23 @@ test("the big ones") if (!smw || !smw_bps || !dl || !dl_ups || !sm64 || !sm64_bps) test_skip("test files not present; see patch/test/readme.txt"); result r; - //array smwhack; - //r = bps::apply(smw_bps, smw, smwhack); - //assert_eq(r, e_ok); - //testcall(createtest(smw, smwhack, 3302980, 2077386)); + array smwhack; + r = bps::apply(smw_bps, smw, smwhack); + assert_eq(r, e_ok); + assert_eq(smwhack.size(), 4194304); + testcall(createtest(smw, smwhack, 3302746, 2077386)); //array sm64hack; - //r = bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack)); + //r = bps::apply(sm64_bps, sm64, sm64hack); //assert_eq(r, e_ok); + //assert_eq(sm64hack.size(), 50331648); //testcall(createtest(sm64, sm64hack, -1, 6788133)); //this is the only UPS test, UPS is pretty much an easter egg in Flips //array dlhack; //r = ups::apply(dl_ups, dl, dlhack); //assert_eq(r, e_ok); + //assert_eq(dlhack.size(), 3145728); //array dl2; //r = ups::apply(dl_ups, dlhack, dl2); //assert_eq(r, e_ok);