From 086498bf01efc4887ed5a1e929792cb05c74cf76 Mon Sep 17 00:00:00 2001 From: Alcaro Date: Sun, 25 Dec 2016 21:44:57 +0100 Subject: [PATCH] Clean up BPS creator a bit... it's already well optimized, not much to do. --- arlib/file.h | 2 +- arlib/memcmp_d.cpp | 23 --------- patch/bps-create.cpp | 115 +++++++++++++++---------------------------- patch/patch.h | 45 ++++++++++------- patch/test.cpp | 5 +- 5 files changed, 72 insertions(+), 118 deletions(-) diff --git a/arlib/file.h b/arlib/file.h index 5e706c8..e6b4099 100644 --- a/arlib/file.h +++ b/arlib/file.h @@ -41,6 +41,7 @@ public: }; private: impl* core; + file(impl* core) : core(core) {} public: enum mode { @@ -54,7 +55,6 @@ public: file() : core(NULL) {} file(file&& f) { core=f.core; f.core=NULL; } file& operator=(file&& f) { delete core; core=f.core; f.core=NULL; return *this; } - file(impl* core) : core(core) {} file(cstring filename, mode m = m_read) : core(NULL) { open(filename, m); } bool open(cstring filename, mode m = m_read) diff --git a/arlib/memcmp_d.cpp b/arlib/memcmp_d.cpp index a70c40e..d83bc0a 100644 --- a/arlib/memcmp_d.cpp +++ b/arlib/memcmp_d.cpp @@ -33,26 +33,3 @@ size_t memcmp_d(const void * a, const void * b, size_t len) while (i v) : ptr((byte*)v.ptr()), len(v.size()) {} + //arrayvieww v() { return arrayvieww(ptr, len); } + //uint8_t * ptr; + //size_t len; +//}; + //These two give minor performance penalties and will print some random stuff to stdout. //The former will verify the correctness of the output patch, the latter will print some performance data. //Can be useful for debugging, but should be disabled for release builds. @@ -131,11 +141,6 @@ static void sufsort(int64_t* SA, uint8_t* T, int64_t n) -template static T min(T a, T b) { return a static T max(T a, T b) { return a outbuflen) - { - if (!outbuflen) outbuflen = 128; - while (outlen+len > outbuflen) outbuflen *= 2; - out = (uint8_t*)realloc(out, outbuflen); - } - } - - void append(const uint8_t * data, size_t len) - { - reserve(len); - memcpy(out+outlen, data, len); - outlen+=len; - } + array out; void appendnum(size_t num) { @@ -177,29 +163,27 @@ struct bps_creator { if (num > 1000000000) printf("ERROR: Attempt to write %.8lX\n",(unsigned long)num),abort(); #endif - reserve(sizeof(size_t)*8/7+1); while (num >= 128) { - out[outlen++]=(num&0x7F); + out.append((num&0x7F)); num>>=7; num--; } - out[outlen++]=num|0x80; + out.append(num|0x80); } void appendnum32(uint32_t num) { - reserve(4); - out[outlen++] = num>>0; - out[outlen++] = num>>8; - out[outlen++] = num>>16; - out[outlen++] = num>>24; + out.append(num>>0); + out.append(num>>8); + out.append(num>>16); + out.append(num>>24); } static size_t maxsize() { - return SIZE_MAX>>2; // can be reduced to SIZE_MAX>>1 by amending append_cmd, but the mallocs overflow at that point anyways. + return SIZE_MAX>>2; // can probably be reduced to SIZE_MAX>>1, but the mallocs overflow at that point anyways. } size_t sourcelen; @@ -215,12 +199,8 @@ struct bps_creator { size_t numtargetread; - bps_creator(const file& source, const file& target, struct mem metadata) + bps_creator(const file& source, const file& target, const file& metadata) { - outlen = 0; - outbuflen = 128; - out = (uint8_t*)malloc(outbuflen); - outpos = 0; sourcelen = source.size(); @@ -231,11 +211,13 @@ struct bps_creator { numtargetread = 0; - append((const uint8_t*)"BPS1", 4); + out += arrayview((byte*)"BPS1", 4); appendnum(sourcelen); appendnum(targetlen); - appendnum(metadata.len); - append(metadata.ptr, metadata.len); + appendnum(metadata.size()); + arrayview tmp = metadata.mmap(); + out += tmp; + metadata.unmap(tmp); } @@ -265,7 +247,7 @@ struct bps_creator { { if (!numtargetread) return; append_cmd(TargetRead, numtargetread); - append(targetmem+outpos-numtargetread, numtargetread); + out += arrayview(targetmem+outpos-numtargetread, numtargetread); numtargetread = 0; } @@ -365,17 +347,15 @@ struct bps_creator { appendnum32(crc32(arrayview(source, sourcelen))); appendnum32(crc32(arrayview(target, targetlen))); - appendnum32(crc32(arrayview(out, outlen))); + appendnum32(crc32(out)); } - struct mem getpatch() + size_t outlen() { return out.size(); } + + array getpatch() { - struct mem ret = { out, outlen }; - out = NULL; - return ret; + return std::move(out); } - - ~bps_creator() { free(out); } }; } @@ -386,10 +366,10 @@ static int match_len_n=0; static int match_len_tot=0; #endif -template -static off_t match_len(const uint8_t* a, const uint8_t* b, off_t len) +static size_t match_len(const uint8_t* a, const uint8_t* b, size_t len) { - off_t i; + //don't replace with memcmp_d, the average match length is so small it's a net loss + size_t i; for (i=0;i0 && *search==*here) { search++; @@ -619,18 +600,6 @@ static off_t find_index(off_t pos, const uint8_t* data, off_t datalen, const off } -template -static void create_reverse_index(off_t* index, off_t* reverse, off_t len) -{ -//testcase: linux 3.18.14 -> 4.0.4 .xz -//without: real23.544 user32.930 -//with: real22.636 user40.168 -//'user' jumps up quite a lot, while 'real' only moves a short bit -//I'm not sure why the tradeoff is so bad (do the cachelines bounce THAT badly?), but I deem it not worth it. -//#pragma omp parallel for - for (off_t i=0;i static off_t nextsize(off_t outpos, off_t sortedsize, off_t targetlen) { @@ -784,26 +753,22 @@ template<> result create_suf_pick(const file& source, const file& targ //This one picks a function based on 32-bit integers if that fits. This halves memory use for common inputs. //It also handles some stuff related to the BPS headers and footers. -result create(const file& source, const file& target, const file& metadata, file& patch, +result create(const file& source, const file& target, const file& metadata, array& patch, function progress) { - mem metamem = metadata.mmap(); - bps_creator bps(source, target, metamem); - metadata.unmap(metamem.v()); + bps_creator bps(source, target, metadata); bps.progress = progress; - size_t maindata = bps.outlen; + size_t maindata = bps.outlen(); //off_t must be signed result err = create_suf_pick(source, target, &bps); if (err!=e_ok) return err; - mem patchmem = bps.getpatch(); - patch.write(patchmem.v()); + patch = bps.getpatch(); - while ((patchmem.ptr[maindata]&0x80) == 0x00) maindata++; - free(patchmem.ptr); - if (maindata==patchmem.len-12-1) return e_identical; + while ((patch[maindata]&0x80) == 0x00) maindata++; + if (maindata==patch.size()-12-1) return e_identical; return e_ok; } diff --git a/patch/patch.h b/patch/patch.h index d51d949..8c78503 100644 --- a/patch/patch.h +++ b/patch/patch.h @@ -59,22 +59,43 @@ result apply(arrayview patch, arrayview source, array& target, // called for done=total, done may or may not increase by the same amount between each call, and // the duration between each call may or may not be constant. //To cancel patch creation, return true from the callback. It's safe to pass in NULL if you're not interested. -result create(const file& source, const file& target, const file& metadata, file& patch, +result create(const file& source, const file& target, const file& metadata, array& patch, function progress); -static inline result create(const file& source, const file& target, const file& metadata, file&& patch, +static inline result create(arrayview source, const file& target, const file& metadata, array& patch, function progress) { - return create(source, target, metadata, (file&)patch, progress); + return create(file::mem(source), target, metadata, patch, progress); } -static inline result create(const file& source, const file& target, file& patch, +static inline result create(const file& source, arrayview target, const file& metadata, array& patch, function progress) { - return create(source, target, file::mem(NULL), (file&)patch, progress); + return create(source, file::mem(target), metadata, patch, progress); } -static inline result create(const file& source, const file& target, file&& patch, +static inline result create(arrayview source, arrayview target, const file& metadata, array& patch, function progress) { - return create(source, target, (file&)patch, progress); + return create(file::mem(source), file::mem(target), metadata, patch, progress); +} + +static inline result create(const file& source, const file& target, array& patch, + function progress) +{ + return create(source, target, file::mem(NULL), patch, progress); +} +static inline result create(arrayview source, const file& target, array& patch, + function progress) +{ + return create(file::mem(source), target, file::mem(NULL), patch, progress); +} +static inline result create(const file& source, arrayview target, array& patch, + function progress) +{ + return create(source, file::mem(target), file::mem(NULL), patch, progress); +} +static inline result create(arrayview source, arrayview target, array& patch, + function progress) +{ + return create(file::mem(source), file::mem(target), file::mem(NULL), patch, progress); } struct info { @@ -233,14 +254,4 @@ public: return crc; } }; - -//Deprecated -struct mem { - mem() : ptr(NULL), len(0) {} - mem(uint8_t* ptr, size_t len) : ptr(ptr), len(len) {} - mem(arrayview v) : ptr((byte*)v.ptr()), len(v.size()) {} - arrayvieww v() { return arrayvieww(ptr, len); } - uint8_t * ptr; - size_t len; -}; } diff --git a/patch/test.cpp b/patch/test.cpp index 8e6592a..5bb203d 100644 --- a/patch/test.cpp +++ b/patch/test.cpp @@ -82,7 +82,7 @@ static void createtest(arrayview a, arrayview b, size_t ipssize, siz if (testbps) { array patch; - result r = bps::create(file::mem(a), file::mem(b), file::mem(patch), NULL); + result r = bps::create(a, b, patch, NULL); if (r!=e_identical) assert_eq(r, e_ok); array b2; r = bps::apply(patch, a, b2); @@ -169,8 +169,9 @@ test("BPS") test("the big ones") { testips=true; + testips=false; testbps=true; - testbps=false; + //testbps=false; array smw = file::read("patch/test/smw.sfc"); array smw_bps = file::read("patch/test/smwcp.bps");