Leahnaya/Flips
1
0
Fork 0
mirror of https://github.com/Alcaro/Flips.git synced 2026-09-09 02:55:18 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

More updates

Browse Source
This commit is contained in:
Alcaro
2016-12-30 01:25:09 +01:00
parent 56cd2b4ca6
commit a6b0176344
17 changed files with 463 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
patch/bps-create.cpp
View File

@@ -1,15 +1,5 @@
#include "patch.h"
//Deprecated
//struct mem {
//mem() : ptr(NULL), len(0) {}
//mem(uint8_t* ptr, size_t len) : ptr(ptr), len(len) {}
//mem(arrayview<byte> v) : ptr((byte*)v.ptr()), len(v.size()) {}
//arrayvieww<byte> v() { return arrayvieww<byte>(ptr, len); }
//uint8_t * ptr;
//size_t len;
//};
//These two give minor performance penalties and will print some random stuff to stdout.
//The former will verify the correctness of the output patch, the latter will print some performance data.
//Can be useful for debugging, but should be disabled for release builds.
@@ -36,6 +26,9 @@
// called O(n) times, with O(log n) comparisons per iteration. Each comparison is potentially O(n),
// but for each matched byte, another iteration is removed from the outer loop, so the comparisons
// can be considered O(1) each; the sum is O(n log n).
//It could be replaced with a reverse index, reverse[sorted[x]]==x for all x, but that would cost a
// lot of memory, and due to the cost of creating said index and only a few entries being used, it
// doesn't save any time in practice.
//
//After it's found sortpos, it scans sorted[] up and down for the closest entry that either starts
// before the current output position, or is somewhere in the source file.
@@ -53,14 +46,15 @@
// This gives O(log n) calls to the suffix sorter.
//Given O(n log n) for one sorting step, the time taken is O(n/1 log n/1 + n/2 log n/2 +
// n/4 log n/4 + ...), which is strictly less than O(n/1 log n + n/2 log n + n/4 log n + ...), which
// equals O(2n log n), which is O(n log n). (The exact value of that infinite sum is 2n*log(n/2).)
// equals O(2n log n), which is O(n log n).
//
//Many details were omitted from the above, but that's the basic setup.
//
//Thus, the program is O(max(n log n, n log n, n, n) = n log n) average and O(max(n log n, n log n,
// n^2, n) = n^2) worst case.
//Thus, the program is O(n log n) + O(n log n) + O(n) + O(n) = O(n log n) average and O(n log n) +
// O(n log n) + O(n^2) + O(n) = O(n^2) worst case.
//
//I conclude that the task of finding, understanding and implementing a sub-O(n^2) algorithm for
//As the quadratic worst case is not hit for random data or any other plausible output file, I
// conclude that the task of finding, understanding and implementing a sub-quadratic algorithm for
// delta patching is resolved.
@@ -73,9 +67,10 @@
// Penalty: Likely O(n) or O(n log log n), with low constants. I'd guess ~1.4% for my 48MB test file.
//However, due to better heuristics and others' performance optimizations, this one still beats its
// competitors.
//Heuristics are likely somewhat mistuned.
//TODO: test multiple same-length matches
// but only for lengths <= 64,
// but only for lengths <= 16 or something, otherwise it'd take too long
//Possible optimizations:
@@ -84,30 +79,32 @@
//If each iteration takes 4 times as long as the previous one, then the last one takes 3/4 of the total time.
//Since divsufsort doesn't depend on anything else, the last iteration can be split off to its own thread.
//This would split it to
//Search, non-final: 1/2 * 1/4 = 1/8
//Search, final: 1/2 * 3/4 = 3/8
//Sort+rev, non-final: 1/2 * 1/4 = 1/8
//Sort+rev, final: 1/2 * 3/4 = 3/8
//Search, non-final: 1/2 * 1/4 = 1/8
//Search, final: 1/2 * 3/4 = 3/8
//Sort, non-final: 1/2 * 1/4 = 1/8
//Sort, final: 1/2 * 3/4 = 3/8
//All non-final must be done sequentially. Both Sort Final and non-final must be done before Search Final can start.
//This means the final time, if Sort Final is split off, is
//max(1/8+1/8, 3/8) + 3/8 = 6/8 = 3/4
//of the original time.
//Due to
//- the considerable complexity costs (OpenMP doesn't seem able to represent the "insert a wait in
// the middle of this while loop" I would need)
// the middle of this while loop" operation I would need)
//- the added memory use, approximately 25% higher - it's already high enough
//- libdivsufsort already using threads, which would make the gains lower
// and would increase complexity, as I have to ensure the big one remains threaded -
// and that the small ones are not, as that'd starve the big one
//I deem a possible 25% boost not worthwhile.
//Both sorting algorithms claim O(1) memory use, in addition to the in/outputs. For most hardware,
// this is 5*(source.len+target.len).
//If the output is stored to disk, that's all this algorithm needs as well.
//Another optimization would be if a faster suffix sorting algorithm available.
//Both SA-IS and libdivsufsort claim O(1) memory use, in addition to the in/outputs. For most
// hardware, this is 5*(source.len+target.len).
//The output file is also stored in memory, which is potentially slightly more than the output file
// size. This could be changed without too much trouble, but is unlikely to be worth it.
namespace patch { namespace bps {
//TODO: HEAVY cleanups needed here
#include "sais.cpp"
template<typename sais_index_type>
static void sufsort(sais_index_type* SA, const uint8_t* T, sais_index_type n) {

14
patch/bps.cpp
View File

@@ -1,14 +1,14 @@
#include "patch.h"
//Things I would've done differently if I had a chance to redesign BPS:
//Things I would've done differently if I had a chance to redesign the BPS format:
//- Don't allow encoding -0 in Source/TargetCopy
//- Ditch metadata, it goes in a separate file
//- Ditch metadata, it goes in a separate file (and it's already used in nonstandard ways, spec says XML but bsnes disagrees)
//- Reconsider SourceRead; maybe patches would be smaller if of the three others was one bit rather than two, or maybe a new command
// or maybe only Read/Copy commands? Read is TargetRead, Copy treats target as concatenated to source
//- Invert 0x80 bit in the encoded numbers, set means continue; it would simplify the decoder
//- Replace BPS1 signature with something not containing an 1
// while DWORD alignment sounds nice, it's useless for a byte-oriented format like this; even the checksums aren't aligned
// four-byte signatures are nicer than three, but '1' is the wrong choice for the last byte; PNG's \x89 would work
// four-byte signatures are nicer than three, but '1' is the wrong choice for the last byte; PNGs \x89 would work
//- Make the checksums mandatory
// (1) Ignoring them allows all files of that size, including ones that are clearly not the proper source
// (2) Even if a ROM hacker is careful to only change a few bytes, BPS likes copying stuff around,
@@ -34,10 +34,7 @@ result apply(arrayview<byte> patchmem, arrayview<byte> inmem, array<byte>& outme
if (!patch.bpsnum(&var)) error(e_too_big); \
} while(false)
if (patch.u8()!='B') error(e_broken);
if (patch.u8()!='P') error(e_broken);
if (patch.u8()!='S') error(e_broken);
if (patch.u8()!='1') error(e_broken);
if (!patch.signature("BPS1")) error(e_broken);
memstream checks = patchmem.slice(patchmem.size()-12, 12);
uint32_t crc_in_e = checks.u32();
@@ -191,7 +188,7 @@ result info::parse(arrayview<byte> data, bool changefrac)
size_t outpos=0; // position in the output file
size_t changeamt=0; // change score
while (patch.remaining())
while (patch.remaining() && outpos<this->size_in)
{
size_t thisinstr;
patch.bpsnum(&thisinstr);
@@ -223,7 +220,6 @@ result info::parse(arrayview<byte> data, bool changefrac)
}
outpos+=length;
}
if (outpos>this->size_out) return e_broken;
this->change_num = (changeamt<this->size_in ? changeamt : this->size_in);
this->change_denom = this->size_in;
}

3
patch/patch.h
View File

@@ -14,6 +14,7 @@ enum result {
e_ok,
//You may get an output file along with some of these errors.
//This is indistinguishable from zero-size output, but such patches are useless and rare anyways.
e_to_output,//You attempted to apply a patch to its output.
e_not_this, //This is not the intended input file for this patch.
e_damaged, //The patch is technically valid, but seems scrambled or malformed.
@@ -28,7 +29,7 @@ enum result {
//All of these functions can be called with arrayview inputs and array& outputs, but
// they give lower memory use and/or better performance if you follow the listed types.
//For example, IPS and UPS application start with copying the input file to the output;
//For example, applying an IPS or UPS starts with copying the input file to the output;
// if you give them a file object directly, they'll read it straight from disk to the target buffer.
namespace ips {

25
patch/test.cpp
View File

@@ -1,5 +1,7 @@
#include "patch.h"
#if 0
/*
make clean; rm callgrind.out.*; make test -j8 TESTRUNNER='time valgrind --tool=callgrind' CFLAGS='-Os -g' && kcachegrind callgrind.out.*
*/
@@ -169,7 +171,7 @@ test("BPS")
test("the big ones")
{
testips=true;
testips=false;
//testips=false;
testbps=true;
//testbps=false;
@@ -180,30 +182,31 @@ test("the big ones")
array<byte> sm64 = file::read("patch/test/sm64.z64");
array<byte> sm64_bps = file::read("patch/test/star.bps");
if (!smw || !smw_bps || !dl || !dl_ups || !sm64 || !sm64_bps) test_skip("test files not present; see patch/test/readme.txt");
result r;
array<byte> smwhack;
r = bps::apply(smw_bps, smw, smwhack);
assert_eq(r, e_ok);
result smwr = bps::apply(smw_bps, smw, smwhack);
assert_eq(smwr, e_ok);
assert_eq(smwhack.size(), 4194304);
testcall(createtest(smw, smwhack, 3302746, 2077386));
//this is the only UPS test, UPS is pretty much an easter egg in Flips
array<byte> dlhack;
r = ups::apply(dl_ups, dl, dlhack);
assert_eq(r, e_ok);
result dlr = ups::apply(dl_ups, dl, dlhack);
assert_eq(dlr, e_ok);
assert_eq(dlhack.size(), 3145728);
array<byte> dl2;
r = ups::apply(dl_ups, dlhack, dl2);
assert_eq(r, e_ok);
dlr = ups::apply(dl_ups, dlhack, dl2);
assert_eq(dlr, e_ok);
assert(dl == dl2);
testcall(createtest(dl, dlhack, 852124, 817190));
array<byte> sm64hack;
r = bps::apply(sm64_bps, sm64, sm64hack);
assert_eq(r, e_ok);
result sm64r = bps::apply(sm64_bps, sm64, sm64hack);
assert_eq(sm64r, e_ok);
assert_eq(sm64hack.size(), 50331648);
testbps=false; // too slow
//removing this makes that entire createtest useless
//testbps=false; // too slow
testcall(createtest(sm64, sm64hack, -1, 6788133));
}
}
#endif

6
patch/ups.cpp
View File

@@ -1,7 +1,6 @@
#include "patch.h"
namespace patch { namespace ups {
//TODO: HEAVY cleanups needed here
result apply(arrayview<byte> patchmem, const file& in, array<byte>& outmem)
{
@@ -20,10 +19,7 @@ result apply(arrayview<byte> patchmem, const file& in, array<byte>& outmem)
bool backwards=false;
if (patch.u8()!='U') error(e_broken);
if (patch.u8()!='P') error(e_broken);
if (patch.u8()!='S') error(e_broken);
if (patch.u8()!='1') error(e_broken);
if (!patch.signature("UPS1")) error(e_broken);
size_t inlen;
size_t outlen;