Clean up UPS handler

This commit is contained in:
Alcaro 2016-12-23 21:03:52 +01:00
parent 63c7cb74cf
commit 2ab71fc9c4
3 changed files with 207 additions and 231 deletions

View File

@ -36,7 +36,7 @@ static inline result create(const file& source, const file& target, file&& patch
namespace ups {
result apply(const file& patch, const file& source, file& target);
static inline result apply(const file& patch, const file& source, file&& target) { return apply(patch, source, (file&)target); }
//ups is worthless
//no need to implement this
//result create(const file& source, const file& target, file& patch);
}
@ -95,102 +95,127 @@ struct info {
}
//Used for patch application.
class filebufreader {
file& f;
size_t fpos;
array<byte> buf;
size_t bufpos;
uint32_t crc;
class memstream {
const byte* start;
const byte* at;
const byte* end;
//arrayview<byte> buf;
//size_t pos;
public:
filebufreader(file& f) : f(f), fpos(0), bufpos(0), crc(0) {}
arrayview<byte> peek(size_t bytes)
{
if (buf.size()-bufpos < bytes)
{
buf = buf.slice(bufpos, buf.size()-bufpos);
bufpos = 0;
size_t bytehave = buf.size();
size_t byteread = bytes + 4096;
buf.resize(bytehave + byteread);
byteread = f.read(buf.slice(bytehave, byteread), fpos);
fpos += byteread;
buf.resize(bytehave + byteread);
}
return buf.slice(bufpos, min(buf.size()-bufpos, bytes));
}
arrayview<byte> read(size_t bytes)
{
arrayview<byte> ret = peek(bytes);
if (ret.size() != bytes) return NULL;
bufpos += bytes;
crc = crc32_update(ret, crc); // TODO: perhaps it's faster if this one is calculated in large batches
return ret;
}
byte read() { return read(1)[0]; }
size_t remaining() { return buf.size()-bufpos + f.size()-fpos; }
uint32_t crc32() { return crc; }
};
class streamreader {
filebufreader f;
public:
streamreader(file& f) : f(f) {}
arrayview<byte> bytes(size_t n) { return f.read(n); }
memstream(arrayview<byte> buf) : start(buf.ptr()), at(buf.ptr()), end(buf.ptr()+buf.size()) {}
arrayview<byte> bytes(size_t n) { arrayview<byte> ret = arrayview<byte>(at, n); at+=n; return ret; }
uint8_t u8()
{
return f.read(1)[0];
return *(at++);
}
uint8_t u8_or(uint8_t otherwise)
{
if (at==end) return otherwise;
return *(at++);
}
uint16_t u16()
{
arrayview<byte> b = f.read(2);
arrayview<byte> b = bytes(2);
return b[0] | b[1]<<8;
}
uint32_t u24()
{
arrayview<byte> b = f.read(3);
arrayview<byte> b = bytes(3);
return b[0] | b[1]<<8 | b[2]<<16;
}
uint32_t u32()
{
arrayview<byte> b = f.read(4);
arrayview<byte> b = bytes(4);
return b[0] | b[1]<<8 | b[2]<<16 | b[3]<<24;
}
// size_t bpsnum() // close to uleb128, but uleb lacks the +1 that ensures there's only one way to encode an integer
// {
// size_t ret = 0;
// size_t shift = 0;
// while (true)
// {
// uint8_t next = f.read();
// if (SIZE_MAX>>shift < (next&0x7F)) return (size_t)-1;
// size_t shifted = (next&0x7F)<<shift;
//
//#define assert_sum(a,b) do { if (SIZE_MAX-(a)<(b)) error(e_too_big); } while(0)
//#define assert_shift(a,b) do { if (SIZE_MAX>>(b)<(a)) error(e_too_big); } while(0)
//
// }
//#define decodeto(var) \
// do { \
// var=0; \
// unsigned int shift=0; \
// while (true) \
// { \
// uint8_t next=readpatch8(); \
// assert_shift(next&0x7F, shift); \
// size_t addthis=(next&0x7F)<<shift; \
// assert_sum(var, addthis); \
// var+=addthis; \
// if (next&0x80) break; \
// shift+=7; \
// assert_sum(var, 1U<<shift); \
// var+=1<<shift; \
// } \
// } while(false)
//
// arrayview<byte> b = f.peek(16);
// }
size_t size() { return end-start; }
size_t remaining() { return end-at; }
//if the bpsnum is too big, number of read bytes is undefined
//does not do bounds checks, there must be at least 10 unread bytes in the buffer
safeint<size_t> bpsnum()
{
//similar to uleb128, but uleb lacks the +1 that ensures there's only one way to encode an integer
uint8_t first = u8();
if (LIKELY(first&0x80)) return first&0x7F;
safeint<size_t> ret = 0;
safeint<size_t> shift = 0;
while (true)
{
shift+=7;
ret+=1<<shift;
uint8_t next = u8();
safeint<size_t> shifted = (next&0x7F)<<shift;
ret+=shifted;
if (next&0x80 || !ret.valid()) break;
}
return ret;
}
};
class filebufwriter {
file& f;
size_t fpos;
array<byte> buf;
size_t totalbytes;
uint32_t crc;
void flush()
{
crc = crc32_update(buf, crc);
f.write(buf, fpos);
fpos += buf.size();
buf.reset();
}
public:
filebufwriter(file& f) : f(f), fpos(0), totalbytes(0), crc(0) {}
void write(arrayview<byte> bytes)
{
buf += bytes;
totalbytes += bytes.size();
if (buf.size() > 65536) flush();
}
void write(byte b)
{
buf.append(b);
totalbytes++;
if (buf.size() > 65536) flush();
}
size_t size() { return totalbytes; }
uint32_t crc32() { flush(); return crc; }
void cancel() { f.resize(0); }
};
class membufwriter {
arrayvieww<byte> buf;
size_t bufpos;
uint32_t crc;
size_t crcpos;
public:
membufwriter(arrayvieww<byte> buf) : buf(buf), bufpos(0), crc(0), crcpos(0) {}
void write(arrayview<byte> bytes)
{
memcpy(buf.slice(bufpos, buf.size()-bufpos).ptr(), bytes.ptr(), bytes.size());
bufpos += bytes.size();
}
void write(byte b)
{
buf[bufpos++] = b;
}
size_t size() { return bufpos; }
uint32_t crc32()
{
crc = crc32_update(buf.slice(crcpos, bufpos-crcpos), crc);
crcpos = bufpos;
return crc;
}
};
//Deprecated

View File

@ -1,57 +1,55 @@
#include "patch.h"
namespace patch {
test("filebufreader")
{
array<byte> bytes;
for (int i=0;i<65536;i++)
{
bytes[i] = i ^ i>>8;
}
file f = file::mem(bytes);
assert_eq(f.size(), 65536);
filebufreader br = f;
size_t pos = 0;
#define EXPECT(n) \
do { \
assert_eq(br.remaining(), 65536-pos); \
arrayview<byte> var = br.read(n); \
for (size_t i=0;i<n;i++) \
assert_eq(var[i], bytes[pos+i]); \
pos += n; \
assert_eq(br.remaining(), 65536-pos); \
assert_eq(br.crc32(), crc32(bytes.slice(0, pos))); \
} while(0)
EXPECT(1);
EXPECT(6);
EXPECT(14);
EXPECT(4000);
EXPECT(4000); // cross the buffers
assert_eq(br.read(), bytes[pos++]); // single-byte reader
assert_eq(br.read(), bytes[pos++]);
assert_eq(br.read(), bytes[pos++]);
assert_eq(br.read(), bytes[pos++]);
EXPECT(16000);
assert(br.read(65536).ptr() == NULL);
EXPECT(4000);
}
test("streamreader")
{
array<byte> bytes;
for (int i=0;i<65536;i++)
{
bytes[i] = i ^ i>>8;
}
file f = file::mem(bytes);
assert_eq(f.size(), 65536);
streamreader r = f;
test_skip("not yet");
}
//test("filebufreader")
//{
// array<byte> bytes;
// for (int i=0;i<65536;i++)
// {
// bytes[i] = i ^ i>>8;
// }
//
// memstream stream = bytes;
//
// size_t pos = 0;
//#define EXPECT(n) \
// do { \
// assert_eq(stream.remaining(), 65536-pos); \
// arrayview<byte> var = stream.read(n); \
// for (size_t i=0;i<n;i++) \
// assert_eq(var[i], bytes[pos+i]); \
// pos += n; \
// assert_eq(stream.remaining(), 65536-pos); \
// /* assert_eq(stream.crc32(), crc32(bytes.slice(0, pos))); */ \
// } while(0)
//
// EXPECT(1);
// EXPECT(6);
// EXPECT(14);
// EXPECT(4000);
// EXPECT(4000); // cross the buffers
// assert_eq(stream.read(), bytes[pos++]); // single-byte reader
// assert_eq(stream.read(), bytes[pos++]);
// assert_eq(stream.read(), bytes[pos++]);
// assert_eq(stream.read(), bytes[pos++]);
// EXPECT(16000);
// assert(br.read(65536).ptr() == NULL);
// EXPECT(4000);
//}
//
//test("streamreader")
//{
// array<byte> bytes;
// for (int i=0;i<65536;i++)
// {
// bytes[i] = i ^ i>>8;
// }
// file f = file::mem(bytes);
// assert_eq(f.size(), 65536);
//
// //streamreader r = f;
// test_skip("not yet");
//}
static bool testips;
static bool testbps;
@ -131,16 +129,16 @@ static void simpletests()
testcall(createtest(seq256, seq128, base+trunc, 23));
testcall(createtest(seq128, seq256, base+record+128, 153));
testcall(createtest(empty, seq256nul4, base+record+255+4, 282));
testcall(createtest(empty, seq256nul5, base+record+255+5, 283));
testcall(createtest(empty, seq256nul6, base+record+255+6, 282));
testcall(createtest(empty, seq256nul5, base+record+255+5, 283)); // strange how this one is bigger
testcall(createtest(empty, seq256nul6, base+record+255+6, 282)); // guess the heuristics don't like EOF
testcall(createtest(empty, seq256nul7, base+record+255+record+1, 282));
testcall(createtest(empty, seq256b4, base+record+255+4, 282));
testcall(createtest(empty, seq256b5, base+record+255+5, 283));
testcall(createtest(empty, seq256b6, base+record+255+6, 284));
testcall(createtest(empty, seq256b7, base+record+255+record+1, 284));
testcall(createtest(empty, eof1, base+record+2, 57));
//if (testips) // don't need these for BPS, 0x454F46 isn't significant there
{ // one's enough, for testing large files
if (testips) // don't need these for BPS, 0x454F46 isn't significant there
{ // big files are sufficiently tested with the bigones tests
testcall(createtest(empty, eof1, base+record+2, 57));
testcall(createtest(empty, eof2, base+record+2+rle, 59));
testcall(createtest(empty, eof3, base+record+2, 55));
testcall(createtest(empty, eof4, base+record+2, 58));
@ -154,7 +152,7 @@ test("IPS")
testips=true;
testbps=false;
simpletests();
//simpletests();
}
test("BPS")
@ -162,7 +160,7 @@ test("BPS")
testips=false;
testbps=true;
simpletests();
//simpletests();
}
test("the big ones")
@ -180,15 +178,15 @@ test("the big ones")
array<byte> smwhack;
bps::apply(file::mem(smw_bps), file::mem(smw), file::mem(smwhack));
testcall(createtest(smw, smwhack, 3302980, 2077386));
//testcall(createtest(smw, smwhack, 3302980, 2077386));
array<byte> sm64hack;
bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack));
testcall(createtest(sm64, sm64hack, -1, 6788133));
//array<byte> sm64hack;
//bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack));
//testcall(createtest(sm64, sm64hack, -1, 6788133));
//this is the only UPS test, UPS is pretty much an easter egg in Flips
array<byte> dlhack;
ups::apply(file::mem(dl_ups), file::mem(dl), file::mem(dlhack));
testcall(createtest(dl, dlhack, 852134, 817190));
//array<byte> dlhack;
//ups::apply(file::mem(dl_ups), file::mem(dl), file::mem(dlhack));
//testcall(createtest(dl, dlhack, 852134, 817190));
}
}

View File

@ -2,124 +2,84 @@
namespace patch { namespace ups {
//TODO: HEAVY cleanups needed here
static uint32_t read32(uint8_t * ptr)
{
uint32_t out;
out =ptr[0];
out|=ptr[1]<<8;
out|=ptr[2]<<16;
out|=ptr[3]<<24;
return out;
}
#define error(which) do { error=which; goto exit; } while(0)
#define assert_sum(a,b) do { if (SIZE_MAX-(a)<(b)) error(e_too_big); } while(0)
#define assert_shift(a,b) do { if (SIZE_MAX>>(b)<(a)) error(e_too_big); } while(0)
result apply(const file& patch_, const file& source_, file& target_)
{
if (patch_.size()<4+2+12) return e_broken;
struct mem patch = patch_.mmap();
struct mem in = source_.mmap();
struct mem out_;
struct mem * out = &out_;
arrayview<byte> patchmem = patch_.mmap();
memstream patch = patchmem;
arrayview<byte> inmem = source_.mmap();
memstream in = inmem;
result error;
out->len=0;
out->ptr=NULL;
if (true)
{
#define readpatch8() (*(patchat++))
#define readin8() (*(inat++))
#define writeout8(byte) (*(outat++)=byte)
#define decodeto(var) \
do { \
var=0; \
unsigned int shift=0; \
while (true) \
{ \
uint8_t next=readpatch8(); \
assert_shift(next&0x7F, shift); \
size_t addthis=(next&0x7F)<<shift; \
assert_sum(var, addthis); \
var+=addthis; \
if (next&0x80) break; \
shift+=7; \
assert_sum(var, 1U<<shift); \
var+=1<<shift; \
} \
safeint<size_t> ret = patch.bpsnum(); \
if (!ret.valid()) error(e_too_big); \
var = ret.val(); \
} while(false)
bool backwards=false;
uint8_t * patchat=patch.ptr;
uint8_t * patchend=patch.ptr+patch.len-12;
if (readpatch8()!='U') error(e_broken);
if (readpatch8()!='P') error(e_broken);
if (readpatch8()!='S') error(e_broken);
if (readpatch8()!='1') error(e_broken);
if (patch.u8()!='U') error(e_broken);
if (patch.u8()!='P') error(e_broken);
if (patch.u8()!='S') error(e_broken);
if (patch.u8()!='1') error(e_broken);
size_t inlen;
size_t outlen;
decodeto(inlen);
decodeto(outlen);
if (inlen!=in.len)
if (inlen!=in.size())
{
size_t tmp=inlen;
inlen=outlen;
outlen=tmp;
backwards=true;
}
if (inlen!=in.len) error(e_not_this);
if (inlen!=in.size()) error(e_not_this);
out->len=outlen;
out->ptr=(uint8_t*)malloc(outlen);
memset(out->ptr, 0, outlen);
array<byte> outmem;
outmem.resize(outlen);
membufwriter out = outmem;
//uint8_t * instart=in.ptr;
uint8_t * inat=in.ptr;
uint8_t * inend=in.ptr+in.len;
//uint8_t * outstart=out->ptr;
uint8_t * outat=out->ptr;
uint8_t * outend=out->ptr+out->len;
while (patchat<patchend)
while (patch.remaining() > 12)
{
size_t skip;
decodeto(skip);
size_t skip_fast = min(skip, outlen-out.size(), in.remaining());
out.write(in.bytes(skip_fast));
skip -= skip_fast;
while (skip>0)
{
uint8_t out;
if (inat>=inend) out=0;
else out=readin8();
if (outat<outend) writeout8(out);
uint8_t outb = in.u8_or(0);
if (out.size()<outlen) out.write(outb);
skip--;
}
uint8_t tmp;
do
{
tmp=readpatch8();
uint8_t out;
if (inat>=inend) out=0;
else out=readin8();
if (outat<outend) writeout8(out^tmp);
tmp=patch.u8();
uint8_t outb = in.u8_or(0);
if (out.size()<outlen) out.write(outb^tmp);
else if (outb != 0) error(e_broken);
}
while (tmp);
}
if (patchat!=patchend) error(e_broken);
while (outat<outend) writeout8(0);
while (inat<inend) (void)readin8();
if (patch.remaining()!=12) error(e_broken);
uint32_t crc_in_expected=read32(patchat);
uint32_t crc_out_expected=read32(patchat+4);
uint32_t crc_patch_expected=read32(patchat+8);
uint32_t crc_in=crc32(inmem);
uint32_t crc_out=crc32(outmem);
uint32_t crc_patch=crc32(patchmem.slice(0, patchmem.size()-4));
uint32_t crc_in=crc32(in.v());
uint32_t crc_out=crc32(out->v());
uint32_t crc_patch=crc32(patch.v().slice(0, patch.len-4));
uint32_t crc_in_expected=patch.u32();
uint32_t crc_out_expected=patch.u32();
uint32_t crc_patch_expected=patch.u32();
if (inlen==outlen)
{
@ -144,23 +104,16 @@ result apply(const file& patch_, const file& source_, file& target_)
}
if (crc_patch!=crc_patch_expected) error(e_broken);
target_.write(out->v());
free(out->ptr);
patch_.unmap(patch.v());
source_.unmap(in.v());
patch_.unmap(patchmem);
source_.unmap(inmem);
target_.write(outmem);
return e_ok;
#undef read8
#undef decodeto
#undef write8
}
exit:
free(out->ptr);
patch_.unmap(patch.v());
source_.unmap(in.v());
out->len=0;
out->ptr=NULL;
patch_.unmap(patchmem);
source_.unmap(inmem);
return error;
}