Rewrite and optimize IPS creator (it's smaller too!)

2026-09-03 00:14:49 -05:00 · 2016-12-25 18:34:49 +01:00
parent 14ea216a2b
commit d8e4ec5fb7
6 changed files with 175 additions and 154 deletions
--- a/arlib/Makefile
+++ b/arlib/Makefile
@@ -41,7 +41,7 @@ CXXFLAGS = $(CFLAGS)
 LD = g++
 LFLAGS =
 OBJNAME =
-CCXXFLAGS = -fvisibility=hidden -Wall -Og
+CCXXFLAGS = -fvisibility=hidden -Wall
 ifneq ($(EXCEPTIONS),1)
  CCXXFLAGS += -fno-exceptions
 endif
@@ -91,10 +91,13 @@ endif
 OPTFLAGS := -Os -fomit-frame-pointer -fmerge-all-constants -fvisibility=hidden
 OPTFLAGS += -fno-unwind-tables -fno-asynchronous-unwind-tables
 OPTFLAGS += -ffunction-sections -fdata-sections
-OPTFLAGS += -Werror
+OPTFLAGS += -Werror -DNDEBUG

-ifeq ($(OPT),1)
-  CFLAGS += $(OPTFLAGS)
+ifneq ($(OPT),0)
+  CONF_CFLAGS += $(OPTFLAGS)
+  ifeq ($(OPT),spd)
+    CONF_CFLAGS += -O3
+  endif
  LFLAGS += -Wl,--gc-sections -s
  DEBUG = 0
  OBJNAME += -opt
--- a/arlib/array.h
+++ b/arlib/array.h
@@ -339,6 +339,44 @@ public:
 		return *this;
 	}
 	
+	array<T>& operator+=(arrayview<T> other)
+	{
+		size_t prevcount = this->count;
+		size_t othercount = other.size();
+		
+		const T* src;
+		T* dst;
+		
+		if (other.ptr() >= this->ptr() && other.ptr() < this->ptr()+this->size())
+		{
+			size_t start = other.ptr()-this->ptr();
+			
+			resize_grow_noinit(prevcount + othercount);
+			src = this->items+start;
+			dst = this->items+prevcount;
+		}
+		else
+		{
+			resize_grow_noinit(prevcount + othercount);
+			src = other.ptr();
+			dst = this->items+prevcount;
+		}
+		
+		if (this->trivial_copy)
+		{
+			memcpy(dst, src, sizeof(T)*othercount);
+		}
+		else
+		{
+			for (size_t i=0;i<othercount;i++)
+			{
+				new(&dst[i]) T(src[i]);
+			}
+		}
+		
+		return *this;
+	}
+	
 	~array()
 	{
 		for (size_t i=0;i<this->count;i++) this->items[i].~T();
@@ -354,29 +392,6 @@ public:
 		ret.resize_grow_noinit(count);
 		return ret;
 	}
-	
-	array<T>& operator+=(arrayview<T> other)
-	{
-		//TODO: x+=x doesn't work
-		size_t prevcount = this->count;
-		size_t othercount = other.size();
-		
-		resize_grow_noinit(prevcount + othercount);
-		
-		if (this->trivial_copy)
-		{
-			memcpy(this->items+prevcount, other.ptr(), sizeof(T)*othercount);
-		}
-		else
-		{
-			for (size_t i=0;i<othercount;i++)
-			{
-				new(&this->items[prevcount + i]) T(other[i]);
-			}
-		}
-		
-		return *this;
-	}
 };


--- a/patch/bps.cpp
+++ b/patch/bps.cpp
@@ -19,7 +19,7 @@
 namespace patch { namespace bps {
 enum { SourceRead, TargetRead, SourceCopy, TargetCopy };

-result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, bool accept_wrong_input)
+result apply(arrayview<byte> patchmem, arrayview<byte> inmem, array<byte>& outmem, bool accept_wrong_input)
 {
 	if (patchmem.size()<4+3+12) return e_broken;
 	
@@ -44,7 +44,7 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 		uint32_t crc_out_e = checks.u32();
 		uint32_t crc_patch_e = checks.u32();
 		
-		uint32_t crc_in_a = crc32(in);
+		uint32_t crc_in_a = crc32(inmem);
 		uint32_t crc_patch_a = crc32(patchmem.slice(0, patchmem.size()-4));
 		
 		if (crc_patch_a != crc_patch_e) error(e_broken);
@@ -55,14 +55,13 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 		size_t outlen;
 		decodeto(outlen);
 		
-		if (inlen!=in.size() || (crc_in_a!=crc_in_e && !accept_wrong_input))
+		if (inlen!=inmem.size() || (crc_in_a!=crc_in_e && !accept_wrong_input))
 		{
-			if (in.size()==outlen && crc_in_a==crc_out_e) error=e_to_output;
+			if (inmem.size()==outlen && crc_in_a==crc_out_e) error=e_to_output;
 			else error=e_not_this;
-			if (inlen==in.size() && !accept_wrong_input) goto exit;
+			if (inlen==inmem.size() && !accept_wrong_input) goto exit;
 		}
 		
-		array<byte> outmem;
 		outmem.reserve_noinit(outlen);
 		membufwriter out = outmem;
 		
@@ -85,8 +84,8 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 			{
 				case SourceRead:
 				{
-					if (out.pos()+length > in.size()) error(e_broken);
-					out.write(in.slice(out.pos(), length));
+					if (out.pos()+length > inmem.size()) error(e_broken);
+					out.write(inmem.slice(out.pos(), length));
 				}
 				break;
 				case TargetRead:
@@ -102,8 +101,8 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 					size_t distance=encodeddistance>>1;
 					if ((encodeddistance&1)==0)
 					{
-						if (inreadat+length > in.size()) error(e_broken);
 						inreadat+=distance;
+						if (inreadat+length > inmem.size()) error(e_broken);
 					}
 					else
 					{
@@ -111,7 +110,7 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 						inreadat-=distance;
 					}
 					
-					out.write(in.slice(inreadat, length));
+					out.write(inmem.slice(inreadat, length));
 					inreadat+=length;
 				}
 				break;
@@ -122,9 +121,9 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 					size_t distance=encodeddistance>>1;
 					if ((encodeddistance&1)==0)
 					{
-						if (distance+outreadat > out.pos()) error(e_broken);
-						if (outreadat+length > out.size()) error(e_broken);
 						outreadat+=distance;
+						if (outreadat+length > out.size()) error(e_broken);
+						if (outreadat >= out.pos()) error(e_broken);
 					}
 					else
 					{
@@ -132,8 +131,16 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 						outreadat-=distance;
 					}
 					
-					out.write(outmem.slice(outreadat, length));
-					outreadat+=length;
+					size_t outreadstart = outreadat;
+					outreadat += length;
+					
+					while (outreadstart+length > out.pos())
+					{
+						size_t chunk = out.pos()-outreadstart;
+						out.write(outmem.slice(outreadstart, chunk));
+						length -= chunk;
+					}
+					out.write(outmem.slice(outreadstart, length));
 				}
 				break;
 			}
@@ -154,7 +161,7 @@ result apply(arrayview<byte> patchmem, arrayview<byte> in, array<byte>& out, boo
 	}
 	
 exit:
-	out.resize(0);
+	outmem.resize(0);
 	return error;
 }

--- a/patch/ips.cpp
+++ b/patch/ips.cpp
@@ -29,7 +29,7 @@ result apply(arrayview<byte> patchmem, const file& in, array<byte>& out)
 			if (patch.remaining() < 2+1+3) error(e_broken);
 			size = patch.u16be();
 			uint8_t b = patch.u8();
-			if (!size) error(e_broken); // is this defined?
+			if (!size) error(e_broken); // don't know if this is defined, probably isn't
 			
 			out.reserve(offset+size);
 			if (!anychanges &&
@@ -52,8 +52,8 @@ result apply(arrayview<byte> patchmem, const file& in, array<byte>& out)
 	}
 	if (patch.remaining()==3)
 	{
-		uint32_t newsize = patch.u24();
-		if (newsize <= out.size() && !error) error = e_not_this;
+		uint32_t newsize = patch.u24be();
+		if (newsize >= in.size() && !error) error = e_not_this;
 		out.resize(newsize);
 	}
 	if (patch.remaining()!=0) error = e_damaged;
@@ -92,52 +92,55 @@ exit:

 //There are no known cases where LIPS wins over this.

-static result create(struct mem sourcemem, struct mem targetmem, struct mem * patchmem)
+result create(array<byte> source, arrayview<byte> target, array<byte>& patchmem)
 {
-	int sourcelen=sourcemem.len;
-	int targetlen=targetmem.len;
-	const unsigned char * source=sourcemem.ptr;
-	const unsigned char * target=targetmem.ptr;
-	
-	patchmem->ptr=NULL;
-	patchmem->len=0;
+	int truesourcelen=source.size();
+	int targetlen=target.size();
+	source.resize(target.size());
+	//const unsigned char * source=sourcemem.ptr();
+	//const unsigned char * target=targetmem.ptr();
 	
 	if (targetlen>=16777216) return e_too_big;
 	
 	int offset=0;
-	int outbuflen=4096;
-	unsigned char * out=(uint8_t*)malloc(outbuflen);
-	int outlen=0;
-#define write8(val) do { out[outlen++]=(val); if (outlen==outbuflen) { outbuflen*=2; out=(uint8_t*)realloc(out, outbuflen); } } while(0)
-#define write16(val) do { write8((val)>>8); write8((val)); } while(0)
-#define write24(val) do { write8((val)>>16); write8((val)>>8); write8((val)); } while(0)
-	write8('P');
-	write8('A');
-	write8('T');
-	write8('C');
-	write8('H');
+	
+	array<byte> out;
+	out.append('P');
+	out.append('A');
+	out.append('T');
+	out.append('C');
+	out.append('H');
 	int lastknownchange=0;
 	int lastwritten=0;
-	//int forcewrite=(targetlen>sourcelen?1:0);
+#define ENC24(n) (byte)((n)>>16), (byte)((n)>>8), (byte)((n)>>0)
+#define ENC16(n) (byte)((n)>>8), (byte)((n)>>0)
+#define ENC8(n) (byte)((n)>>0)
 	while (offset<targetlen)
 	{
-		while (offset<targetlen && (offset<sourcelen?source[offset]:0)==target[offset]) offset++;
-		//check how much we need to edit until it starts getting similar
+		//skip unchanged bytes
+		while (offset<targetlen && source[offset]==target[offset]) offset++;
+		
+		//how many bytes to edit
 		int thislen=0;
-		int consecutiveunchanged=0;
-		thislen=lastknownchange-offset;
+		thislen=lastknownchange-offset; // cache results of this loop
 		if (thislen<0) thislen=0;
-		while (true)
+		
+		int searchat = offset+thislen;
+		int unchangedtimer = 6;
+		//int consecutiveunchanged=0;
+		int stop = min(targetlen, offset+65535);
+		
+		while (searchat < stop)
 		{
-			int thisbyte=offset+thislen+consecutiveunchanged;
-			if (thisbyte<targetlen && (thisbyte<sourcelen?source[thisbyte]:0)==target[thisbyte]) consecutiveunchanged++;
-			else
+			byte b = source[searchat];
+			if (target[searchat++] == b)
 			{
-				thislen+=consecutiveunchanged+1;
-				consecutiveunchanged=0;
+				unchangedtimer--;
+				if (!unchangedtimer) break;
 			}
-			if (consecutiveunchanged>=6 || thislen>65535) break;
+			else unchangedtimer=6;
 		}
+		thislen = searchat-offset-6+unchangedtimer;
 		
 		//avoid premature EOF
 		if (offset==0x454F46)
@@ -147,11 +150,11 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
 		}
 		
 		lastknownchange=offset+thislen;
-		if (thislen>65535) thislen=65535;
+		
 		if (offset+thislen>targetlen) thislen=targetlen-offset;
 		if (offset==targetlen) continue;
 		
-		//check if RLE here is worthwhile
+		//check if starting a RLE here is worthwhile
 		int byteshere;
 		for (byteshere=0;byteshere<thislen && target[offset]==target[offset+byteshere];byteshere++) {}
 		if (byteshere==thislen)
@@ -162,7 +165,7 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
 			{
 				int pos=offset+byteshere+i-1;
 				if (pos>=targetlen || target[pos]!=thisbyte || byteshere+i>65535) break;
-				if (pos>=sourcelen || (pos<sourcelen?source[pos]:0)!=thisbyte)
+				if (pos>=truesourcelen || source[pos]!=thisbyte)
 				{
 					byteshere+=i;
 					thislen+=i;
@@ -173,39 +176,46 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
 		}
 		if ((byteshere>8-5 && byteshere==thislen) || byteshere>8)
 		{
-			write24(offset);
-			write16(0);
-			write16(byteshere);
-			write8(target[offset]);
+			byte bytes[] = { ENC24(offset), ENC16(0), ENC16(byteshere), target[offset] };
+			out += arrayview<byte>(bytes);
 			offset+=byteshere;
 			lastwritten=offset;
 		}
 		else
 		{
 			//check if we'd gain anything from ending the block early and switching to RLE
-			int byteshere=0;
-			int stopat=0;
-			while (stopat+byteshere<thislen)
+			//int byteshere=0;
+			//int stopat=0;
+			
+			//RLE is a win if
+			//- 13 consecutive bytes are same
+			//- 8 consecutive bytes are same, followed by 8 consecutive but different bytes (two RLE blocks)
+			//- 8 consecutive bytes are same, followed by end of the changed block
+			//for all of them, unchanged (compared to input) bytes before, after or between them count towards said 8 or 13 bytes
+			//however, counting unchanged bytes isn't implemented; it only gave me losses and infinite loops when I tried
+			//nor is 8+8, seems too rare to hit
+			
+			int rlestart = 0;
+			int rlebyte = -1;
+			for (int i=0;i<thislen;i++)
 			{
-				if (target[offset+stopat]==target[offset+stopat+byteshere]) byteshere++;
-				else
+				if (target[offset+i] != rlebyte) rlestart = i;
+				rlebyte = target[offset+i];
+				if (i-rlestart >= 12)
 				{
-					stopat+=byteshere;
-					byteshere=0;
-				}
-				if (byteshere>8+5 || //rle-worthy despite two ips headers
-						(byteshere>8 && stopat+byteshere==thislen) || //rle-worthy at end of data
-						(byteshere>8 && offset+stopat+byteshere+8 <= thislen &&
-							!memcmp(&target[offset+stopat+byteshere], &target[offset+stopat+byteshere+1], 9-1)))//rle-worthy before another rle-worthy
-				{
-					if (stopat) thislen=stopat;
-					break;//we don't scan the entire block if we know we'll want to RLE, that'd gain nothing.
+					thislen=rlestart;
+					break;
 				}
 			}
-			//don't write unchanged bytes at the end of a block if we want to RLE the next couple of bytes
+			if (thislen-rlestart >= 8)
+			{
+				thislen=rlestart;
+			}
+			
+			//don't copy unchanged bytes at the end of a block
 			if (offset+thislen!=targetlen)
 			{
-				while (offset+thislen-1<sourcelen && target[offset+thislen-1]==(offset+thislen-1<sourcelen?source[offset+thislen-1]:0))
+				while (target[offset+thislen-1]==source[offset+thislen-1])
 				{
 					thislen--;
 				}
@@ -217,66 +227,47 @@ static result create(struct mem sourcemem, struct mem targetmem, struct mem * pa
 				if (thislen==0xFFFF) thislen--;
 				else thislen++;
 			}
-			if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//still worth it?
+			
+			if (thislen>3 && !memcmp(&target[offset], &target[offset+1], thislen-1))//can we switch to RLE for these few bytes?
 			{
-				write24(offset);
-				write16(0);
-				write16(thislen);
-				write8(target[offset]);
+				byte bytes[] = { ENC24(offset), ENC16(0), ENC16(thislen), target[offset] };
+				out += arrayview<byte>(bytes);
 			}
 			else
 			{
-				write24(offset);
-				write16(thislen);
-				for (int i=0;i<thislen;i++)
-				{
-					write8(target[offset+i]);
-				}
+				byte bytes[] = { ENC24(offset), ENC16(thislen) };
+				out += arrayview<byte>(bytes);
+				out += target.slice(offset, thislen);
 			}
 			offset+=thislen;
 			lastwritten=offset;
 		}
 	}
-	if (sourcelen<targetlen && lastwritten!=targetlen)
+	
+	if (truesourcelen<targetlen && lastwritten!=targetlen)
 	{
 		if (targetlen-1==0x454F46)
 		{
-			write24(targetlen-2);
-			write16(2);
-			write8(target[targetlen-2]);
-			write8(target[targetlen-1]);
+			byte bytes[] = { ENC24(targetlen-2), ENC16(2), target[targetlen-2], target[targetlen-1] };
+			out += arrayview<byte>(bytes);
 		}
 		else
 		{
-			write24(targetlen-1);
-			write16(1);
-			write8(target[targetlen-1]);
+			byte bytes[] = { ENC24(targetlen-1), ENC16(1), target[targetlen-1] };
+			out += arrayview<byte>(bytes);
 		}
 	}
-	write8('E');
-	write8('O');
-	write8('F');
-	if (sourcelen>targetlen) write24(targetlen);
-#undef write
-	patchmem->ptr=out;
-	patchmem->len=outlen;
-	if (outlen==8) return e_identical;
+	byte bytes[] = { 'E', 'O', 'F', ENC24(targetlen) };
+	if (truesourcelen>targetlen) out+=arrayview<byte>(bytes, 6);
+	else out+=arrayview<byte>(bytes, 3);
+#undef ENC8
+#undef ENC16
+#undef ENC24
+	patchmem = std::move(out);
+	if (patchmem.size()==8) return e_identical;
 	return e_ok;
 }

-result create(const file& source, const file& target, file& patch)
-{
-	struct mem sourcemem = source.mmap();
-	struct mem targetmem = target.mmap();
-	struct mem patchmem;
-	result r = create(sourcemem, targetmem, &patchmem);
-	source.unmap(sourcemem.v());
-	target.unmap(targetmem.v());
-	patch.write(patchmem.v());
-	free(patchmem.ptr);
-	return r;
-}
-
 #if 0
 #warning Disable this in release versions.
 #include <stdio.h>
--- a/patch/patch.h
+++ b/patch/patch.h
@@ -26,15 +26,15 @@ enum result {
 	e_canceled   //Patch creation callback said cancel.
 };

+//All of these functions can be called with arrayview inputs and array& outputs, but
+// they give lower memory use and/or better performance if you follow the listed types.
+//For example, IPS and UPS application start with copying the input file to the output;
+// if you give them a file object directly, they'll read it straight from disk to the target buffer.
+
 namespace ips {
 result apply(arrayview<byte> patch, const file& in, array<byte>& out);
-static inline result apply(arrayview<byte> patch, arrayview<byte> in, array<byte>& out)
-{
-	file inf = file::mem(in);
-	return apply(patch, inf, out);
-}
-result create(const file& source, const file& target, file& patch);
-static inline result create(const file& source, const file& target, file&& patch) { return create(source, target, (file&)patch); }
+static inline result apply(arrayview<byte> patch, arrayview<byte> in, array<byte>& out) { return apply(patch, file::mem(in), out); }
+result create(array<byte> source, arrayview<byte> target, array<byte>& patch);
 }

 namespace ups {
--- a/patch/test.cpp
+++ b/patch/test.cpp
@@ -62,7 +62,7 @@ static void createtest(arrayview<byte> a, arrayview<byte> b, size_t ipssize, siz
 	if (testips && b.size()<=16777216)
 	{
 		array<byte> patch;
-		result r = ips::create(file::mem(a), file::mem(b), file::mem(patch));
+		result r = ips::create(a, b, patch);
 		if (r!=e_identical) assert_eq(r, e_ok);
 		array<byte> b2;
 		r = ips::apply(patch, a, b2);
@@ -98,6 +98,7 @@ static void createtest(arrayview<byte> a, arrayview<byte> b, size_t ipssize, siz
 	}
 }

+MAYBE_UNUSED
 static void simpletests()
 {
 	array<byte> empty;
@@ -168,7 +169,8 @@ test("BPS")
 test("the big ones")
 {
 	testips=true;
-	testbps=true;
+	//testbps=true;
+	testbps=false;
 	
 	array<byte> smw      = file::read("patch/test/smw.sfc");
 	array<byte> smw_bps  = file::read("patch/test/smwcp.bps");
@@ -179,20 +181,23 @@ test("the big ones")
 	if (!smw || !smw_bps || !dl || !dl_ups || !sm64 || !sm64_bps) test_skip("test files not present; see patch/test/readme.txt");
 	result r;
 	
-	//array<byte> smwhack;
-	//r = bps::apply(smw_bps, smw, smwhack);
-	//assert_eq(r, e_ok);
-	//testcall(createtest(smw, smwhack, 3302980, 2077386));
+	array<byte> smwhack;
+	r = bps::apply(smw_bps, smw, smwhack);
+	assert_eq(r, e_ok);
+	assert_eq(smwhack.size(), 4194304);
+	testcall(createtest(smw, smwhack, 3302746, 2077386));
 	
 	//array<byte> sm64hack;
-	//r = bps::apply(file::mem(sm64_bps), file::mem(sm64), file::mem(sm64hack));
+	//r = bps::apply(sm64_bps, sm64, sm64hack);
 	//assert_eq(r, e_ok);
+	//assert_eq(sm64hack.size(), 50331648);
 	//testcall(createtest(sm64, sm64hack, -1, 6788133));
 	
 	//this is the only UPS test, UPS is pretty much an easter egg in Flips
 	//array<byte> dlhack;
 	//r = ups::apply(dl_ups, dl, dlhack);
 	//assert_eq(r, e_ok);
+	//assert_eq(dlhack.size(), 3145728);
 	//array<byte> dl2;
 	//r = ups::apply(dl_ups, dlhack, dl2);
 	//assert_eq(r, e_ok);