Optimize Gameboy payloads for size by generating the payloads at runtime and generate binary patches

This commit moves payload_builder and the z80_asm code to the data-generator subproject in order to generate the gameboy payloads
at compile time instead of at runtime.

In addition, we select a couple of base payloads (more than 1 for compressibility's sake) and generate binary patches to transform them into
other payloads. We then generate a binary file with both the base payload and binary patches and compress these files with zx0.

This reduces the rom size by about 8 KB.

include/payload_file_reader.h

@@ -0,0 +1,17 @@
+#ifndef _PAYLOAD_FILE_READER_H
+#define _PAYLOAD_FILE_READER_H
+
+#include <cstdint>
+
+class payload_file_reader {
+public:
+    payload_file_reader(const uint8_t *file_buffer, uint16_t buffer_size);
+
+    bool read_payload(uint8_t *buffer, uint8_t language, uint8_t game_variant);
+protected:
+private:
+    const uint8_t *file_buffer_;
+    const uint8_t *file_buffer_end_;
+};
+
+#endif

include/pokemon_party.h

@@ -33,6 +33,9 @@ public:
     byte box_data_array[0x462];
 
 private:
+    void init_payload();
+
+    u8 current_payload[PAYLOAD_SIZE];
     int last_error;
     bool has_new_pkmn = false;
     bool contains_mythical = false;

source/gameboy_colour.cpp

@@ -15,7 +15,17 @@
 #include "global_frame_controller.h"
 #include "background_engine.h"
 #include "sprite_data.h"
-#include "payload_builder.h"
+
+#define DATA_PER_PACKET 8
+#define PACKET_DATA_START 2
+#define PACKET_DATA_AT(i) (PACKET_DATA_START + (i * 2))
+#define PACKET_FLAG_AT(i) (PACKET_DATA_START + (i * 2) + 1)
+#define PACKET_CHECKSUM (PACKET_DATA_START + (2 * DATA_PER_PACKET))
+#define PACKET_LOCATION_UPPER (PACKET_CHECKSUM + 1)
+#define PACKET_LOCATION_LOWER (PACKET_CHECKSUM + 2)
+
+// 0xFD, 0x00, data bytes per packet, flag bytes per packet, the checksum, and two location bytes
+#define PACKET_SIZE (1 + 1 + (2 * DATA_PER_PACKET) + 1 + 2) // Originally 13
 
 #define TIMEOUT 2
 #define TIMEOUT_ONE_LENGTH 1000000 // Maybe keep a 10:1 ratio between ONE and TWO?

source/main.cpp

@@ -213,6 +213,59 @@ void first_load_message(void)
 	tte_erase_rect(0, 0, H_MAX, V_MAX);
 }
 
+#include "translated_text.h"
+
+#define TIMER_ENABLE 0x80
+#define TIMER_CASCADE 0x4
+#define TIMER_FREQ_1 0x0    // 16.78 MHz
+#define TIMER_FREQ_64 0x1   // 262,144 Hz
+#define TIMER_FREQ_256 0x2  // 65,536 Hz
+#define TIMER_FREQ_1024 0x3 // 16,384 Hz
+
+int test_decompress()
+{
+	uint16_t charset[256];
+
+    // Reset both timers
+    REG_TM0CNT = 0;
+    REG_TM1CNT = 0;
+    REG_TM0D = 0;
+    REG_TM1D = 0;
+
+	// Set up TIMER0: count with no prescaler
+	REG_TM0CNT = TIMER_ENABLE | TIMER_FREQ_1;
+	// Set up TIMER1: cascade mode (increment when TIMER0 overflows)
+	REG_TM1CNT = TIMER_ENABLE | TIMER_CASCADE;
+
+	load_localized_charset(charset, 3, ENG_ID);
+
+    // Read combined 32-bit timer value
+    const u32 ticks = ((u32)REG_TM1D << 16) | REG_TM0D;
+
+    // Stop timers
+    REG_TM0CNT = 0;
+    REG_TM1CNT = 0;
+
+	create_textbox(4, 1, 160, 80, true);
+
+	ptgb_write_debug(charset, "Test results:\n\nDecompress: ", true);
+	ptgb_write_debug(charset, ptgb::to_string(ticks * 1000 / 16777), true);
+	ptgb_write_debug(charset, " usec\n", true);
+
+
+	while (true)
+	{
+		if (key_hit(KEY_B))
+		{
+			hide_text_box();
+			reset_textbox();
+			return 0;
+		}
+		global_next_frame();
+	}
+	return 0;
+}
+
 int credits()
 {
 	u8 text_decompression_buffer[2048];
@@ -249,6 +302,11 @@ int credits()
 			curr_credits_num++;
 			update = true;
 		}
+		if(key_hit(KEY_SELECT))
+		{
+			return test_decompress();
+		}
+	#if 0
 		if (ENABLE_DEBUG_SCREEN && key_hit(KEY_SELECT))
 		{
 			char hexBuffer[16];
@@ -325,6 +383,7 @@ int credits()
 				global_next_frame();
 			}
 		}
+#endif
 
 		global_next_frame();
 	}

source/payload_file_reader.cpp

@@ -0,0 +1,132 @@
+#include "payloads/payload_file_reader.h"
+#include "payloads/payload_file_writer.h"
+#include "gb_rom_values/base_gb_rom_struct.h"
+#include <cstring>
+
+// Reads a payload from the specified buffer pointer based on generation, language, and game variant
+static const uint8_t* read_payload_metadata(const uint8_t *buffer, payload_metadata &out_metadata)
+{
+    out_metadata.language = buffer[0];
+    out_metadata.game_variant = buffer[1];
+    out_metadata.size = *(uint16_t *)(buffer + 2);
+    
+    return buffer + sizeof(struct payload_metadata); // Move past the metadata header
+}
+
+// Reads the payload data from the file buffer and returns a pointer to where we stopped reading
+static const uint8_t* read_payload_data(const uint8_t *input_buffer, uint8_t *output_buffer, size_t size)
+{
+    memcpy(output_buffer, input_buffer, size);
+    return input_buffer + size;
+}
+
+/**
+ * @brief Calculate the next 4 byte aligned offset.
+ * 
+ * The Gameboy Advance needs alignment to read data correctly.
+ * For (u)int16_t data types, for example, the data needs to be aligned to 2 bytes
+ * For (u)int32_t data types, the data needs to be aligned to 4 bytes.
+ */
+static uint16_t get_aligned_offset(uint16_t current_offset)
+{
+    // Align to 4-byte boundary
+    return (current_offset + 3) & ~3; // Round up to the next multiple of 4
+}
+
+static const uint8_t* skip_to_aligned_offset(const uint8_t* buffer_start, const uint8_t* input_buffer)
+{
+    // Skip to the next 4-byte aligned offset
+    const uint16_t offset = static_cast<uint16_t>(input_buffer - buffer_start);
+    return buffer_start + get_aligned_offset(offset);
+}
+
+static const uint8_t* read_binary_patch(const uint8_t *input_buffer, uint8_t *output_buffer, uint16_t &out_offset, uint16_t &out_patch_size, bool skip)
+{
+    out_patch_size = static_cast<uint16_t>(input_buffer[0]) | (static_cast<uint16_t>(input_buffer[1]) << 8);
+
+    if(skip)
+    {
+        return input_buffer + 4 + out_patch_size;
+    }
+
+    input_buffer += 2; // Move past the size
+
+    out_offset = static_cast<uint16_t>(input_buffer[0]) | (static_cast<uint16_t>(input_buffer[1]) << 8);
+    input_buffer += 2; // Move past the offset
+
+    memcpy(output_buffer, input_buffer, out_patch_size);
+    return input_buffer + out_patch_size; // Return the pointer to where we stopped reading
+}
+
+payload_file_reader::payload_file_reader(const uint8_t *file_buffer, uint16_t buffer_size)
+    : file_buffer_(file_buffer)
+    , file_buffer_end_(file_buffer + buffer_size)
+{
+}
+
+bool payload_file_reader::read_payload(uint8_t *buffer, uint8_t language, uint8_t game_variant)
+{
+    uint8_t binary_patch_buffer[512]; // Buffer to hold binary patches
+    payload_metadata base_payload_metadata;
+    payload_metadata current_variant_metadata;
+    const uint8_t *cur = file_buffer_;
+    uint16_t i;
+    uint16_t binary_patch_offset;
+    uint16_t binary_patch_size;
+    bool found = false;
+
+    // first read the base payload metadata and the base payload
+    cur = read_payload_metadata(cur, base_payload_metadata);
+    cur = skip_to_aligned_offset(file_buffer_, cur);
+
+    cur = read_payload_data(cur, buffer, base_payload_metadata.size);
+    cur = skip_to_aligned_offset(file_buffer_, cur);
+
+    // check if the base payload is actually the payload we requested.
+    found = (base_payload_metadata.language == language &&
+        base_payload_metadata.game_variant == game_variant);
+    
+    if(found)
+    {
+        return true;
+    }
+
+    // now start reading the other payloads (which consist of binary patches)
+    while(cur < file_buffer_end_)
+    {
+        cur = read_payload_metadata(cur, current_variant_metadata);
+        cur = skip_to_aligned_offset(file_buffer_, cur);
+
+        found = (current_variant_metadata.language == language &&
+                current_variant_metadata.game_variant == game_variant);
+
+        // we need to read every binary patch, because we either need to apply them or skip them
+        for(i = 0; i < current_variant_metadata.size; ++i)
+        {
+            cur = read_binary_patch(cur, binary_patch_buffer, binary_patch_offset, binary_patch_size, !found);
+            cur = skip_to_aligned_offset(file_buffer_, cur);
+
+            if(found)
+            {
+                // apply the binary patch to the payload data
+                if(binary_patch_offset + binary_patch_size <= PAYLOAD_SIZE)
+                {
+                    memcpy(buffer + binary_patch_offset, binary_patch_buffer, binary_patch_size);
+                }
+                else
+                {
+                    return false; // Patch exceeds payload size
+                }
+            }
+            // else -> the patch is skipped, we've just read it to move the cur pointer forward
+        }
+
+        if(found)
+        {
+            return true;
+        }
+    }
+
+    return false; // No matching payload found
+}
+

source/pokemon_party.cpp

@@ -7,10 +7,13 @@
 #include "gb_rom_values/gb_rom_values.h"
 #include "sprite_data.h"
 #include "box_menu.h"
-#include "payload_builder.h"
 #include "zx0_decompressor.h"
+#include "payload_file_reader.h"
 #include "gb_rom_values_eng_zx0_bin.h"
 #include "gb_rom_values_fre_zx0_bin.h"
+#include "gb_gen1_payloads_RB_zx0_bin.h"
+#include "gb_gen1_payloads_Y_zx0_bin.h"
+#include "gb_gen2_payloads_zx0_bin.h"
 
 static const byte gen1_rb_debug_box_data[0x462] = {
 	// Num of Pokemon
@@ -176,12 +179,12 @@ void Pokemon_Party::start_link()
 		u16 debug_charset[256];
 
 		load_localized_charset(debug_charset, 3, ENG_ID);
-		init_payload(curr_gb_rom, TRANSFER, false);
+		init_payload();
 
 		setup(debug_charset);
 		memset(box_data_array, 0, curr_gb_rom.box_data_size);
 
-		last_error = loop(&box_data_array[0], get_payload(), &curr_gb_rom, simple_pkmn_array, debug_charset, false);
+		last_error = loop(&box_data_array[0], current_payload, &curr_gb_rom, simple_pkmn_array, debug_charset, false);
 	}
 }
 
@@ -193,7 +196,7 @@ void Pokemon_Party::continue_link(bool cancel_connection)
 
 		load_localized_charset(debug_charset, 3, ENG_ID);
 
-		last_error = loop(&box_data_array[0], get_payload(), &curr_gb_rom, simple_pkmn_array, debug_charset, cancel_connection);
+		last_error = loop(&box_data_array[0], current_payload, &curr_gb_rom, simple_pkmn_array, debug_charset, cancel_connection);
 	}
 }
 
@@ -338,4 +341,34 @@ bool Pokemon_Party::get_contains_invalid()
 bool Pokemon_Party::get_contains_missingno()
 {
 	return contains_missingno;
+}
+
+void Pokemon_Party::init_payload()
+{
+	u8 decompression_buffer[1512];
+	const u8 *payload_src;
+
+	//WARNING: Ensure sure decompression_buffer is large enough!
+
+	if(curr_gb_rom.generation == 1)
+	{
+		if(curr_gb_rom.version == YELLOW_ID)
+		{
+			payload_src = gb_gen1_payloads_Y_zx0_bin;
+		}
+		else
+		{
+			payload_src = gb_gen1_payloads_RB_zx0_bin;
+		}
+	}
+	else // if(curr_gb_rom.generation == 2)
+	{
+		payload_src = gb_gen2_payloads_zx0_bin;
+	}
+
+	zx0_decompressor_start(decompression_buffer, payload_src);
+	zx0_decompressor_read(zx0_decompressor_get_decompressed_size());
+
+	payload_file_reader payload_reader(decompression_buffer, zx0_decompressor_get_decompressed_size());
+	payload_reader.read_payload(current_payload, curr_gb_rom.language, curr_gb_rom.version);
 }

tools/data-generator/include/gb_rom_values/base_gb_rom_struct.h

@@ -44,6 +44,13 @@ extern "C"
 
 #define GB_TILE_WIDTH 20
 
+// TODO? : Since we moved payload_builder into data-generator, we no longer need most of the fields of GB_ROM in our PTGB rom.
+// It might be better to split it up into a struct that contains everything for data-generator and a separate one that gets
+// included into the ptgb rom, which only includes whatever PTGB needs at runtime. (which is probably a fraction of the data)
+// However, right now the compressed gb_rom_values_eng file is only 353 bytes and the gb_rom_values_fre is only 351 bytes.
+// Therefore we'd save at most about 0,5 KB if we do this optimization. So it might not be worth it right now.
+// If we expand the data at some point, however, it might be!
+
 // WARNING: We must be explicit in the type declarations of the struct members here.
 // After all: data-generator runs on x86 and the structs will be read by the GBA's ARM processor.
 // So we must ensure the sizes of the struct members, as well as padding, remains consistent across platforms.

tools/data-generator/include/payloads/binary_patch_generator.h

@@ -0,0 +1,33 @@
+#ifndef BINARY_PATCH_GENERATOR_H
+#define BINARY_PATCH_GENERATOR_H
+
+#include <vector>
+#include <cstdint>
+#include <cstddef>
+
+/// @brief This struct represents a single binary patch to transform a chunk of buffer1's data into buffer2's variant.
+typedef struct binary_patch_data
+{
+    /// the offset in the original buffer where the patch should be applied
+    uint16_t offset;
+    /// @brief the data to be patched in
+    std::vector<uint8_t> data;
+} binary_patch_data;
+
+typedef std::vector<binary_patch_data> binary_patch_list;
+
+/// @brief This class is used to generate binary patches by diffing 2 buffers.
+/// The buffers need to have the same size though.
+class binary_patch_generator
+{
+public:
+    binary_patch_list diff(const uint8_t* buffer1, const uint8_t* buffer2, size_t size) const;
+protected:
+private:
+};
+
+/// @brief This function writes a binary patch to the specified buffer.
+/// It's the caller's responsibility to ensure that the buffer is large enough to accommodate the patch.
+uint16_t write_binary_patch_to_buffer(const binary_patch_data &patch, uint8_t *buffer);
+
+#endif

tools/data-generator/include/payloads/payload_builder.h

@@ -19,9 +19,6 @@
 #define PACKET_SIZE (1 + 1 + (2 * DATA_PER_PACKET) + 1 + 2) // Originally 13
 
 
-void init_payload(GB_ROM curr_rom, int type, bool debug);
-
-/// @brief Note: call @see init_payload before using this function.
-byte* get_payload();
+void init_payload(byte *payload_buffer, const GB_ROM& curr_rom, int type, bool debug);
 
 #endif

tools/data-generator/include/payloads/payload_file_reader.h

@@ -0,0 +1,17 @@
+#ifndef _PAYLOAD_FILE_READER_H
+#define _PAYLOAD_FILE_READER_H
+
+#include <cstdint>
+
+class payload_file_reader {
+public:
+    payload_file_reader(const uint8_t *file_buffer, uint16_t buffer_size);
+
+    bool read_payload(uint8_t *buffer, uint8_t language, uint8_t game_variant);
+protected:
+private:
+    const uint8_t *file_buffer_;
+    const uint8_t *file_buffer_end_;
+};
+
+#endif

tools/data-generator/include/payloads/payload_file_writer.h

@@ -0,0 +1,35 @@
+#ifndef _PAYLOAD_FILE_WRITER_H
+#define _PAYLOAD_FILE_WRITER_H
+
+#include "payloads/binary_patch_generator.h"
+#include <vector>
+
+typedef std::vector<std::pair<uint16_t, binary_patch_list>> binary_patch_map;
+
+typedef struct payload_metadata
+{
+    uint8_t language;
+    uint8_t game_variant;
+    // for the first payload in the file, the size field represents the number of bytes, because it's the base payload
+    // for all subsequent payloads, however, the size field represents the number of binary patches!
+    uint16_t size;
+} payload_metadata;
+
+/// @brief This class writes a file containing a base payload alongside various
+/// payloads represented as binary patches that are supposed to be applied on top of the base payload.
+class payload_file_writer
+{
+public:
+    payload_file_writer();
+    void set_base_payload(uint8_t language, uint8_t game_variant, const uint8_t *payload, uint16_t size);
+    void add_binary_patches(uint8_t language, uint8_t game_variant, const binary_patch_list &patches);
+
+    void write_to_file(const char *path_to_file) const;
+protected:
+private:
+    const uint8_t *base_payload_;
+    payload_metadata base_payload_metadata_;
+    binary_patch_map binary_patches_;
+};
+
+#endif

tools/data-generator/include/payloads/z80_asm.h

@@ -2,7 +2,7 @@
 #define Z80_ASM_H
 
 #include <stdarg.h>
-#include "libstd_replacements.h"
+#include <vector>
 
 /*
 All registers are above 16 to not confuse them with u8 or u16
@@ -55,7 +55,7 @@ class z80_asm_handler
 public:
     int index;
     int memory_offset;
-    ptgb::vector<byte> data_vector;
+    std::vector<byte> data_vector;
 
     z80_asm_handler(int data_size, int mem_offset);
     void add_byte(u8 value);
@@ -115,25 +115,25 @@ private:
 class z80_variable
 {
 public:
-    ptgb::vector<byte> data;
+    std::vector<byte> data;
     int size;
-    z80_variable(ptgb::vector<z80_variable*> *var_vec, int data_size, ...);
-    z80_variable(ptgb::vector<z80_variable*> *var_vec);
+    z80_variable(std::vector<z80_variable*> *var_vec, int data_size, ...);
+    z80_variable(std::vector<z80_variable*> *var_vec);
     void load_data(int data_size, byte array_data[]);
     int place_ptr(z80_asm_handler *z80_instance);
     void insert_variable(z80_asm_handler *var);
     void update_ptrs();
 
 private:
-    ptgb::vector<int> ptr_locations;
-    ptgb::vector<z80_asm_handler *> asm_handlers;
+    std::vector<int> ptr_locations;
+    std::vector<z80_asm_handler *> asm_handlers;
     int var_mem_location;
 };
 
 class z80_jump
 {
 public:
-    z80_jump(ptgb::vector<z80_jump*> *jump_vec);
+    z80_jump(std::vector<z80_jump*> *jump_vec);
     int place_relative_jump(z80_asm_handler *z80_instance);
     int place_direct_jump(z80_asm_handler *z80_instance);
     int place_pointer(z80_asm_handler *z80_instance);
@@ -141,9 +141,9 @@ public:
     void update_jumps();
 
 private:
-    ptgb::vector<int> ptr_locations;
-    ptgb::vector<z80_asm_handler *> asm_handlers;
-    ptgb::vector<bool> jump_types;
+    std::vector<int> ptr_locations;
+    std::vector<z80_asm_handler *> asm_handlers;
+    std::vector<bool> jump_types;
     int jump_mem_location;
 };
 

tools/data-generator/src/gb_rom_values/gb_rom_values_fre.cpp

@@ -45,6 +45,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC42F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     },
     { // FRE_BLUE
         .language = FRE_ID,
@@ -90,6 +91,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC42F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     },
     { // FRE_YELLOW
         .language = FRE_ID,
@@ -135,6 +137,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC42F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     },
     { // FRE_GOLD
         .language = FRE_ID,
@@ -180,6 +183,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC42F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     },
     { // FRE_SILVER
         .language = FRE_ID,
@@ -225,6 +229,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC42F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     },
     {
         .language = FRE_ID,
@@ -270,6 +275,7 @@ const struct GB_ROM gb_rom_values_fre[] = {
         .textBorderUppLeft = 0xC52F,
         .textBorderWidth = 12,
         .textBorderHeight = 1,
+        .padding_2 = 0
     }
 };
 

tools/data-generator/src/main.cpp

@@ -2,7 +2,14 @@
 #include "common.h"
 #include "gba_rom_values/gba_rom_values.h"
 #include "gb_rom_values/gb_rom_values.h"
+#include "payloads/payload_file_writer.h"
+#include "payloads/payload_file_reader.h"
+#include "payloads/binary_patch_generator.h"
+#include "payloads/payload_builder.h"
 #include <cstdio>
+#include <cstring>
+#include <cstdlib>
+
 // This application holds the various long static data arrays that Poke Transporter GB uses
 // and it writes them to .bin files that can be compressed with compressZX0 later.
 // it's useful to do it this way because it keeps this data easy to view, edit and document
@@ -18,6 +25,176 @@ void generate_gb_rom_value_tables(const char *output_path, const char *filename,
     writeTable(output_path, filename, reinterpret_cast<const char*>(rom_data_values), num_elements * sizeof(struct GB_ROM));
 }
 
+/**
+ * Generates the payloads for the specific pokémon generation.
+ * Note: yellow_version indicates that we'd want to generate the payloads for pokémon yellow.
+ * The reason we single out pokémon yellow, is because the binary patch diff is significant compared to Blue, Red and Green.
+ * It's easier to compress the data if we generate it into a separate file.
+ */
+void generate_payloads_for(uint8_t generation, bool yellow_version, const char* output_path, const char* filename)
+{
+    uint8_t base_payload_buffer[PAYLOAD_SIZE];
+    uint8_t other_payload_buffer[PAYLOAD_SIZE];
+    payload_file_writer payload_writer;
+    binary_patch_generator patch_generator;
+    u16 rom_set_index;
+    u16 base_payload_index;
+    u16 index;
+
+    char full_path[4096];
+
+    if(output_path[0] != '\0')
+    {
+        snprintf(full_path, sizeof(full_path), "%s/%s", output_path, filename);
+    }
+    else
+    {
+        strncpy(full_path, filename, sizeof(full_path));
+    }
+
+    const struct GB_ROM *rom_value_sets[] = {
+        gb_rom_values_eng,
+        gb_rom_values_fre
+    };
+
+    const u16 rom_value_sizes[] = {
+        gb_rom_values_eng_size,
+        gb_rom_values_fre_size
+    };
+
+    const u8 num_elements = sizeof(rom_value_sizes) / sizeof(u16);
+
+    // search for the first english GB_ROM struct for the given generation
+    for(base_payload_index = 0; base_payload_index < gb_rom_values_eng_size; ++base_payload_index)
+    {
+        if(gb_rom_values_eng[base_payload_index].generation == generation)
+        {
+            if((!yellow_version && gb_rom_values_eng[base_payload_index].version != YELLOW_ID) || (yellow_version && gb_rom_values_eng[base_payload_index].version == YELLOW_ID))
+            {
+                break;
+            }
+        }
+    }
+
+    memset(base_payload_buffer, 0, sizeof(base_payload_buffer));
+
+    // initialize the found GB_ROM struct as the base payload
+    init_payload(base_payload_buffer, gb_rom_values_eng[base_payload_index], TRANSFER, false);
+    payload_writer.set_base_payload(gb_rom_values_eng[base_payload_index].language, gb_rom_values_eng[base_payload_index].version, base_payload_buffer, gb_rom_values_eng[base_payload_index].payload_size);
+
+    for(rom_set_index = 0; rom_set_index < num_elements; ++rom_set_index)
+    {
+        for(index = 0; index < rom_value_sizes[rom_set_index]; ++index)
+        {
+            const struct GB_ROM *curr_rom = &rom_value_sets[rom_set_index][index];
+            if(curr_rom->generation != generation || 
+                (rom_set_index == 0 && index == base_payload_index) || 
+                (yellow_version && curr_rom->version != YELLOW_ID) || 
+                (!yellow_version && curr_rom->version == YELLOW_ID))
+            {
+                // skip if:
+                // - the generation does not match
+                // - if it's the base payload
+                // - if we specified yellow_version == true and the current rom is not a pokémon yellow rom.
+                // - if we specified yellow_version == false and the current rom IS a pokémon yellow rom.
+                continue; 
+            }
+
+            // add the binary patches for this ROM
+            memset(other_payload_buffer, 0, PAYLOAD_SIZE);
+            init_payload(other_payload_buffer, *curr_rom, TRANSFER, false);
+            binary_patch_list patches = patch_generator.diff(base_payload_buffer, other_payload_buffer, PAYLOAD_SIZE);
+            payload_writer.add_binary_patches(curr_rom->language, curr_rom->version, patches);
+        }
+    }
+
+    payload_writer.write_to_file(full_path);
+}
+
+void test_payloads(const char* output_path, const char* filename)
+{
+    char full_path[4096];
+    uint8_t buffer[2048]; // 2048 bytes is enough for the payloads
+    uint8_t reference_payload_buffer[PAYLOAD_SIZE];
+    uint8_t reconstructed_payload_buffer[PAYLOAD_SIZE];
+    FILE *file;
+    size_t size;
+    u16 rom_set_index;
+    u16 index;
+
+    if(output_path[0] != '\0')
+    {
+        snprintf(full_path, sizeof(full_path), "%s/%s", output_path, filename);
+    }
+    else
+    {
+        strncpy(full_path, filename, sizeof(full_path));
+    }
+
+    file = fopen(full_path,"rb"); /*open file*/
+    fseek(file, 0, SEEK_END); 
+    size = ftell(file);         /*calc the size needed*/
+    fseek(file, 0, SEEK_SET);
+
+    payload_file_reader payload_reader(buffer, size);
+    
+    const struct GB_ROM *rom_value_sets[] = {
+        gb_rom_values_eng,
+        gb_rom_values_fre
+    };
+
+    const u16 rom_value_sizes[] = {
+        gb_rom_values_eng_size,
+        gb_rom_values_fre_size
+    };
+
+    const u8 num_elements = sizeof(rom_value_sizes) / sizeof(u16);
+
+    fread(&buffer, 1, size, file);
+
+    for(rom_set_index = 0; rom_set_index < num_elements; ++rom_set_index)
+    {
+        for(index = 0; index < rom_value_sizes[rom_set_index]; ++index)
+        {
+            const struct GB_ROM *curr_rom = &rom_value_sets[rom_set_index][index];
+            
+            // first generate the reference payload
+            memset(reference_payload_buffer, 0, PAYLOAD_SIZE);
+            init_payload(reference_payload_buffer, *curr_rom, TRANSFER, false);
+
+            // now read the payload from the file
+            memset(reconstructed_payload_buffer, 0, PAYLOAD_SIZE);
+
+            // okay, so, the given file may or may not contain the desired payload.
+            // we should just skip if the read call returns false
+            if(!payload_reader.read_payload(reconstructed_payload_buffer, curr_rom->language, curr_rom->version))
+            {
+                continue; // skip if the payload was not found
+            }
+
+            printf("Testing payload from file %s for language %u, variant %u: ", full_path, curr_rom->language, curr_rom->version);
+            if(!memcmp(reference_payload_buffer, reconstructed_payload_buffer, PAYLOAD_SIZE))
+            {
+                printf("PASS!\n");
+            }
+            else
+            {
+                printf("FAILED!\n");
+                // print the differences
+                for(size_t i = 0; i < PAYLOAD_SIZE; ++i)
+                {
+                    if(reference_payload_buffer[i] != reconstructed_payload_buffer[i])
+                    {
+                        printf("Byte %zu: expected 0x%02X, got 0x%02X\n", i, reference_payload_buffer[i], reconstructed_payload_buffer[i]);
+                    }
+                }
+                abort(); // stop execution on failure
+            }
+
+        }
+    }
+}
+
 int main(int argc, char **argv)
 {
     const char *output_path = (argc > 1) ? argv[1] : "";
@@ -33,6 +210,14 @@ int main(int argc, char **argv)
 
     generate_gb_rom_value_tables(output_path, "gb_rom_values_eng.bin", gb_rom_values_eng, gb_rom_values_eng_size);
     generate_gb_rom_value_tables(output_path, "gb_rom_values_fre.bin", gb_rom_values_fre, gb_rom_values_fre_size);
+
+    generate_payloads_for(1, false, output_path, "gb_gen1_payloads_RB.bin");
+    test_payloads(output_path, "gb_gen1_payloads_RB.bin");
+    generate_payloads_for(1, true, output_path, "gb_gen1_payloads_Y.bin");
+    test_payloads(output_path, "gb_gen1_payloads_Y.bin");
+    generate_payloads_for(2, false, output_path, "gb_gen2_payloads.bin");
+    test_payloads(output_path, "gb_gen2_payloads.bin");
+    
     
     printf("sizeof ROM_DATA: %zu\n", sizeof(struct ROM_DATA));
     printf("sizeof GB_ROM: %zu\n", sizeof(struct GB_ROM));

tools/data-generator/src/payloads/binary_patch_generator.cpp

@@ -0,0 +1,60 @@
+#include "payloads/binary_patch_generator.h"
+
+#include <cstdio>
+binary_patch_list binary_patch_generator::diff(const uint8_t* buffer1, const uint8_t* buffer2, size_t size) const
+{
+    binary_patch_list patches;
+    binary_patch_data current_patch;
+    bool diff_started = false;
+    
+    // Reserve some space for the patch data to avoid frequent reallocations
+    current_patch.data.reserve(10);
+
+    for (uint16_t i = 0; i < size; ++i)
+    {
+        if (buffer1[i] != buffer2[i])
+        {
+            if(!diff_started)
+            {
+                // Start a new patch
+                diff_started = true;
+                current_patch.offset = i;
+            }
+
+            current_patch.data.push_back(buffer2[i]);
+        }
+        else if(diff_started)
+        {
+            // If we were in a diff and found a match, finalize the current patch
+          printf("Generated patch: offset: %hu, size: %zu\n", current_patch.offset, current_patch.data.size());
+            patches.push_back(current_patch);
+            current_patch.data.clear();
+            diff_started = false;
+        }
+    }
+
+  printf("Finished generating binary patches... num=%zu\n", patches.size());
+    return patches;
+}
+
+uint16_t write_binary_patch_to_buffer(const binary_patch_data &patch, uint8_t *buffer)
+{
+    // start by writing the size of the data as an uint16_t, little endian.
+    const uint16_t size = static_cast<uint16_t>(patch.data.size());
+    *buffer = size & 0xFF;
+    *(buffer + 1) = (size >> 8);
+    buffer += 2;
+
+    // now write the 16 bit offset in little endian format
+    *buffer = patch.offset & 0xFF;
+    *(buffer + 1) = (patch.offset >> 8);
+    buffer += 2;
+
+    // finally write the data itself
+    for (const uint8_t &byte : patch.data)
+    {
+        *buffer++ = byte;
+    }
+
+    return 4 + patch.data.size(); // 4 bytes for offset and size, plus the size of the data
+}

tools/data-generator/src/payloads/payload_builder.cpp

@@ -1,14 +1,14 @@
-#include "payload_builder.h"
+#include "payloads/payload_builder.h"
 #include "gb_rom_values/base_gb_rom_struct.h"
-#include "debug_mode.h"
-#include "z80_asm.h"
+#include "payloads/z80_asm.h"
+#include "../../../include/debug_mode.h"
+#include <cstring>
 
 #define DATA_LOC (SHOW_DATA_PACKETS ? curr_rom.transferStringLocation : curr_rom.wEnemyMonSpecies)
 
-static byte payload_buffer[PAYLOAD_SIZE];
-
-void init_payload(GB_ROM curr_rom, int type, bool debug)
+void init_payload(byte *payload_buffer, const GB_ROM& curr_rom, int type, bool debug)
 {
+    (void)type;
     /*  10 RNG bytes
         8 Preamble bytes
         418 / 441 Party bytes
@@ -19,8 +19,8 @@ void init_payload(GB_ROM curr_rom, int type, bool debug)
         */
     if ((curr_rom.generation == 1 && curr_rom.version != YELLOW_ID))
     {
-        ptgb::vector<z80_jump *> jump_vector;
-        ptgb::vector<z80_variable *> var_vector;
+        std::vector<z80_jump *> jump_vector;
+        std::vector<z80_variable *> var_vector;
 
         z80_asm_handler z80_rng_seed(0x0A, curr_rom.wSerialOtherGameboyRandomNumberListBlock + 8);
         z80_asm_handler z80_payload(0x1AA, curr_rom.wSerialEnemyDataBlock);
@@ -287,8 +287,8 @@ void init_payload(GB_ROM curr_rom, int type, bool debug)
 
     else if ((curr_rom.generation == 1 && curr_rom.version == YELLOW_ID))
     {
-        ptgb::vector<z80_jump *> jump_vector;
-        ptgb::vector<z80_variable *> var_vector;
+        std::vector<z80_jump *> jump_vector;
+        std::vector<z80_variable *> var_vector;
 
         z80_asm_handler z80_rng_seed(0x0A, curr_rom.wSerialOtherGameboyRandomNumberListBlock + 8);
         z80_asm_handler z80_payload(0x1AA, curr_rom.wSerialEnemyDataBlock - 8); // Subtracting 8 is because the data is shifted after patching, removing part of the enemy name. May change depending on language
@@ -658,8 +658,8 @@ void init_payload(GB_ROM curr_rom, int type, bool debug)
 
     else if (curr_rom.generation == 2)
     {
-        ptgb::vector<z80_jump *> jump_vector;
-        ptgb::vector<z80_variable *> var_vector;
+        std::vector<z80_jump *> jump_vector;
+        std::vector<z80_variable *> var_vector;
 
         z80_asm_handler z80_rng_seed(0x0A, curr_rom.wSerialOtherGameboyRandomNumberListBlock);
         z80_asm_handler z80_payload(0x1CD, curr_rom.wSerialEnemyDataBlock);      // wOTPartyData
@@ -947,11 +947,6 @@ void init_payload(GB_ROM curr_rom, int type, bool debug)
     memset(payload_buffer, 0x00, PAYLOAD_SIZE);
 };
 
-byte* get_payload()
-{
-    return payload_buffer;
-}
-
 #if PAYLOAD_EXPORT_TEST
 #include <cstdio>
 int main() // Rename to "main" to send the payload to test_payload.txt

tools/data-generator/src/payloads/payload_file_reader.cpp

@@ -0,0 +1,132 @@
+#include "payloads/payload_file_reader.h"
+#include "payloads/payload_file_writer.h"
+#include "gb_rom_values/base_gb_rom_struct.h"
+#include <cstring>
+
+// Reads a payload from the specified buffer pointer based on generation, language, and game variant
+static const uint8_t* read_payload_metadata(const uint8_t *buffer, payload_metadata &out_metadata)
+{
+    out_metadata.language = buffer[0];
+    out_metadata.game_variant = buffer[1];
+    out_metadata.size = *(uint16_t *)(buffer + 2);
+    
+    return buffer + sizeof(struct payload_metadata); // Move past the metadata header
+}
+
+// Reads the payload data from the file buffer and returns a pointer to where we stopped reading
+static const uint8_t* read_payload_data(const uint8_t *input_buffer, uint8_t *output_buffer, size_t size)
+{
+    memcpy(output_buffer, input_buffer, size);
+    return input_buffer + size;
+}
+
+/**
+ * @brief Calculate the next 4 byte aligned offset.
+ * 
+ * The Gameboy Advance needs alignment to read data correctly.
+ * For (u)int16_t data types, for example, the data needs to be aligned to 2 bytes
+ * For (u)int32_t data types, the data needs to be aligned to 4 bytes.
+ */
+static uint16_t get_aligned_offset(uint16_t current_offset)
+{
+    // Align to 4-byte boundary
+    return (current_offset + 3) & ~3; // Round up to the next multiple of 4
+}
+
+static const uint8_t* skip_to_aligned_offset(const uint8_t* buffer_start, const uint8_t* input_buffer)
+{
+    // Skip to the next 4-byte aligned offset
+    const uint16_t offset = static_cast<uint16_t>(input_buffer - buffer_start);
+    return buffer_start + get_aligned_offset(offset);
+}
+
+static const uint8_t* read_binary_patch(const uint8_t *input_buffer, uint8_t *output_buffer, uint16_t &out_offset, uint16_t &out_patch_size, bool skip)
+{
+    out_patch_size = static_cast<uint16_t>(input_buffer[0]) | (static_cast<uint16_t>(input_buffer[1]) << 8);
+
+    if(skip)
+    {
+        return input_buffer + 4 + out_patch_size;
+    }
+
+    input_buffer += 2; // Move past the size
+
+    out_offset = static_cast<uint16_t>(input_buffer[0]) | (static_cast<uint16_t>(input_buffer[1]) << 8);
+    input_buffer += 2; // Move past the offset
+
+    memcpy(output_buffer, input_buffer, out_patch_size);
+    return input_buffer + out_patch_size; // Return the pointer to where we stopped reading
+}
+
+payload_file_reader::payload_file_reader(const uint8_t *file_buffer, uint16_t buffer_size)
+    : file_buffer_(file_buffer)
+    , file_buffer_end_(file_buffer + buffer_size)
+{
+}
+
+bool payload_file_reader::read_payload(uint8_t *buffer, uint8_t language, uint8_t game_variant)
+{
+    uint8_t binary_patch_buffer[512]; // Buffer to hold binary patches
+    payload_metadata base_payload_metadata;
+    payload_metadata current_variant_metadata;
+    const uint8_t *cur = file_buffer_;
+    uint16_t i;
+    uint16_t binary_patch_offset;
+    uint16_t binary_patch_size;
+    bool found = false;
+
+    // first read the base payload metadata and the base payload
+    cur = read_payload_metadata(cur, base_payload_metadata);
+    cur = skip_to_aligned_offset(file_buffer_, cur);
+
+    cur = read_payload_data(cur, buffer, base_payload_metadata.size);
+    cur = skip_to_aligned_offset(file_buffer_, cur);
+
+    // check if the base payload is actually the payload we requested.
+    found = (base_payload_metadata.language == language &&
+        base_payload_metadata.game_variant == game_variant);
+    
+    if(found)
+    {
+        return true;
+    }
+
+    // now start reading the other payloads (which consist of binary patches)
+    while(cur < file_buffer_end_)
+    {
+        cur = read_payload_metadata(cur, current_variant_metadata);
+        cur = skip_to_aligned_offset(file_buffer_, cur);
+
+        found = (current_variant_metadata.language == language &&
+                current_variant_metadata.game_variant == game_variant);
+
+        // we need to read every binary patch, because we either need to apply them or skip them
+        for(i = 0; i < current_variant_metadata.size; ++i)
+        {
+            cur = read_binary_patch(cur, binary_patch_buffer, binary_patch_offset, binary_patch_size, !found);
+            cur = skip_to_aligned_offset(file_buffer_, cur);
+
+            if(found)
+            {
+                // apply the binary patch to the payload data
+                if(binary_patch_offset + binary_patch_size <= PAYLOAD_SIZE)
+                {
+                    memcpy(buffer + binary_patch_offset, binary_patch_buffer, binary_patch_size);
+                }
+                else
+                {
+                    return false; // Patch exceeds payload size
+                }
+            }
+            // else -> the patch is skipped, we've just read it to move the cur pointer forward
+        }
+
+        if(found)
+        {
+            return true;
+        }
+    }
+
+    return false; // No matching payload found
+}
+

tools/data-generator/src/payloads/payload_file_writer.cpp

@@ -0,0 +1,107 @@
+#include "payloads/payload_file_writer.h"
+#include <cstdio>
+#include <cstring>
+
+payload_file_writer::payload_file_writer()
+    : base_payload_(nullptr)
+    , base_payload_metadata_()
+    , binary_patches_()
+{
+    memset(&base_payload_metadata_, 0, sizeof(struct payload_metadata));
+}
+
+void payload_file_writer::set_base_payload(uint8_t language, uint8_t game_variant, const uint8_t *payload, uint16_t size)
+{
+    base_payload_ = payload;
+    base_payload_metadata_.language = language;
+    base_payload_metadata_.game_variant = game_variant;
+    base_payload_metadata_.size = size;
+}
+
+void payload_file_writer::add_binary_patches(uint8_t language, uint8_t game_variant, const binary_patch_list &patches)
+{
+    const uint16_t key = (static_cast<uint16_t>(language) << 8) | static_cast<uint16_t>(game_variant);
+    binary_patches_.push_back(std::make_pair(key, patches));
+}
+
+static void align_next(FILE *file)
+{
+    // Align to 4-byte boundary
+    long pos = ftell(file);
+    long aligned_pos = (pos + 3) & ~3; // Round up to the next multiple of 4
+    if (aligned_pos > pos)
+    {
+        uint8_t padding[3] = {0, 0, 0};
+        fwrite(padding, 1, aligned_pos - pos, file);
+    }
+}
+
+static void write_payload_metadata(FILE *file, const payload_metadata &metadata)
+{
+    fwrite(&metadata.language, 1, 1, file);
+    fwrite(&metadata.game_variant, 1, 1, file);
+    fwrite(&metadata.size, sizeof(metadata.size), 1, file);
+
+    // align to 4-byte boundary
+    align_next(file);
+}
+
+void payload_file_writer::write_to_file(const char *path_to_file) const
+{
+    uint8_t binary_patch_buffer[512]; // Buffer to hold binary patches
+    uint16_t binary_patch_buffer_depth;
+
+    FILE *file = fopen(path_to_file, "wb");
+    if (!file)
+    {
+        fprintf(stderr, "Failed to open file for writing: %s", path_to_file);
+        return;
+    }
+
+    if(!base_payload_)
+    {
+        fprintf(stderr, "Base payload is not set. Cannot write to file: %s", path_to_file);
+        fclose(file);
+        return;
+    }
+
+    // Write base payload metadata
+    write_payload_metadata(file, base_payload_metadata_);
+    
+    // Write base payload
+    if (base_payload_)
+    {
+        fwrite(base_payload_, 1, base_payload_metadata_.size, file);
+    }
+
+    align_next(file);
+
+    // Write binary patches
+    for (const auto &pair : binary_patches_)
+    {
+        const uint32_t key = pair.first;
+
+        const uint8_t language = static_cast<uint8_t>((key >> 8) & 0xFF);
+        // The last byte of the key is the game variant
+        const uint8_t game_variant = static_cast<uint8_t>(key & 0xFF);
+        const binary_patch_list &patches = pair.second;
+        const uint16_t patch_count = static_cast<uint16_t>(patches.size());
+
+        const payload_metadata metadata = {
+            language,
+            game_variant,
+            patch_count, // in the case of binary patches, the size field indicates the number of binary patches.
+        };
+        write_payload_metadata(file, metadata);
+
+        for (const auto &patch : patches)
+        {
+            binary_patch_buffer_depth = write_binary_patch_to_buffer(patch, binary_patch_buffer);
+            fwrite(binary_patch_buffer, 1, binary_patch_buffer_depth, file);
+            // Align after each binary patch
+            align_next(file);
+        }
+    }
+
+    fclose(file);
+}

tools/data-generator/src/payloads/z80_asm.cpp

@@ -1,7 +1,8 @@
-#include "z80_asm.h"
+#include "payloads/z80_asm.h"
+
 #include <stdarg.h>
-#include "libraries/nanoprintf/nanoprintf.h"
-#include "libstd_replacements.h"
+#include <cstdio>
+#include <cstring>
 
 #define DIRECT false
 #define RELATIVE true
@@ -15,7 +16,7 @@ static void throw_error(const char* format, ...)
     va_list args;
 
     va_start(args, format);
-    npf_vsnprintf(error_msg_buffer, sizeof(error_msg_buffer), format, args);
+    vsnprintf(error_msg_buffer, sizeof(error_msg_buffer), format, args);
     va_end(args);
 
     // we should avoid exceptions and <stdexcept>
@@ -675,12 +676,12 @@ void z80_asm_handler::SET(int bit, int reg)
     }
 }
 
-z80_variable::z80_variable(ptgb::vector<z80_variable *> *var_vec)
+z80_variable::z80_variable(std::vector<z80_variable *> *var_vec)
 {
     var_vec->push_back(this);
 }
 
-z80_variable::z80_variable(ptgb::vector<z80_variable *> *var_vec, int data_size, ...)
+z80_variable::z80_variable(std::vector<z80_variable *> *var_vec, int data_size, ...)
 {
     var_vec->push_back(this);
     data.resize(data_size);
@@ -723,7 +724,7 @@ void z80_variable::update_ptrs()
     }
 }
 
-z80_jump::z80_jump(ptgb::vector<z80_jump *> *jump_vec)
+z80_jump::z80_jump(std::vector<z80_jump *> *jump_vec)
 {
     jump_vec->push_back(this);
 }