From c45566e0f6a0c03b8f34795fbcf5971b1f55cb9d Mon Sep 17 00:00:00 2001 From: ronitsinha Date: Sat, 22 Jul 2023 19:40:00 -0400 Subject: [PATCH] fixes and cleanup --- .gitignore | 1 + Makefile | 13 +- PokeID.bin | Bin 14040 -> 0 bytes README.md | 3 + fieldwaza_name_us.mes | Bin 356 -> 0 bytes main.cpp | 34 +++ lcs.cpp => old/lcs.cpp | 0 lcs.h => old/lcs.h | 0 pokeid.cpp => old/pokeid.cpp | 20 +- pokeid.h => old/pokeid.h | 0 pokedatastructure.h | 82 +++--- pokemon_name_us.mes | Bin 6804 -> 0 bytes romprocessor.cpp | 311 ++++++++++++++++++++ romprocessor.h | 48 ++++ test.cpp | 532 ----------------------------------- util.cpp | 170 ++++++++++- util.h | 17 ++ 17 files changed, 633 insertions(+), 598 deletions(-) delete mode 100644 PokeID.bin create mode 100644 README.md delete mode 100644 fieldwaza_name_us.mes create mode 100644 main.cpp rename lcs.cpp => old/lcs.cpp (100%) rename lcs.h => old/lcs.h (100%) rename pokeid.cpp => old/pokeid.cpp (93%) rename pokeid.h => old/pokeid.h (100%) delete mode 100644 pokemon_name_us.mes create mode 100644 romprocessor.cpp create mode 100644 romprocessor.h delete mode 100644 test.cpp diff --git a/.gitignore b/.gitignore index d9a3e61..be92174 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.nds *.out +range_randomizer \ No newline at end of file diff --git a/Makefile b/Makefile index f7b9377..fbc6293 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -all: test +CXX = g++ +CXXFLAGS = -Wall -Wextra -Wpedantic -Wshadow -test: clean - g++ util.cpp pokedatastructure.cpp \ - pokeid.cpp test.cpp -o test.out +all: main + +main: clean + $(CXX) ${CXXFLAGS} util.cpp pokedatastructure.cpp \ + romprocessor.cpp main.cpp -o range_randomizer clean: - rm -f *.out \ No newline at end of file + rm -f range_randomizer \ No newline at end of file diff --git a/PokeID.bin b/PokeID.bin deleted file mode 100644 index 3f2e188ee80c461d9a5c302ffa8e5fae5633c547..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14040 zcmeHNX^fp!89v|l&2sPT%bj65PR2>aIxHgk!zlcq1rj4v5{)L9C=skNE(l2>5l}Qy zpn~E8?uolnb`+@M!lHmEASfuv<^m-IYHg>}nJGl#^StMrJNJF(%!z5Df4nzsI@4#) zci#7T_xGEJU+dh2bM89)``5#q>#mCK0-v(vpYXS;I`Jc`ema~haLO%>IM?VYcJl`} zxk34P90XRnr3UWlD;HX^Zs*FfD}AmeSm_oM=jz1z>u!G9xtXErh1O^GJJ;LWfoBdm zH&QRueUPZPw=CUGkyFHL$~nq!c@Ef6P3r&=pha4TG3V;_BI>{&T-JSDu*%JW;E6ts zt#Pgzu2*iK$f9|f6s$vJ8SmBDOPs6KYVkgeO+^d>Pxfo<<-tY^2|m{e*5T&yfT;n^ z=PM|7;KkZ@Rxry1me$(WCWyC0w_= zsN4cbw|@3I=eoK|&)Imdw(aW$!?^@6M?~KhIrX_CBWLiz*@ouqDCc^5z}X(rLEC); zvA(+8H{PqUqbUpiYhhre?#B@8D3fv?8+4c#KhwOtA#k<}PGz}%Bjbp6fDTi*lb;y3 z$2r&62hQf81L8=<^znjuyH5@1`!)*ZeaXh=n}QFfa={6LRmqrsv%~}LOYr#?!8${J z(K@`9-}ZoO_=NS(w-E!MdxbM?zqiwu%E~L%z6oyXVFn0y|jj zV{)JO7mIzI>XH}u;M!r$^?8ED8ld$%KX8_^`vn=F*b?F|b?!nwlXge^SwFid+{gTB zW9$;=1_qjbrscdW)M?hgi9U^O6%6r*XIekITxtjW8oW&V7NhAeS3tkvdX$qSE&I}y zf}y609W-{8V4hD~bFLT8JZ9y*K`?JWTaRuOtV-sWZIY9HY}#`-wd7Wd-Au6qpEf7o zCiNDwfESy~Zx6nkvh5wt4GzNRQ`nt?rTy$K<}--nYnA5nZr%rfVQ#g4eNV7^sxP<~ zWp9XiP-09!^FHJk_E~eVpUnjihz~|N^|=T6%mVcYyjWj)i24N`tY1G8YZd&{=9foX zV%qxrW012GHhMl0D~}6?yaZmXEKg7e=+m<{Iy*!E0yzbYIr8s3cR=ig^|>dh z`#>6din`O6z^Bz=JF(ESPY>$cvqO4~$hF4)8RH?L`>qx~E!Waqz+dKJcPq;RIAb4@ z$~}v~YOE^>pIAa14VE2l_kk<>vlm1B?Md{za52m)={{t*KyMo2&*q5{!P2>4wD2*O zsw<6Rd4909cuX)KM{9?*ZO04PJjaWvVl>-$UTk|6E{*nN6XKl5b;YpU15#_epuzT9(WYw*2&!iQ88TRwnDwYXlQNs}3F4yc|iN;M@patiK!$KdW(O zhVNS4k7@Bi`^;mBjr(51_94f&_@|ArjpPNKrDmrS3e@Q^-`bw^_Jkfj4_)h zKE=F*XD;EH)?Y3p1|Lk#Ko=1kE8QIATsy2|?9bbC!*xpR{mo23$?v1?$IJ2>NpV%F~B(t{8pTCzJ0{s#-^+BF% z&f3&&S-(HEw!th3vK|r}fWU>AN_`p2+IHf%g%E4u>OlTR%BJ`ziAo&&e5W zquhPlL*L`?(?0P`$OQ`tUjE2hh5Oi}tQ+-G#(e!SVvIe?PGB~F{yD_v{0#Ne_uU^c zoI7|-W5-FJ@V%Se_nb)#voCZgtzUy@%(v;D@AeSWIQOFUms~vLdazub&gHY*yqTM) zbG0hBJIvLG4J|ukyYtWG>s+1A%`&;Xv@Nqc%*|@K_gpC7tL{8bQtvIa4rVLzI= z{ijIJ=X+XP8*@9rUr2uG6#GpNYM*#Od?}6nAtTKy8=8D+fm>5*xodE-~amD8tiFT``v?_?hIS`tY2A+1L9m8Yf_=UX!D}zXnxb? zrAdYQqRoqFWS5)I-1}7AM%ph30#bstL}FRE9SUM z`TIS5H;8@64EAgM#GS{}nDDrR*a^(;sPW5h^1h0{|A|$uI>+4;-`l5lPy4&`_^Rx8 z{Y+wizcsff%FQ{s9a3(O^1Cnt?FKJ)F1|mr@5#+hxtZ&~v6p(3`?>;rVgbeMM zEJXN8?IX8G4Eou9-7^96`$5giS>e4JcOLN_rkBXMv%`)$g`LCq7L+A%?n%z!z{tz` z8Gm1V=2Od9=Mj?{+reEkez5g-l;XLZZFBya^TC{)IV{T5{b>!3{-&*9XfRGq^~%2y%h*ze}HfphcGP{v=qY+>e_xFf65n1N{Il9NAP$& z>G +#include +#include +#include +#include +#include +#include +#include +#include + +#include "romprocessor.h" + +using namespace std; + +// const char* rom_file_path = "rom.nds"; +// const char* out_file_path = "randomized.nds"; + +int main (int argc, char *argv[]) { + if (argc < 3) { + cout << + "Usage: ./ranger_randomizer [input file name] [output file name]" + << endl; + + return 0; + } + + srand(time(NULL)); + + RomProcessor rp(argv[1], argv[2]); + + rp.randomize_rom(); + + return 0; +} \ No newline at end of file diff --git a/lcs.cpp b/old/lcs.cpp similarity index 100% rename from lcs.cpp rename to old/lcs.cpp diff --git a/lcs.h b/old/lcs.h similarity index 100% rename from lcs.h rename to old/lcs.h diff --git a/pokeid.cpp b/old/pokeid.cpp similarity index 93% rename from pokeid.cpp rename to old/pokeid.cpp index 7c292d7..3d0e176 100644 --- a/pokeid.cpp +++ b/old/pokeid.cpp @@ -195,22 +195,4 @@ vector get_pokemon_names_rom (const char *path, uint32_t offset) { vector get_field_moves_rom (const char *path, uint32_t offset) { return read_mes_from_rom(path, offset); -} - -vector get_pokemon_names () { - return read_mes_file("pokemon_name_us.mes"); -} - -vector get_field_moves () { - return read_mes_file("fieldwaza_name_us.mes"); -} - -// int main () { -// vector pkmn_names = get_pokemon_names (); -// vector field_moves = get_field_moves (); - - -// read_pokeID_bin (pkmn_names, field_moves); - -// return 0; -// } \ No newline at end of file +} \ No newline at end of file diff --git a/pokeid.h b/old/pokeid.h similarity index 100% rename from pokeid.h rename to old/pokeid.h diff --git a/pokedatastructure.h b/pokedatastructure.h index 0297e37..903175f 100644 --- a/pokedatastructure.h +++ b/pokedatastructure.h @@ -11,54 +11,54 @@ #include "util.h" class PokeDataStructure { - public: - PokeDataStructure(); - ~PokeDataStructure(); +public: + PokeDataStructure(); + ~PokeDataStructure(); + + void add_pokemon(uint16_t poke_id, uint16_t name_id, uint8_t field_id, uint8_t field_level); + std::pair get_field_move(uint16_t poke_id); + + std::vector get_pokemon_with_geq_field_move (uint16_t poke_id); + + void print_level_order(); + void self_test(); +private: + std::map pokemon_field_moves; + // ugh, hate having two maps + std::map pokemon_names_to_reals; + std::map pokemon_reals_to_names; + + struct TreeNode { + // first is (field_id << 8) | field_level, second is poke_id + // for leaves, field signature + // for non-leaves, max value of left subtree + uint16_t data; + std::vector poke_ids; - void add_pokemon(uint16_t poke_id, uint16_t name_id, uint8_t field_id, uint8_t field_level); - std::pair get_field_move(uint16_t poke_id); + int height; + TreeNode *left; + TreeNode *right; + }; - std::vector get_pokemon_with_geq_field_move (uint16_t poke_id); + TreeNode *root; - void print_level_order(); - void self_test(); - private: - std::map pokemon_field_moves; - // ugh, hate having two maps - std::map pokemon_names_to_reals; - std::map pokemon_reals_to_names; + TreeNode* find_vsplit (TreeNode *node, + uint16_t min, uint16_t max); + void collect_subtree (TreeNode *node, std::vector *vec); + std::vector range_query (uint16_t min, uint16_t max); - struct TreeNode { - // first is (field_id << 8) | field_level, second is poke_id - // for leaves, field signature - // for non-leaves, max value of left subtree - uint16_t data; - std::vector poke_ids; - - int height; - TreeNode *left; - TreeNode *right; - }; + TreeNode* add_node (TreeNode *nod, uint16_t field_signature, + uint16_t poke_id); - TreeNode *root; + bool is_leaf(TreeNode *node); + int height(TreeNode* node); + int get_balance(TreeNode *node); + TreeNode* left_rotate(TreeNode *x); + TreeNode* right_rotate(TreeNode *y); - TreeNode* find_vsplit (TreeNode *node, - uint16_t min, uint16_t max); - void collect_subtree (TreeNode *node, std::vector *vec); - std::vector range_query (uint16_t min, uint16_t max); + void print_current_level (TreeNode* node, int level); - TreeNode* add_node (TreeNode *nod, uint16_t field_signature, - uint16_t poke_id); - - bool is_leaf(TreeNode *node); - int height(TreeNode* node); - int get_balance(TreeNode *node); - TreeNode* left_rotate(TreeNode *x); - TreeNode* right_rotate(TreeNode *y); - - void print_current_level (TreeNode* node, int level); - - void post_order_delete(TreeNode *node); + void post_order_delete(TreeNode *node); }; diff --git a/pokemon_name_us.mes b/pokemon_name_us.mes deleted file mode 100644 index a1e31de296cc14bfd8172dbf5d084d41cbfdc203..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6804 zcma)BJ$56x5tiTEz`+KN9c|`lH}DNeN}^^&YPf2yWKV|f2D@Q`jUIGEoL1!LGdS@9 z95}Hru$gb-3#^ae@2f&n3O(~Qs_Obd0jOWhUw?ab_1|AzUH#_j>gt!D|M|<$|B`?I z4)p3<{4A3%&0tal{A>Jtc$owv?a6j2lZ1EGns_o@ZNW^s;MUvFz2WZF#2Pfxi!aH= z(OZCmKbts){6W_CW*CDT$RPU6Xk+h$pSEo6X!=kKZJ9<857B~3+OxGaH50x!$J80s zvBdZ2yvJvu?Ofg1NofLyaSZQiVHCbcGmd67vKC-h!IoJ^(OVP4S@>=h6!_+7x*_XY z#4;Q&dXJ`RQ`YMUirxZeH?+~7!6)ZK9B0sG{K3^Bn(jY;`?Jyocih<<{v}z9;}-b~ z|6ktWuH*JD^rP#V8DDvOFfQUWlrDIw_rr61O1^hrXW0&??=AO7TR*y{@l&4;2gRQS z9|foE=Tw>zzX`7cdK$IEW{fU~%+`i8xkR^(gEO7N!&Im0C@}22z+2NDFMbl;$(mm7 zpKGph`QD8!KM-HU%5<)~_{T1IcQ%dC1a4jQnOE>=#9-R8D5Kmb;d@gXqN3BAqj@zv z9;2HjkZ@AeftcuBSK{k@#?^>`!3V|nwi`_q0#M;sJSNg< z&)SAM)mCWhXkHN7Obhc_SsEwj8M+CfB7BdgceT2(;Drz8KET+_OT3IhXj{{C_T(h6 zz6D+Ik=a9a-I{o?W7Yl+354rh!<;!*0rP6*dtqsD=CS7{z{SOW-Wx3b2DEKDsy!`^ zLo$}GvGQRWkV$6wHByjAL{R?Dq>d!fWxK|Ji}0t8QIXe!FT5fN3IEhfuo1qGrYy6b zd&C@%j^CexN6st7mKzVJjsYjEw%mNRO;e@Ic?h1sp4VEXGSJJYXUhy^bWe9~JcZ6P z6^dTPs}?RU_{LR7Hzo;uu`L`9Oap}BBd;_3*iLPT8W(_T-UNA&*Si71KzMJu#(MS9 zd%$Km$~vUog*aIF0pY@$nx0LVam6f^ze@@dcrLQoZ;={NcMTt`4F~t0n}S zp5Mo*ld~tEoS!Mv>#Op?e@^oq?OZpwd|!K%vw&oheaszAL?kJ`o{(s2QdP&s^pbdJ z)3tl64LA1Xa=uTfLQ$%m%6!C~5~a+oH$o!33vq%o%J)wwloM{2-vJNUy1@6ap9P4v zDcp-K#1|IMVXATR`5d|sz4qbc`j%#bqaB2{wGHxKk{l(xQ&$_*c$h1^KLRu_ z>MQ%HZ+#dXeTuyMFoqgAAE2%iLj~Jd3I)Hl(RkOGG2dZ*#tcY7@c^1A!bm9V(U&SD`D(}5R)`f>E-^O%f%JH$& zGA{D0n+L2znGYQ*1RM{ipX{0jH?yi=TP$W=^lcwT{H(A2Adx8A}`n~ z$J=e7A0bk5Zn#d}(3@D%IP$LPfV2ONC^AUX7j57OnRo}zNE zOybm8wCFbA_LRGWk5{-;2E5QZYX+1u()JdOQi=o5koZpkWA5FA*C!-9r6KQCCN{iY z)pK^RP-!~{HvvBHX@wjea{hj_C)}75Kj0dlsu50Or6DAKjUAUx+IP{5o(Q=_Z7+~S zA${o`??US8AY+5>O>l7)k!1~JL7?qBRE$2?#_Q0*vDJrizWhB{L`N}V!Q6Auo&sH1 zL?tDidiGNcP0D8}HE?8$q_U?F&}d)!C)A1~cS<>&7fZ9k&SMqr|CZhjbDeuM$a}dq z-JlCV2n&Can09r|jfpSFjJR2)?NTo&ZG~;fjY{O2o7A{D9`3^TpV}<5LQ_@F`2nF{ z9kpLYkNJX3t9z+zkDGbhcZgMsJi>KW;Z&yKc$oo~-n=?Rb$_WwM9+Q_hjt9;7V{aM z(K*^Exs$qfJhI8kfI)ogns$7Le|F6)DhN}!=-yzqU_-+FJ*S97p>W|mx=Ov%r8Q}E z2ewqgw=#p!D07`K?xC8?`GnEX5fx@4P#ZU?c2)3izKwZ~*`9n4>D60v5V&aFVT3#K z``*Bf^L$pEnJ9Hudxj4(Y-O+T?Ow8y>UuPOoa68w?K&@ud~1xa_9aNor0xPSiDpV@ zTQgw%LIllqrHvS9H1Ns0j8;;q$f2L-m_nUTt_kto%u;|h_h1jMt1t%W{#^Gb^j9KN zv?yReAuhD%)(t0w8+C6&LO!eozWIvuopeUtVh(P#bc>EVWRS^7S{7c^38@ma=^+fZ zHZOO!o){hsUg0;$*O=m}y!OtgVP0>GG&lPq`s&i=nec9X#tWq_ zYtFRUuHtMweTnO=5X+`(<$Q@3sJ1;^4WP(9!>2e*PzT|Fv`F{?Bdf=6)d7ywOFWUb z5Ahg`_98_e9MX*15K&Z7k;ubd1lKL{`-tHQ9S1%O?a5&pME^v-TQ9Aoz$;UsQp{^s z>`5-U0edQa&RYwlHncXP`!jm=fOD927xu-`{b~0GuO%3}-e|$28&khq8a!lR_M{wn z>37w;`407@$EOQ@70tQk+yq|Qy+m9Ecn;x}HyA2%Zw9ZK`y9xi71Y>WpvT2%E&L0- zE_qx5Wp6RJ@44_^kGnD4iL8o;oBp{a(Ok?wImxkG2&jDeL#G#q}!;$U+mp zKN}w{oKV*Al7p#MXm?Nq_Y&F~Q${&yOp^BzURsrY{MP8~7}=kLb}*&;-7=;p>#;_bHC&94C;MmqV_PxWZVj zy(V$YC@16)sl&v#p*x^2m3t%pK~R0bp)EVKxGC4f9hxe|bZjFAuuCa6_I(favd!(RQ< zlVfLzhnV@DkogQD18KDj@zY^xmM^^@xD?8kd=R@^hg%o-5c(41r^7ff`^yrwo5`KL z&xj?|pG!P#QKww?7xo$)g30$}hu)$3Q3vrIZ!o%nLyhb|xmP*}X}3O@Q81;gFw7@3 zJ>(;dXPSHtGSr9+-(fKQUTj>PN|`*U?>eHP&^APPiI<*M&TAt*=3ZbE`qXtgLt$<> znTFA=cHIReTbVLc4zYW}GojeBOEF=R6aDZnJm9cGD_?=Pl;i9Zcxsjga$Xyij?F&p zTna2=dM2S8W?$m_#&;j%4`378oxuzSJM~PEj&of?eLz0M!Dqg;(+67etxYMFlxv|+ zL8g+{PS)Ng?2V5FS?gJS?F_&$7fm}O+B7P^U@dwH8XZe7=L&?9Yw>MRIg?t^p1l;1OvOJC+$aZv`m zlwi2ae9xHq*lbV-dic)1YS9v%eB(lTobLtiS14~z-p)whBhz7?l-x;N`V1I2_nFX# zaYp-f;m53g#5e#}v61sEo;&pX&~(o4J@A?84MsN3cfU2K%iKkMdA9lfbJn*u-ZO-S zhBt?hn=j(5Po1`$Lcc{`#(xW3Y`IP+iC3Q8{{=haVY=A9Gycd7M%g inv_start_offsets; // sorted by key in increasing order by default + + rom_file.seekg(0x20, ios::beg); + + for (auto it = file_locations.begin(); it != file_locations.end(); ++it) { + uint32_t file_in_fat = fat_offset + it->second*8; + rom_file.seekg(file_in_fat, ios::beg); + uint32_t file_start = read_int(&rom_file); + inv_start_offsets[file_start] = it->first; + } + + uint32_t last_end_offset = 0; + + rom_file.seekg(0x80, ios::beg); + uint32_t total_size = read_int(&rom_file); + cout << "total size: " << dec << total_size << endl; + + for (auto it = inv_start_offsets.begin(); it != inv_start_offsets.end(); ++it) { + uint32_t file_fat_offset = fat_offset + file_locations[it->second]*8; + rom_file.seekg(file_fat_offset, ios::beg); + uint32_t file_start = read_int(&rom_file); + uint32_t file_end = read_int(&rom_file); + uint32_t file_size = file_end - file_start; + + uint32_t new_file_start = file_start; + uint32_t new_file_end = file_end; + + if (file_start < last_end_offset) { + new_file_start = round_to_multiple(last_end_offset, 512); + + // pad with FF between last_end offset and new start + out_file.seekp(last_end_offset, ios::beg); + for (uint32_t i = last_end_offset; i < new_file_start; i++) + out_file.put((uint8_t) 0xFF); + + } + + if (str_ends_with(it->second, ".map.dat.lz") and !str_ends_with(it->second, "m001_013.map.dat.lz")) { + char *file_bytes = new char[file_size]; + rom_file.seekg(file_start, ios::beg); + rom_file.read(file_bytes, file_size); + + // decompress map file + uint8_t* decompressed_file; + uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); + + delete[] file_bytes; + + // unpack NARC + vector> narc_offsets = decode_NARC(decompressed_file); + uint32_t lyr_start; + for (auto jt = narc_offsets.begin(); jt != narc_offsets.end(); ++jt) { + uint32_t stamp = byte_array_to_int((char*) decompressed_file + jt->first, true); + if (stamp == LYR_MAGIC) { + lyr_start = jt->first; + } + } + + // decode NARC of LYR file + vector> lyr_narc_offsets = decode_NARC(decompressed_file + lyr_start + 0x4); + + for (auto jt = lyr_narc_offsets.begin(); jt != lyr_narc_offsets.end(); ++jt) { + uint32_t narc_container_start = jt->first; + vector> layers = decode_NARC(decompressed_file + lyr_start + 0x4 + narc_container_start); + + // find LYR layer that contains pokemon data and randomize it + for (auto kt = layers.begin(); kt != layers.end(); ++kt) { + uint32_t start = lyr_start + 0x4 + narc_container_start + kt->first; + uint32_t identifier = byte_array_to_int((char*) decompressed_file + start); + if (identifier == 0x9) { // Pokemon data + uint32_t num_entries = byte_array_to_int((char*) decompressed_file + start + 0x4); + for (uint32_t i = 0; i < num_entries; i++) { + + uint32_t pokeid_offset = 0x6 + (i*0xA); + uint16_t poke_id = byte_array_to_short((char*) decompressed_file + start + 0x8 + pokeid_offset); + + vector replacement_mons = pds.get_pokemon_with_geq_field_move(poke_id); + uint16_t new_pokemon = replacement_mons[rand() % replacement_mons.size()]; + + decompressed_file[start+0x8+pokeid_offset] = new_pokemon & 0xFF; + decompressed_file[start+0x8+pokeid_offset+1] = (new_pokemon >> 8) & 0xFF; + } + } + } + } + + // compress file + vector compressed_file; + uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); + + delete[] decompressed_file; + + new_file_end = new_file_start + compressed_size; + + out_file.seekp(file_fat_offset, ios::beg); + write_int((ofstream*) &out_file, new_file_start); // write new start in FAT + + out_file.seekp(file_fat_offset + 4, ios::beg); + write_int((ofstream*) &out_file, new_file_end); // write new end in FAT + + out_file.seekp(new_file_start, ios::beg); + out_file.write((char*) compressed_file.data(), compressed_size); + + for (uint32_t i = compressed_size; i < file_size; i++) + out_file.put((uint8_t) 0xFF); + + } else if (new_file_start != file_start) { + // need to copy over file starting at new offset + char *file_bytes = new char[file_size]; + rom_file.seekg(file_start, ios::beg); + rom_file.read(file_bytes, file_size); + + out_file.seekp(new_file_start, ios::beg); + out_file.write(file_bytes, file_size); + + delete[] file_bytes; + + new_file_end = new_file_start + file_size; + + out_file.seekp(file_fat_offset, ios::beg); + write_int((ofstream*) &out_file, new_file_start); // write new start in FAT + + out_file.seekp(file_fat_offset + 4, ios::beg); + write_int((ofstream*) &out_file, new_file_end); // write new end in FAT + } + + last_end_offset = new_file_end; + } + + cout << "new rom size: " << last_end_offset << endl; + + // update total size in header + out_file.seekp(0x80, ios::beg); + write_int((ofstream*) &out_file, last_end_offset); + + // recompute header checksum + uint8_t header_data[0x15E]; + out_file.seekg(0, ios::beg); + out_file.read((char *) header_data, 0x15E); + + // https://github.com/Luca1991/NDSFactory/blob/master/ndsfactory/ndsfactory.cpp#L156 + uint16_t header_checksum = compute_crc16(header_data, 0x15E); + uint8_t checksum_LE[] = {(uint8_t) (header_checksum & 0xFF), + (uint8_t) ((header_checksum >> 8) & 0xFF)}; + + out_file.seekp(0x15E, ios::beg); + out_file.write((char *) checksum_LE, 2); + + cout << "new header checksum: " << hex << header_checksum << endl; +} + +void RomProcessor::get_file_locations () { + // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#dsmemorymaps + // mapping of filenames (strings) to FAT offsets + rom_file.seekg(0x40, ios::beg); + uint32_t fnt_offset = read_int(&rom_file); + + cout << "FNT offset: " << hex << fnt_offset << endl; + cout << "FAT offset: " << hex << fat_offset << endl; + + // Go to start of fnt + rom_file.seekg(fnt_offset, ios::beg); + + uint32_t subtable_offset = read_int(&rom_file); + uint16_t first_file_id = read_short(&rom_file); + + cout << "First subtable offset: " << hex << subtable_offset << endl; + cout << "First file id: " << hex << first_file_id << endl; + + explore_subtable(subtable_offset, fnt_offset, "", &first_file_id); +} + +void RomProcessor::explore_subtable (uint32_t subtable_offset, + uint32_t fnt_offset, string path, uint16_t *file_id) { + + rom_file.seekg(fnt_offset + subtable_offset, ios::beg); + + // get first byte + char length_type_byte = 0; + rom_file.read(&length_type_byte, 1); + + uint32_t length_type = (uint32_t) (unsigned char)length_type_byte; + + while (length_type != 0) { + if (length_type & 0x80) { + // this is a directory + // get name, appended to end of path + char *new_path_bytes = new char[path.length() + length_type - 0x80 + 2]; + strcpy(new_path_bytes, path.c_str()); + rom_file.read(new_path_bytes + path.length(), length_type - 0x80); + + new_path_bytes [path.length() + length_type - 0x80] = '/'; + new_path_bytes[path.length() + length_type - 0x80+1] = '\0'; + + string new_path(new_path_bytes); + + // after name, offset for subdirectory from start of fnt + uint32_t subdir_id = (uint32_t) read_short(&rom_file); + + int pos_in_subtable = rom_file.tellg(); + + rom_file.seekg(fnt_offset + (subdir_id & 0xFFF)*8, ios::beg); + + uint32_t new_subtable = read_int(&rom_file); + + uint16_t first_file_id = read_short(&rom_file); + + // now, explore this subtable + explore_subtable (new_subtable, fnt_offset, new_path, + &first_file_id); + + // move filestream back to continue while loop + rom_file.seekg(pos_in_subtable, ios::beg); + + delete [] new_path_bytes; + } else { + char *filename_bytes = new char[length_type + 1]; + rom_file.read(filename_bytes, length_type); + filename_bytes[length_type] = '\0'; + + string filename_str(filename_bytes); + string full_path_str(path); + + full_path_str += filename_str; + + file_locations.insert(pair(full_path_str, *file_id)); + *file_id += 1; + + delete[] filename_bytes; + } + + rom_file.read(&length_type_byte, 1); + length_type = (uint32_t) (unsigned char)length_type_byte; + } +} + +void RomProcessor::read_pokeID_rom (uint32_t offset) { + rom_file.seekg(offset, ios::beg); + + // size of file + uint32_t total_size = read_int(&rom_file); + + cout << "File size: " << dec << total_size << " bytes" << endl; + + // other header stuff + uint32_t unique_size = read_int(&rom_file); + uint32_t data_size = read_int(&rom_file); + + char unknown_bytes[4]; + rom_file.read(&unknown_bytes[0], 4); + + + cout << "Unique size: " << hex << unique_size << endl; + cout << "Data size: " << hex << data_size << endl; + + assert(unique_size == UNIQUE_SIZE); + + char unique[UNIQUE_SIZE]; + + rom_file.read(&unique[0], UNIQUE_SIZE); + + for (int i = 0; i < UNIQUE_SIZE; i ++) { + assert(unique[i] == UNIQUE[i]); + } + + uint32_t num_entries = data_size / ENTRY_SIZE; + + cout << "Number of entries: " << dec << num_entries << endl; + + for (uint32_t i = 0; i < num_entries; i ++) { + char entry_data[ENTRY_SIZE]; + rom_file.read(&entry_data[0], ENTRY_SIZE); + + uint16_t name_id = byte_array_to_short(&entry_data[0]); + uint8_t field_id = (uint8_t) entry_data[5]; + uint8_t field_level = (uint8_t) entry_data[6]; + + pds.add_pokemon(i, name_id, field_id, field_level); + } +} \ No newline at end of file diff --git a/romprocessor.h b/romprocessor.h new file mode 100644 index 0000000..07e6b53 --- /dev/null +++ b/romprocessor.h @@ -0,0 +1,48 @@ +#ifndef ROMPROCESSOR_H +#define ROMPROCESSOR_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pokedatastructure.h" +#include "util.h" + +// For PokeID.bin: each pokemon entry is 28 bytes long +// from: https://github.com/SunakazeKun/AlmiaE/blob/6af31527f579037b71fe3461309820f481c42be9/src/com/aurum/almia/game/param/PokeID.java +#define ENTRY_SIZE 0x1C +#define UNIQUE_SIZE 24 + +class RomProcessor { +public: + RomProcessor(const char *in_path, const char *out_path); + ~RomProcessor(); + + void randomize_rom(); + +private: + std::ifstream rom_file; std::fstream out_file; + std::map file_locations; + PokeDataStructure pds; + + uint32_t fat_offset; + char UNIQUE[24] = { + 0x07, 0x03, 0x01, 0x07, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x01, 0x02, + 0x02, 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x05 + }; + + void get_file_locations(); + void read_pokeID_rom (uint32_t offset); + void process_map_files (); + + void explore_subtable (uint32_t subtable_offset, uint32_t fnt_offset, + std::string path, uint16_t *file_id); + +}; + +#endif \ No newline at end of file diff --git a/test.cpp b/test.cpp deleted file mode 100644 index 77e1297..0000000 --- a/test.cpp +++ /dev/null @@ -1,532 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "util.h" -#include "lcs.h" -#include "pokeid.h" -#include "pokedatastructure.h" - -using namespace std; - -const char* rom_file_path = "rom.nds"; -const char* out_file_path = "randomized.nds"; - -// DFS through subtable, return a map from filepath to FAT offset -void explore_subtable (uint32_t subtable_offset, uint32_t fnt_offset, string path, uint16_t *file_id, map *file_locations) { - ifstream rom_file; - rom_file.open(rom_file_path); - - rom_file.seekg(fnt_offset + subtable_offset, ios::beg); - - // get first byte - char length_type_byte = 0; - rom_file.read(&length_type_byte, 1); - - uint32_t length_type = (uint32_t) (unsigned char)length_type_byte; - - while (length_type != 0) { - if (length_type & 0x80) { - // this is a directory - // get name, appended to end of path - char *new_path_bytes = new char[path.length() + length_type - 0x80 + 2]; - strcpy(new_path_bytes, path.c_str()); - rom_file.read(new_path_bytes + path.length(), length_type - 0x80); - - new_path_bytes [path.length() + length_type - 0x80] = '/'; - new_path_bytes[path.length() + length_type - 0x80+1] = '\0'; - - string new_path(new_path_bytes); - - // after name, offset for subdirectory from start of fnt - uint32_t subdir_id = (uint32_t) read_short(&rom_file); - - int pos_in_subtable = rom_file.tellg(); - - rom_file.seekg(fnt_offset + (subdir_id & 0xFFF)*8, ios::beg); - - uint32_t new_subtable = read_int(&rom_file); - - uint16_t first_file_id = read_short(&rom_file); - - // now, explore this subtable - explore_subtable (new_subtable, fnt_offset, new_path, &first_file_id, file_locations); - - // move filestream back to continue while loop - rom_file.seekg(pos_in_subtable, ios::beg); - - delete [] new_path_bytes; - } else { - char *filename_bytes = new char[length_type + 1]; - rom_file.read(filename_bytes, length_type); - filename_bytes[length_type] = '\0'; - - string filename_str(filename_bytes); - string full_path_str(path); - - full_path_str += filename_str; - - // cout << full_path_str << endl; - - file_locations->insert(pair(full_path_str, *file_id)); - *file_id += 1; - - delete[] filename_bytes; - } - - rom_file.read(&length_type_byte, 1); - length_type = (uint32_t) (unsigned char)length_type_byte; - } - - rom_file.close(); -} - -map get_file_locations () { - // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#dsmemorymaps - // mapping of filenames (strings) to FAT offsets - map file_locations; - - ifstream rom_file; - rom_file.open(rom_file_path); - - rom_file.seekg(0x40, ios::beg); - uint32_t fnt_offset = read_int(&rom_file); - - rom_file.seekg(0x48, ios::beg); - uint32_t fat_offset = read_int(&rom_file); - - cout << "FNT offset: " << hex << fnt_offset << endl; - cout << "FAT offset: " << hex << fat_offset << endl; - - // Go to start of fnt - rom_file.seekg(fnt_offset, ios::beg); - - uint32_t subtable_offset = read_int(&rom_file); - uint16_t first_file_id = read_short(&rom_file); - - cout << "First subtable offset: " << hex << subtable_offset << endl; - cout << "First file id: " << hex << first_file_id << endl; - - explore_subtable(subtable_offset, fnt_offset, "", &first_file_id, &file_locations); - - rom_file.close(); - return file_locations; -} - -// https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski -// https://magikos.livejournal.com/7375.html? -// https://github.com/SciresM/FEAT/blob/master/FEAT/DSDecmp/Formats/Nitro/LZ10.cs#L83 -uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, uint8_t **uncompressed_data) { - assert(compressed_size >= 4); - - // First byte 0x10 means file is LZ10 compressed - assert(*compressed_bytes == 0x10); - - // next three bytes are size of uncompressed file - uint32_t uncompressed_size = (uint8_t) *(compressed_bytes + 3) << 16 | - (uint8_t) *(compressed_bytes + 2) << 8 | - (uint8_t) *(compressed_bytes + 1); - - uint8_t* uncompressed_file = new uint8_t[uncompressed_size]; - - uint32_t input_pos = 4; - uint32_t output_pos = 0; - - while (output_pos < uncompressed_size) { - assert(input_pos < compressed_size); - uint8_t flag_byte = (uint8_t) compressed_bytes[input_pos++]; - - for (int i = 1; i <= 8 and output_pos < uncompressed_size and input_pos < compressed_size; i ++) { - bool flag_bit = (flag_byte >> 8-i) & 1; - if (flag_bit) { - // Dictionary entry - uint8_t token1 = (uint8_t) compressed_bytes[input_pos++]; - uint8_t token2 = (uint8_t) compressed_bytes[input_pos++]; - - int disp = ((token1 & 0x0F) << 8) | token2; disp += 1; - uint8_t length = (token1 >> 4) + 3; // plus 3 for some reason? - - int read_start = output_pos - disp; - - assert (disp <= output_pos); - - for (int j = 0; j < length; j++) { - uncompressed_file[output_pos++] = uncompressed_file[read_start + (j % disp)]; - } - } else { - // Raw byte - uncompressed_file[output_pos++] = compressed_bytes[input_pos++]; - } - } - } - - *uncompressed_data = uncompressed_file; - return uncompressed_size; -} - -uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, vector *compressed_data) { - assert(uncompressed_size <= 0xFFFFFF); - - vector buffer; - - buffer.push_back(0x10); - // load uncompressed size into buffer - buffer.push_back((uint8_t) (uncompressed_size & 0xFF)); - buffer.push_back((uint8_t) ((uncompressed_size >> 8) & 0xFF)); - buffer.push_back((uint8_t) ((uncompressed_size >> 16) & 0xFF)); - - int max_disp = 0xFFF; - int max_length = 0xF + 3; - - uint32_t cur_position = 0; - - while (cur_position < uncompressed_size) { - // https://en.wikipedia.org/wiki/Longest_common_substring - // get longest common substring (up to max length of dictionary entry) - // if passes threshold (length >= 3), make it a dictionary entry - // otherwise just copy raw bytes - - buffer.push_back(0); // flag byte - int flag_index = buffer.size() - 1; - - uint8_t flag_byte = 0; - - for (int i = 1; i <= 8 and cur_position < uncompressed_size; i++) { - - // get longest common substring - int start_idx = get_max(0, cur_position - max_disp - 1); - int length_from_start = cur_position - start_idx; - int length_from_current = uncompressed_size - cur_position; - - int best_run_length = 0; - int best_run_start = start_idx; - int cur_run_start = start_idx; - - for (int j = start_idx; j < cur_position and best_run_length < max_length; j++) { - int length = 0, k = 0; - while (cur_position + k < uncompressed_size && uncompressed_data[j+k] == uncompressed_data[cur_position+k] && length < max_length) { - length ++; k++; - } - - if (length > best_run_length) { - best_run_length = get_min(length, max_length); - best_run_start = j; - } - } - - if (best_run_length < 3) // store as raw byte - buffer.push_back(uncompressed_data[cur_position++]); - else { // store as dictionary entry - uint16_t disp = cur_position - best_run_start -1; - - uint8_t length = best_run_length -3; - uint8_t token1 = ((disp >> 8) & 0x0F) | ((length << 4) & 0xF0); - uint8_t token2 = disp & 0xFF; - - buffer.push_back(token1); buffer.push_back(token2); - - flag_byte |= 1 << (8-i); // set corresponding flag bit - cur_position += best_run_length; - } - - } - - buffer[flag_index] = flag_byte; - } - - uint32_t compressed_size = buffer.size(); - - *compressed_data = buffer; - - return compressed_size; -} - -#define NARC_MAGIC 0x4E415243 -#define NARC_FATB 0x46415442 -#define NARC_FNTB 0x464E5442 -#define NARC_FIMG 0x46494D47 -#define LYR_MAGIC 0x4C595200 - -// http://llref.emutalk.net/docs/?file=xml/narc.xml#xml-doc -// http://www.pipian.com/ierukana/hacking/ds_narc.html -// https://www.romhacking.net/documents/%5B469%5Dnds_formats.htm#NARC -// returns vector of start/end offsets for files -vector> decode_NARC (uint8_t *data) { - uint32_t magic = byte_array_to_int((char*) data, true); // big endian due to byte order of magic (bom) - assert(magic == NARC_MAGIC); // magic is "NARC" - - uint32_t file_size = byte_array_to_int((char*) data +0x8); - - uint32_t fatb_offset = 0x10; - uint32_t fatb_stamp = byte_array_to_int((char*) data + fatb_offset); - assert(fatb_stamp == NARC_FATB); - - uint32_t num_files_offset = fatb_offset + 0x8; - uint32_t num_files = byte_array_to_int((char*) data + num_files_offset); - - vector> file_positions; - - for (int i = 0; i < num_files; i++) { - uint32_t start_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + i*8); - uint32_t end_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + (i*8) + 0x4); - file_positions.push_back(pair(start_offset, end_offset)); - } - - uint32_t fntb_offset = num_files_offset + 0x4 + num_files * 8; - uint32_t fntb_stamp = byte_array_to_int((char*) data + fntb_offset); - assert(fntb_stamp == NARC_FNTB); - - uint32_t fntb_size = byte_array_to_int((char *) data + fntb_offset + 0x4); - - uint32_t fimg_offset = fntb_offset + fntb_size; - - uint32_t fimg_stamp = byte_array_to_int((char *) data + fimg_offset); - - assert(fimg_stamp == NARC_FIMG); - - // update vector so it contains absolute offsets (not just relative to start of FIMG) - for (auto it = file_positions.begin(); it != file_positions.end(); ++it) { - it->first = fimg_offset + 0x8 + it->first; // +8 b/c relative offsets start after FIMG header - it->second = fimg_offset + 0x8 + it->second; - } - - return file_positions; -} - -void process_map_files (PokeDataStructure* pds, map file_locations, uint32_t fat_offset) { - map inv_start_offsets; // sorted by key in increasing order by default - - ifstream rom_file(rom_file_path); - fstream out_file(out_file_path); - - rom_file.seekg(0x20, ios::beg); - uint32_t arm9_offset = read_int(&rom_file); - uint32_t arm9_size = read_int(&rom_file); - - for (auto it = file_locations.begin(); it != file_locations.end(); ++it) { - uint32_t file_in_fat = fat_offset + it->second*8; - rom_file.seekg(file_in_fat, ios::beg); - uint32_t file_start = read_int(&rom_file); - inv_start_offsets[file_start] = it->first; - } - - uint32_t last_end_offset = 0; - - for (auto it = inv_start_offsets.begin(); it != inv_start_offsets.end(); ++it) { - uint32_t file_fat_offset = fat_offset + file_locations[it->second]*8; - rom_file.seekg(file_fat_offset, ios::beg); - uint32_t file_start = read_int(&rom_file); - uint32_t file_end = read_int(&rom_file); - uint32_t file_size = file_end - file_start; - - uint32_t new_file_start = file_start; - uint32_t new_file_end = file_end; - - if (file_start < last_end_offset) { - new_file_start = round_to_multiple(last_end_offset, 512); - - // pad with FF between last_end offset and new start - out_file.seekp(last_end_offset, ios::beg); - for (uint32_t i = last_end_offset; i < new_file_start; i++) - out_file.put(0xFF); - - } - - if (str_ends_with(it->second, ".map.dat.lz") and !str_ends_with(it->second, "m001_013.map.dat.lz")) { - char file_bytes[file_size]; - rom_file.seekg(file_start, ios::beg); - rom_file.read(file_bytes, file_size); - - // decompress map file - uint8_t* decompressed_file; - uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); - - // unpack NARC - vector> narc_offsets = decode_NARC(decompressed_file); - uint32_t lyr_start, lyr_end; - for (auto it = narc_offsets.begin(); it != narc_offsets.end(); ++it) { - uint32_t stamp = byte_array_to_int((char*) decompressed_file + it->first, true); - if (stamp == LYR_MAGIC) { - lyr_start = it->first; lyr_end = it->second; - } - } - - // decode NARC of LYR file - vector> lyr_narc_offsets = decode_NARC(decompressed_file + lyr_start + 0x4); - - for (auto jt = lyr_narc_offsets.begin(); jt != lyr_narc_offsets.end(); ++jt) { - uint32_t narc_container_start = jt->first; - vector> layers = decode_NARC(decompressed_file + lyr_start + 0x4 + narc_container_start); - - // find LYR layer that contains pokemon data and randomize it - for (auto it = layers.begin(); it != layers.end(); ++it) { - uint32_t start = lyr_start + 0x4 + narc_container_start + it->first; - uint32_t identifier = byte_array_to_int((char*) decompressed_file + start); - if (identifier == 0x9) { // Pokemon data - uint32_t num_entries = byte_array_to_int((char*) decompressed_file + start + 0x4); - for (int i = 0; i < num_entries; i++) { - - uint32_t pokeid_offset = 0x6 + (i*0xA); - uint16_t poke_id = byte_array_to_short((char*) decompressed_file + start + 0x8 + pokeid_offset); - - vector replacement_mons = pds->get_pokemon_with_geq_field_move(poke_id); - uint16_t new_pokemon = replacement_mons[rand() % replacement_mons.size()]; - - decompressed_file[start+0x8+pokeid_offset] = new_pokemon & 0xFF; - decompressed_file[start+0x8+pokeid_offset+1] = (new_pokemon >> 8) & 0xFF; - } - } - } - } - - // compress file - vector compressed_file; - uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); - - new_file_end = new_file_start + compressed_size; - - if (round_to_multiple(new_file_end, 512) > round_to_multiple(file_end,512)) continue; - - out_file.seekp(file_fat_offset, ios::beg); - write_int((ofstream*) &out_file, new_file_start); // write new start in FAT - - out_file.seekp(file_fat_offset + 4, ios::beg); - write_int((ofstream*) &out_file, new_file_end); // write new end in FAT - - out_file.seekp(new_file_start, ios::beg); - out_file.write((char*) compressed_file.data(), compressed_size); - - for (uint32_t i = compressed_size; i < file_size; i++) - out_file.put(0xFF); - - delete[] decompressed_file; - - } else if (new_file_start != file_start) { - // need to copy over file starting at new offset - char file_bytes[file_size]; - rom_file.seekg(file_start, ios::beg); - rom_file.read(file_bytes, file_size); - - out_file.seekp(new_file_start, ios::beg); - out_file.write(file_bytes, file_size); - - new_file_end = new_file_start + file_size; - - out_file.seekp(file_fat_offset, ios::beg); - write_int((ofstream*) &out_file, new_file_start); // write new start in FAT - - out_file.seekp(file_fat_offset + 4, ios::beg); - write_int((ofstream*) &out_file, new_file_end); // write new end in FAT - } - - last_end_offset = new_file_end; - } - - cout << "new rom size: " << last_end_offset << endl; - - // update total size in header - out_file.seekp(0x80, ios::beg); - write_int((ofstream*) &out_file, last_end_offset); - - // recompute header checksum - uint8_t header_data[0x15E]; - out_file.seekg(0, ios::beg); - out_file.read((char *) header_data, 0x15E); - - // https://github.com/Luca1991/NDSFactory/blob/master/ndsfactory/ndsfactory.cpp#L156 - uint16_t header_checksum = compute_crc16(header_data, 0x15E); - uint8_t checksum_LE[] = {(uint8_t) (header_checksum & 0xFF), - (uint8_t) ((header_checksum >> 8) & 0xFF)}; - - out_file.seekp(0x15E, ios::beg); - out_file.write((char *) checksum_LE, 2); - - cout << "new header checksum: " << hex << header_checksum << endl; - - // TODO decompress .map.dat.lz files (LZ10 compression) - // https://github.com/SunakazeKun/AlmiaE/blob/master/src/com/aurum/almia/game/Compression.java - // https://ndspy.readthedocs.io/en/latest/_modules/ndspy/lz10.html#decompress - - // m001_004 - // uint32_t file_fat_offset = fat_offset + file_locations["data/field/map/m001_004.map.dat.lz"]*8; - // rom_file.seekg(file_fat_offset, ios::beg); - // uint32_t file_start = read_int(&rom_file); - // uint32_t file_end = read_int(&rom_file); - // uint32_t file_size = file_end - file_start; - - // char file_bytes[file_size]; - // rom_file.seekg(file_start, ios::beg); - // rom_file.read(file_bytes, file_size); - - // uint8_t* decompressed_file; - // uint32_t decompressed_size = decompress_LZ10(file_bytes, file_size, &decompressed_file); - - // vector compressed_file; - // uint32_t compressed_size = compress_LZ10(decompressed_file, decompressed_size, &compressed_file); - - // ofstream decomp_file("recompressed_001_004.map.dat"); - // decomp_file.write((char *) compressed_file.data(), compressed_size); - - // delete[] decompressed_file; - - rom_file.close(); out_file.close(); -} - -int main () { - srand(time(NULL)); - - map file_locations = get_file_locations (); - - ifstream rom_file(rom_file_path); - - ofstream out_file(out_file_path); // copy over file - out_file << rom_file.rdbuf(); out_file.close(); - - rom_file.seekg(0x48, ios::beg); - uint32_t fat_offset = read_int(&rom_file); - - uint32_t pkmn_name_in_fat = fat_offset + file_locations["data/message/etc/pokemon_name_us.mes"]*8; - uint32_t field_move_in_fat = fat_offset + file_locations["data/message/etc/fieldwaza_name_us.mes"]*8; - uint32_t pokeid_bin_in_fat = fat_offset + file_locations["data/param/PokeID.bin"]*8; - - rom_file.seekg(pkmn_name_in_fat, ios::beg); - uint32_t pkmn_name_offset = read_int(&rom_file); - - rom_file.seekg(field_move_in_fat, ios::beg); - uint32_t field_move_offset = read_int(&rom_file); - - rom_file.seekg(pokeid_bin_in_fat, ios::beg); - uint32_t pokeid_bin_offset = read_int(&rom_file); - - cout << "Pokemon name offset: " << pkmn_name_offset << endl; - cout << "Field name offset: " << field_move_offset << endl; - cout << "PokeID bin offset: " << pokeid_bin_offset << endl; - - vector pkmn_names = get_pokemon_names_rom (rom_file_path, - pkmn_name_offset); - vector field_moves = get_field_moves_rom (rom_file_path, - field_move_offset); - - // cout << pkmn_names.size() << endl; - - PokeDataStructure pds; - - // TODO: make this a PDS member function - read_pokeID_rom (&pds, rom_file_path, pokeid_bin_offset); - - rom_file.seekg(0x80, ios::beg); - uint32_t total_size = read_int(&rom_file); - cout << "total size: " << dec << total_size << endl; - - process_map_files(&pds, file_locations, fat_offset); - - rom_file.close(); - - return 0; -} \ No newline at end of file diff --git a/util.cpp b/util.cpp index a6ee1d1..7d17302 100644 --- a/util.cpp +++ b/util.cpp @@ -77,7 +77,7 @@ uint32_t round_to_multiple (uint32_t n, uint32_t multiple) { // https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#biosmiscfunctions // https://github.com/Zetten/hachoir/blob/master/hachoir-parser/hachoir_parser/program/nds.py uint16_t compute_crc16 (uint8_t *bytes, int size) { - uint16_t val[8] = {0xC0C1,0xC181,0xC301,0xC601,0xCC01,0xD801,0xF001,0xA001}; + // uint16_t val[8] = {0xC0C1,0xC181,0xC301,0xC601,0xCC01,0xD801,0xF001,0xA001}; uint16_t crc = 0xFFFF; for (int i = 0; i < size; i++) { @@ -91,4 +91,172 @@ uint16_t compute_crc16 (uint8_t *bytes, int size) { } return crc; +} + +// https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski +// https://magikos.livejournal.com/7375.html? +// https://github.com/SciresM/FEAT/blob/master/FEAT/DSDecmp/Formats/Nitro/LZ10.cs#L83 +uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, uint8_t **uncompressed_data) { + assert(compressed_size >= 4); + + // First byte 0x10 means file is LZ10 compressed + assert(*compressed_bytes == 0x10); + + // next three bytes are size of uncompressed file + uint32_t uncompressed_size = (uint8_t) *(compressed_bytes + 3) << 16 | + (uint8_t) *(compressed_bytes + 2) << 8 | + (uint8_t) *(compressed_bytes + 1); + + uint8_t* uncompressed_file = new uint8_t[uncompressed_size]; + + uint32_t input_pos = 4; + uint32_t output_pos = 0; + + while (output_pos < uncompressed_size) { + assert(input_pos < compressed_size); + uint8_t flag_byte = (uint8_t) compressed_bytes[input_pos++]; + + for (int i = 1; i <= 8 and output_pos < uncompressed_size and input_pos < compressed_size; i ++) { + bool flag_bit = (flag_byte >> (8-i)) & 1; + if (flag_bit) { + // Dictionary entry + uint8_t token1 = (uint8_t) compressed_bytes[input_pos++]; + uint8_t token2 = (uint8_t) compressed_bytes[input_pos++]; + + int disp = ((token1 & 0x0F) << 8) | token2; disp += 1; + uint8_t length = (token1 >> 4) + 3; // plus 3 for some reason? + + int read_start = output_pos - disp; + + assert ((uint32_t) disp <= output_pos); + + for (int j = 0; j < length; j++) { + uncompressed_file[output_pos++] = uncompressed_file[read_start + (j % disp)]; + } + } else { + // Raw byte + uncompressed_file[output_pos++] = compressed_bytes[input_pos++]; + } + } + } + + *uncompressed_data = uncompressed_file; + return uncompressed_size; +} + +uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, vector *compressed_data) { + assert(uncompressed_size <= 0xFFFFFF); + + vector buffer; + + buffer.push_back(0x10); + // load uncompressed size into buffer + buffer.push_back((uint8_t) (uncompressed_size & 0xFF)); + buffer.push_back((uint8_t) ((uncompressed_size >> 8) & 0xFF)); + buffer.push_back((uint8_t) ((uncompressed_size >> 16) & 0xFF)); + + int max_disp = 0xFFF; + int max_length = 0xF + 3; + + uint32_t cur_position = 0; + + while (cur_position < uncompressed_size) { + // https://en.wikipedia.org/wiki/Longest_common_substring + // get longest common substring (up to max length of dictionary entry) + // if passes threshold (length >= 3), make it a dictionary entry + // otherwise just copy raw bytes + + buffer.push_back(0); // flag byte + int flag_index = buffer.size() - 1; + + uint8_t flag_byte = 0; + + for (int i = 1; i <= 8 and cur_position < uncompressed_size; i++) { + + // get longest common substring + int start_idx = get_max(0, cur_position - max_disp - 1); + int best_run_length = 0; + int best_run_start = start_idx; + + for (uint32_t j = start_idx; j < cur_position and best_run_length < max_length; j++) { + int length = 0, k = 0; + while (cur_position + k < uncompressed_size && uncompressed_data[j+k] == uncompressed_data[cur_position+k] && length < max_length) { + length ++; k++; + } + + if (length > best_run_length) { + best_run_length = get_min(length, max_length); + best_run_start = j; + } + } + + if (best_run_length < 3) // store as raw byte + buffer.push_back(uncompressed_data[cur_position++]); + else { // store as dictionary entry + uint16_t disp = cur_position - best_run_start -1; + + uint8_t length = best_run_length -3; + uint8_t token1 = ((disp >> 8) & 0x0F) | ((length << 4) & 0xF0); + uint8_t token2 = disp & 0xFF; + + buffer.push_back(token1); buffer.push_back(token2); + + flag_byte |= 1 << (8-i); // set corresponding flag bit + cur_position += best_run_length; + } + + } + + buffer[flag_index] = flag_byte; + } + + uint32_t compressed_size = buffer.size(); + + *compressed_data = buffer; + + return compressed_size; +} + +// http://llref.emutalk.net/docs/?file=xml/narc.xml#xml-doc +// http://www.pipian.com/ierukana/hacking/ds_narc.html +// https://www.romhacking.net/documents/%5B469%5Dnds_formats.htm#NARC +// returns vector of start/end offsets for files +vector> decode_NARC (uint8_t *data) { + uint32_t magic = byte_array_to_int((char*) data, true); // big endian due to byte order of magic (bom) + assert(magic == NARC_MAGIC); // magic is "NARC" + + uint32_t fatb_offset = 0x10; + uint32_t fatb_stamp = byte_array_to_int((char*) data + fatb_offset); + assert(fatb_stamp == NARC_FATB); + + uint32_t num_files_offset = fatb_offset + 0x8; + uint32_t num_files = byte_array_to_int((char*) data + num_files_offset); + + vector> file_positions; + + for (uint32_t i = 0; i < num_files; i++) { + uint32_t start_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + i*8); + uint32_t end_offset = byte_array_to_int((char *) data + num_files_offset + 0x4 + (i*8) + 0x4); + file_positions.push_back(pair(start_offset, end_offset)); + } + + uint32_t fntb_offset = num_files_offset + 0x4 + num_files * 8; + uint32_t fntb_stamp = byte_array_to_int((char*) data + fntb_offset); + assert(fntb_stamp == NARC_FNTB); + + uint32_t fntb_size = byte_array_to_int((char *) data + fntb_offset + 0x4); + + uint32_t fimg_offset = fntb_offset + fntb_size; + + uint32_t fimg_stamp = byte_array_to_int((char *) data + fimg_offset); + + assert(fimg_stamp == NARC_FIMG); + + // update vector so it contains absolute offsets (not just relative to start of FIMG) + for (auto it = file_positions.begin(); it != file_positions.end(); ++it) { + it->first = fimg_offset + 0x8 + it->first; // +8 b/c relative offsets start after FIMG header + it->second = fimg_offset + 0x8 + it->second; + } + + return file_positions; } \ No newline at end of file diff --git a/util.h b/util.h index 0ff7f8f..f2bdeaf 100644 --- a/util.h +++ b/util.h @@ -2,8 +2,16 @@ #define UTIL_H #include +#include #include #include +#include + +#define NARC_MAGIC 0x4E415243 +#define NARC_FATB 0x46415442 +#define NARC_FNTB 0x464E5442 +#define NARC_FIMG 0x46494D47 +#define LYR_MAGIC 0x4C595200 uint32_t byte_array_to_int (char* bytes, bool big_endian=false); uint16_t byte_array_to_short (char* bytes, bool big_endian=false); @@ -22,4 +30,13 @@ uint32_t round_to_multiple (uint32_t n, uint32_t multiple); uint16_t compute_crc16 (uint8_t *bytes, int size); +uint32_t decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size, + uint8_t **uncompressed_data); + +uint32_t compress_LZ10 (uint8_t *uncompressed_data, uint32_t uncompressed_size, + std::vector *compressed_data); + +std::vector> decode_NARC (uint8_t *data); + + #endif \ No newline at end of file