Leahnaya/almia-randomizer
1
0
Fork 0
mirror of https://github.com/ronitsinha/almia-randomizer.git synced 2026-09-09 02:45:23 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

stuff with LCP inside of suffix array algorithm

Browse Source
This commit is contained in:
ronitsinha
2023-06-13 19:28:47 -04:00
commit bf4734ca95
17 changed files with 9052 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile Normal file
View File

@@ -0,0 +1,8 @@
all: test
test: clean
g++ -Wall -Werror -Wpedantic util.cpp lcs.cpp pokedatastructure.cpp \
pokeid.cpp test.cpp -o test.out
clean:
rm -f *.out

BIN
PokeID.bin Normal file
View File

Binary file not shown.

BIN
fieldwaza_name_us.mes Normal file
View File

Binary file not shown.

285
lcs.cpp Normal file
View File

@@ -0,0 +1,285 @@
#include "lcs.h"
using namespace std;
/* */
/* Longest Common Substring */
/* */
// Sources:
// https://web.archive.org/web/20120616234651/http://www.cs.sysu.edu.cn/nong/index.files/Two%20Efficient%20Algorithms%20for%20Linear%20Suffix%20Array%20Construction.pdf
// https://link.springer.com/chapter/10.1007/978-3-642-22300-6_32
// https://zork.net/~st/jottings/sais.html
// https://www.youtube.com/watch?v=Ic80xQFWevc
// https://www.youtube.com/watch?v=OIuG_Dqyl_s
// https://link.springer.com/chapter/10.1007/978-3-540-79709-8_10
// http://www.cs.cmu.edu/~guyb/paralg/papers/KarkkainenSanders03.pdf
// https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=2151c8d46282010599d125f01015dc2fc3d2fe4d
// https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=7e70b4ca7d8f7598dccdf0b9f671b30f628fea9f
int longest_common_substring () {
return 0;
}
// Skew algorithm: linear time algorithm for suffix array and LCP calculation
// source: http://www.cs.cmu.edu/~guyb/paralg/papers/KarkkainenSanders03.pdf#cite.AGKR02
inline bool leq(int a1, int a2, int b1, int b2) // lexicographic order
{ return (a1 < b1 || (a1 == b1 && a2 <= b2)); } // for pairs
inline bool leq(int a1, int a2, int a3, int b1, int b2, int b3)
{ return (a1 < b1 || (a1 == b1 && leq(a2, a3, b2, b3))); } // and triples
// stably sort a[0...n-1] to b[0...n-1] with keys 0...K from string r
void radix_pass(int *a, int *b, int *r, int n, int K) {
int *c = new int[K + 1]; // counter array
for (int i = 0; i <= K; i++) c[i] = 0; // initialize counter
for (int i = 0; i < n; i++) c[r[a[i]]]++; // count occurences of each character
for (int i = 0, sum = 0; i <= K; i++) { // turn it into prefix array
int t = c[i];
c[i] = sum;
sum += t;
}
// put a[i] in the prefix denoted by c[r[a[i]]], then incremenet the prefix
// so next a[i] will be put after it
for (int i = 0; i < n; i++) b[c[r[a[i]]]++] = a[i]; // sort
delete [] c;
}
// range minimum query -- if in block
// https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=2151c8d46282010599d125f01015dc2fc3d2fe4d
// https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=7e70b4ca7d8f7598dccdf0b9f671b30f628fea9f
int rmq(int *L, int *LCP12, int i, int j, int block_size, int *left_min, int *right_min, int *M, int log_A_len) {
if (i > j) { // ensure i < j
int tmp = i;
i = j;
j = tmp;
}
int i_block = i / block_size;
int j_block = j / block_size;
if (i_block == j_block) { // i and j in same block
int w = L[j] & (~0 << i);
if (w == 0) return LCP12[j];
return 0;
}
// i and j in different blocks
// largest power of two that fits between i and j blocks, not inclusive
int num_blocks = (j_block-1) - (i_block+1) + 1;
if (num_blocks != 0) {
int k = (int)log2(num_blocks);
int block_min = get_min(LCP12[ M[(i_block+1)*log_A_len + k] ], LCP12[ M[((j_block-1) - (1 << k) + 1)*log_A_len + k] ]);
// minimum between (1) min values from blocks between i and j,
// not including i and j's blocks, (2) from i rightwards to the end of its
// block, and (3) from j leftwards to the beginning of its block.
return get_min(block_min, get_min(right_min[i], left_min[j]));
}
return get_min(right_min[i], left_min[j]);
}
void calculate_LCP(int *s, int *SA, int *LCP, int *s12, int *LCP12, int n, int n02) {
// preprocess LCP12 for O(1) range minimum queries
int block_size = (int)log2(n02); // number of bits of a number = log2 of that number
int *G = new int[n02]; // G[i] = index of min value from the start of i's block up to i
// since block size is a log, we know it cannot be larger than an int
// so each label l_i can fit in an int
int *L = new int[n02];
int *left_min = new int[n02]; // left_min[i] = minimum value over range [a, i], where a is the start of a block
int *right_min = new int[n02]; // right_min[i] = minimum value over range [i,b], where b is the end of a block
int A_len = n02 / block_size;
int log_A_len = (int)log2(A_len);
int *A_prime = new int[A_len]; // A_prime[i] = minimum value in ith block
int *M = new int[A_len * log_A_len]; // lookup table for sparse table on A'
// populate G and A'
for (int a = 0; a < n02; a += block_size) {
int argmin = a;
// compute g_i for each i
for (int b = 1; b < block_size; b++) {
if (LCP12[argmin] < LCP12[a+b]) G[a+b] = argmin;
else argmin = a+b;
}
// minimum value in this block
A_prime[a / block_size] = LCP[argmin];
}
// sparse table for A' using dynamic programming
for (int i = 0; i < A_len; i++) {
// for queries of length 1, min is just the only element
M[i*log_A_len + 0] = i;
}
// Compute values from smaller to bigger intervals
for (int j = 1; (1 << j) <= A_len; j++) {
// Compute minimum value for all intervals with size 2^j
for (int i = 0; (i + (1 << j) -1) < A_len; i++) {
if (LCP12[ M[i*log_A_len + (j-1)] ] <
LCP12[ M[(i + (1 << (j-1)))*log_A_len + (j-1)] ])
M[i*log_A_len + j] = M[i*log_A_len + (j-1)];
else
M[i*log_A_len + j] = M[(i + (1 << (j-1)))*log_A_len + (j-1)];
}
}
// compute L_i
for (int i = 0; i < n02; i++) {
if (i == 0) {
L[i] = 0;
} else {
L[i] = L[G[i]] | (1 << G[i]);
}
}
for (int i = 0; i < n-1; i ++) {
int j = SA[i];
int k = SA[i+1];
cout << j << " " << k << " test" << endl;
if (j % 3 == 1 or j % 3 == 2) {
int j2 = (j % 3 == 1) ? (j - 1)/3 : (n+j-2)/3;
if (k % 3 == 2 or k % 3 == 1) {
// int k2 = (k % 3 == 2) ? (n+k-2)/3 : (k-1)/3; // k'
// since j and k adjacent in SA, j2 and k2 adjacent in SA12
int l = LCP12[s12[j2]-1];
int lcp = 3*l;
// if there is shared lcp beyond the blocks of three (i.e. a remainder, at most 2)
for (int t=0,a=j+3*l, b=k+3*l; t < 2; t++) {
if (a+t >= n or b+t >= n) break;
if (s[a+t] == s[b+t]) lcp++;
}
LCP[i] = lcp;
} else { // k % 3 = 0
if (s[j] != s[k]) LCP[i] = 0;
else // range minimum query to compute l
LCP[i] = 1 + rmq(L, LCP12, j+1, k+1, block_size, left_min, right_min, M, log_A_len);
}
} else { // j % 3 = 0
if (s[j] != s[k]) LCP[i] = 0;
else // range minimum query to compute l
LCP[i] = 1 + rmq(L, LCP12, j+1, k+1, block_size, left_min, right_min, M, log_A_len);
}
}
delete [] G; delete [] L; delete[] left_min; delete [] right_min;
delete [] A_prime; delete [] M;
}
// find the suffix array SA of s[0...n-1] in alphabet of size K
// require s[n]=s[n+1]=s[n+2]=0 and n >= 2
void suffix_array(int *s, int *SA, int *LCP, int n, int K) {
int n0 = (n+2)/3, n1 = (n+1)/3, n2 = n/3, n02 = n0+n2;
int *s12 = new int[n02+3];
s12[n02] = s12[n02+1] = s12[n02+2] = 0; // pad the end of the string w/ zeros
int *SA12 = new int[n02 + 3];
SA12[n02] = SA12[n02+1] = SA12[n02+2] = 0;
int *LCP12 = new int[n02];
int *s0 = new int[n0];
int *SA0 = new int[n0];
/* First step: sort suffixes S_i with i % 3 != 0 */
// +(n0-n1) generates a dummy mod 1 suffix if n%3 == 1
for (int i = 0, j = 0; i < n+(n0-n1); i++) if (i%3 != 0) s12[j++] = i;
// lsb radix sort the mod 1 and mod 2 triplets
radix_pass(s12, SA12, s+2, n02, K); // radix pass on every third character
radix_pass(SA12, s12, s+1, n02, K); // on every second character
radix_pass(s12, SA12, s, n02, K); // on every first character
// find lexicographic names of triples
int name = 0, c0 = -1, c1 = -1, c2 = -1;
for (int i = 0; i < n02; i++) {
if (s[SA12[i]] != c0 or s[SA12[i]+1] != c1 or s[SA12[i]+2] != c2) {
name ++; // names start at 1
if (i > 0) {
int overlap = 0;
if (c0 == s[SA12[i]] and c1 == s[SA12[i]+1]) overlap = 2;
else if (c0 == s[SA12[i]]) overlap = 1;
LCP12[i-1] = overlap;
}
c0 = s[SA12[i]]; c1 = s[SA12[i]+1]; c2 = s[SA12[i]+2];
} else
if (i > 0) LCP12[i-1] = 3;
// s12 = [s_i : i % 3 = 1] + [s_i : i % 3 = 2]
// and s12 is of length 2n/3, so first half (0 to n/3) is for mod 1
// and second half (n/3 to 2n/3) is for mod 2
if (SA12[i] % 3 == 1)
s12[SA12[i]/3] = name; // left half
else {
s12[SA12[i]/3 + n0] = name; // right half
}
}
// recurse if names are not unique
if (name < n02) {
suffix_array(s12, SA12, LCP12, n02, name);
// store unique names in s12 using the suffix array
for (int i = 0; i < n02; i++) s12[SA12[i]] = i + 1;
} else { // generate suffix array of s12 directly
for (int i = 0; i < n02; i++) SA12[s12[i] - 1] = i;
}
// stably sort the mod 0 suffixes from SA12 by their first character
for (int i=0, j=0; i < n02; i++) if (SA12[i] < n0) s0[j++] = 3*SA12[i];
radix_pass(s0, SA0, s, n0, K);
// merge sorted SA0 suffixes and SA12 suffixes
for (int p=0, t=n0-n1, k=0; k < n; k++) {
#define GetI() (SA12[t] < n0 ? SA12[t]*3 + 1 : (SA12[t]-n0)*3 + 2)
int i = GetI(); // position of current offset 12 suffix
int j = SA0[p]; // position of current offset 0 suffix
if (SA12[t] < n0 ? // different compares for mod 1 and mod 2 suffixes
leq(s[i], s12[SA12[t] + n0], s[j], s12[j/3]) :
leq(s[i], s[i+1], s12[SA12[t]-n0+1], s[j], s[j+1], s12[j/3+n0])) // compare triple here because i+1 mod 3 is 0
{ // suffix from SA12 is smaller
SA[k] = i; t++;
if (t == n02) // done -- only SA0 suffixes left
for (k ++; p < n0; p++, k++) SA[k] = SA0[p];
} else { // suffix from SA0 is smaller
SA[k] = j; p++;
if (p == n0) // done -- only SA12 suffixes left
for (k++; t < n02; t++, k++) SA[k] = GetI();
}
}
// at this point, s12 contains SA12' (bar over SA12), the unique names of the triples
// compute LCP from LCP12
calculate_LCP(s, SA, LCP, s12, LCP12, n, n02);
// cout << "LCP: ";
// for (int i = 0; i < n02; i++) cout << LCP12[i] << " ";
// cout << endl;
delete [] s12; delete [] SA12; delete [] SA0; delete [] s0; delete [] LCP12;
}

14
lcs.h Normal file
View File

@@ -0,0 +1,14 @@
#ifndef LCS_H
#define LCS_H
#include <iostream>
#include <stdint.h>
#include <string.h>
#include <math.h> /* log2 */
#include "util.h"
int longest_common_substring();
void suffix_array(int *s, int *SA, int *LCP, int n, int K);
#endif

337
pokedatastructure.cpp Normal file
View File

@@ -0,0 +1,337 @@
#include "pokedatastructure.h"
#define RED 1
#define BLACK 0
using namespace std;
PokeDataStructure::PokeDataStructure()
{
root = nullptr;
}
PokeDataStructure::~PokeDataStructure()
{
// walk tree in post-order traversal and delete
post_order_delete(root);
root = nullptr; // not really necessary, since the tree is going
// away, but might want to guard against someone
// using a pointer after deleting
}
void PokeDataStructure::post_order_delete(TreeNode* node)
{
if (node == nullptr) return; // Empty tree
post_order_delete(node->left);
post_order_delete(node->right);
delete node;
}
void PokeDataStructure::print_current_level (TreeNode* node, int level)
{
if (node == nullptr) return;
if (level == 1) {
if (is_leaf(node))
cout << "[" << node->data << ":" << node->poke_ids.size() << "] ";
else {
int num_children = (node->left == nullptr or node->right == nullptr) ? 1 : 2;
cout << "(" << node->data << " : " << num_children << ") ";
}
} else if (level > 1) {
print_current_level(node->left, level-1);
print_current_level(node->right, level-1);
}
}
void PokeDataStructure::print_level_order()
{
int h = height(root);
for (int i = 1; i <= h; i++) {
cout << "LEVEL: " << i << endl;
print_current_level(root, i);
cout << endl;
}
}
bool PokeDataStructure::is_leaf (TreeNode *node) {
if (node == nullptr) return false;
return (node->left == nullptr and node->right == nullptr);
}
int PokeDataStructure::height(TreeNode* node)
{
if (node == nullptr) return 0;
return node->height;
}
int PokeDataStructure::get_balance(TreeNode* node) {
if (node == nullptr) return 0;
return height(node->left) - height(node->right);
}
// https://www.geeksforgeeks.org/insertion-in-an-avl-tree/#
PokeDataStructure::TreeNode* PokeDataStructure::left_rotate(TreeNode *x) {
TreeNode *y = x->right;
TreeNode *T2 = y->left;
y->left = x;
x->right = T2;
x->height = get_max(height(x->left), height(x->right)) + 1;
y->height = get_max(height(y->left), height(y->right)) + 1;
return y;
}
PokeDataStructure::TreeNode* PokeDataStructure::right_rotate(TreeNode *y) {
TreeNode *x = y->left;
TreeNode *T2 = x->right;
x->right = y;
y->left = T2;
x->height = get_max(height(x->left), height(x->right)) + 1;
y->height = get_max(height(y->left), height(y->right)) + 1;
return x;
}
PokeDataStructure::TreeNode* PokeDataStructure::add_node(TreeNode *node,
uint16_t field_signature, uint16_t poke_id)
{
if (node == nullptr) {
// data always stored in the leaves
TreeNode *new_node = new TreeNode;
new_node->left = new_node->right = nullptr;
new_node->height = 1;
new_node->data = field_signature;
new_node->poke_ids.push_back(poke_id);
return new_node;
}
if (is_leaf(node)) {
// make a new node with two leaves
if (node->data == field_signature) {
node->poke_ids.push_back(poke_id);
return node;
}
TreeNode *new_leaf = new TreeNode;
new_leaf->left = new_leaf->right = nullptr;
new_leaf->height = 1;
new_leaf->data = field_signature;
new_leaf->poke_ids.push_back(poke_id);
TreeNode *new_root = new TreeNode;
new_root->left = new_root->right = nullptr;
new_root->height = 2;
if (field_signature < node->data) {
new_root->left = new_leaf;
new_root->right = node;
new_root->data = field_signature;
} else {
new_root->left = node;
new_root->right = new_leaf;
new_root->data = node->data;
}
return new_root;
}
if (field_signature > node->data)
node->right = add_node(node->right, field_signature, poke_id);
else
node->left = add_node(node->left, field_signature, poke_id);
node->height = 1 + get_max(height(node->left), height(node->right));
int balance = get_balance(node);
// 4 cases for rebalancing AVL tree
while (balance > 1 or balance < -1) {
// Left-Left
if (balance > 1 and field_signature < node->left->data)
node = right_rotate(node);
// Right-Right
else if (balance < -1 and field_signature > node->right->data)
node = left_rotate(node);
// Left-Right
else if (balance > 1 and field_signature > node->left->data) {
node->left = left_rotate(node->left);
node = right_rotate(node);
}
// Right-Left
else if (balance < -1 and field_signature < node->right->data) {
node->right = right_rotate(node->right);
node = left_rotate(node);
}
balance = get_balance(node);
field_signature = node->data;
}
// no unbalance; return unchanged node
return node;
}
void PokeDataStructure::add_pokemon(uint16_t poke_id, uint8_t field_id,
uint8_t field_level)
{
uint16_t field_signature = ((uint16_t) field_id << 8) | field_level;
assert(field_id == (field_signature >> 8));
assert(field_level == (uint8_t) field_signature);
if (pokemon_field_moves.find(poke_id) == pokemon_field_moves.end()) {
pokemon_field_moves[poke_id] = field_signature;
root = add_node(root, field_signature, poke_id);
}
}
pair<uint8_t, uint8_t> PokeDataStructure::get_field_move(uint16_t poke_id) {
uint16_t field_signature = pokemon_field_moves[poke_id];
uint8_t field_move = (uint8_t)(field_signature >> 8);
// https://stackoverflow.com/questions/27889213/c-integer-downcast
// downcast truncates most significant bytes
uint8_t field_level = (uint8_t) field_signature;
return pair<uint8_t, uint8_t>(field_move, field_level);
}
vector<uint16_t> PokeDataStructure::get_pokemon_with_geq_field_move(
uint16_t poke_id) {
assert (pokemon_field_moves.find(poke_id) != pokemon_field_moves.end());
uint16_t field_signature = pokemon_field_moves[poke_id];
uint16_t field_sig_max = field_signature | 0xFF;
return range_query(field_signature, field_sig_max);
}
void PokeDataStructure::collect_subtree(TreeNode *node, vector<uint16_t> *vec)
{
if (node == nullptr) return;
if (is_leaf(node)) {
vec->insert(vec->end(), node->poke_ids.begin(), node->poke_ids.end());
return;
}
collect_subtree(node->left, vec);
collect_subtree(node->right, vec);
}
PokeDataStructure::TreeNode* PokeDataStructure::find_vsplit (TreeNode *node,
uint16_t min, uint16_t max)
{
if (node == nullptr) return nullptr;
if (node->data > min and node->data > max)
return find_vsplit(node->left, min, max);
if (node->data < min and node->data < max)
return find_vsplit(node->right, min, max);
return node;
}
// TODO: report in order (not really necessary...)
vector<uint16_t> PokeDataStructure::range_query(uint16_t min, uint16_t max) {
vector<uint16_t> results;
TreeNode* v_split = find_vsplit(root, min, max);
if (v_split == nullptr) return results;
if (is_leaf(v_split)) {
results.insert(results.end(), v_split->poke_ids.begin(),
v_split->poke_ids.end());
return results;
}
// get all right subtrees on path to min
TreeNode *min_path = v_split->left;
while (min_path != nullptr) {
if (is_leaf(min_path)) {// leaf
if (min_path->data >= min and min_path->data <= max) {
results.insert(results.end(), min_path->poke_ids.begin(),
min_path->poke_ids.end());
break;
}
}
if (min_path->data >= min) {
collect_subtree(min_path->right, &results);
min_path = min_path->left;
} else
min_path = min_path->right;
}
// get all left subtrees on path to max
TreeNode *max_path = v_split->right;
while (max_path != nullptr) {
if (is_leaf(max_path)) {// leaf
if (max_path->data >= min and max_path->data <= max) {
results.insert(results.end(), max_path->poke_ids.begin(),
max_path->poke_ids.end());
break;
}
}
if (max_path->data < max) {
collect_subtree(max_path->left, &results);
max_path = max_path->right;
} else
max_path = max_path->left;
}
return results;
}
void PokeDataStructure::self_test () {
for (auto it = pokemon_field_moves.begin(); it != pokemon_field_moves.end(); ++it) {
vector<uint16_t> manually_checked;
uint8_t field_move = (uint8_t) (it->second >> 8);
uint8_t field_level = (uint8_t) it->second;
for (auto it2 = pokemon_field_moves.begin(); it2 != pokemon_field_moves.end(); ++it2) {
uint8_t field_move2 = (uint8_t) (it2->second >> 8);
uint8_t field_level2 = (uint8_t) it2->second;
if (field_move2 == field_move and field_level2 >= field_level)
manually_checked.push_back(it2->first);
}
vector<uint16_t> range_query = get_pokemon_with_geq_field_move(it->first);
sort(manually_checked.begin(), manually_checked.end());
sort(range_query.begin(), range_query.end());
assert(range_query.size() == manually_checked.size());
for (unsigned int i = 0; i < range_query.size(); i ++) {
assert(range_query[i] == manually_checked[i]);
}
}
cout << "self-test passed." << endl;
}

62
pokedatastructure.h Normal file
View File

@@ -0,0 +1,62 @@
#ifndef POKEDATASTRUCTURE_H
#define POKEDATASTRUCTURE_H
#include <iostream>
#include <map>
#include <vector>
#include <cassert>
#include <stdint.h>
#include <algorithm>
#include "util.h"
class PokeDataStructure {
public:
PokeDataStructure();
~PokeDataStructure();
void add_pokemon(uint16_t poke_id, uint8_t field_id, uint8_t field_level);
std::pair<uint8_t, uint8_t> get_field_move(uint16_t poke_id);
std::vector<uint16_t> get_pokemon_with_geq_field_move (uint16_t poke_id);
void print_level_order();
void self_test();
private:
std::map<uint16_t, uint16_t> pokemon_field_moves;
struct TreeNode {
// first is (field_id << 8) | field_level, second is poke_id
// for leaves, field signature
// for non-leaves, max value of left subtree
uint16_t data;
std::vector<uint16_t> poke_ids;
int height;
TreeNode *left;
TreeNode *right;
};
TreeNode *root;
TreeNode* find_vsplit (TreeNode *node,
uint16_t min, uint16_t max);
void collect_subtree (TreeNode *node, std::vector<uint16_t> *vec);
std::vector<uint16_t> range_query (uint16_t min, uint16_t max);
TreeNode* add_node (TreeNode *nod, uint16_t field_signature,
uint16_t poke_id);
bool is_leaf(TreeNode *node);
int height(TreeNode* node);
int get_balance(TreeNode *node);
TreeNode* left_rotate(TreeNode *x);
TreeNode* right_rotate(TreeNode *y);
void print_current_level (TreeNode* node, int level);
void post_order_delete(TreeNode *node);
};
#endif

215
pokeid.cpp Normal file
View File

@@ -0,0 +1,215 @@
#include "pokeid.h"
using namespace std;
// TODO: cleanup
// each pokemon entry is 28 bytes long
#define ENTRY_SIZE 0x1C
#define UNIQUE_SIZE 24
char UNIQUE[] = {
0x07, 0x03, 0x01, 0x07, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x01, 0x02,
0x02, 0x01, 0x01, 0x02, 0x02, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, 0x05
};
vector<string> read_mes_from_rom (const char* path, uint32_t offset) {
vector<string> messages;
ifstream mes_file(path);
mes_file.seekg(offset, ios::beg);
char total_size_bytes[4];
char misc_header_bytes[16];
mes_file.read(&total_size_bytes[0], 4);
uint32_t num_mesgs = read_int(&mes_file);
mes_file.read(&misc_header_bytes[0], 16);
for (uint32_t i = 0; i < num_mesgs; i++) {
uint32_t msg_length = read_int(&mes_file);
char *msg = new char[msg_length];
mes_file.read(&msg[0], msg_length);
string msg_str(&msg[0], msg_length);
messages.push_back(msg_str);
delete [] msg;
}
mes_file.close();
return messages;
}
vector<string> read_mes_file (const char* path) {
vector<string> messages;
ifstream mes_file;
mes_file.open(path);
char total_size_bytes[4];
char num_mesgs_bytes[4];
char misc_header_bytes[16];
mes_file.read(&total_size_bytes[0], 4);
mes_file.read(&num_mesgs_bytes[0], 4);
mes_file.read(&misc_header_bytes[0], 16);
uint32_t num_mesgs = byte_array_to_int(&num_mesgs_bytes[0]);
for (uint32_t i = 0; i < num_mesgs; i++) {
char msg_length_bytes[4];
mes_file.read(&msg_length_bytes[0], 4);
uint32_t msg_length = byte_array_to_int(&msg_length_bytes[0]);
char *msg = new char[msg_length];
mes_file.read(&msg[0], msg_length);
string msg_str(&msg[0], msg_length);
messages.push_back(msg_str);
delete [] msg;
}
mes_file.close();
return messages;
}
void read_pokeID_rom (PokeDataStructure *pds, const char* rom_path, uint32_t offset) {
ifstream bin_file(rom_path);
bin_file.seekg(offset, ios::beg);
// size of file
uint32_t total_size = read_int(&bin_file);
cout << "File size: " << dec << total_size << " bytes" << endl;
// other header stuff
uint32_t unique_size = read_int(&bin_file);
uint32_t data_size = read_int(&bin_file);
char unknown_bytes[4];
bin_file.read(&unknown_bytes[0], 4);
cout << "Unique size: " << hex << unique_size << endl;
cout << "Data size: " << hex << data_size << endl;
assert(unique_size == UNIQUE_SIZE);
char unique[UNIQUE_SIZE];
bin_file.read(&unique[0], UNIQUE_SIZE);
for (int i = 0; i < UNIQUE_SIZE; i ++) {
assert(unique[i] == UNIQUE[i]);
}
uint32_t num_entries = data_size / ENTRY_SIZE;
cout << "Number of entries: " << dec << num_entries << endl;
for (uint32_t i = 0; i < num_entries; i ++) {
char entry_data[ENTRY_SIZE];
bin_file.read(&entry_data[0], ENTRY_SIZE);
uint16_t name_id = byte_array_to_short(&entry_data[0]);
uint8_t field_id = (uint8_t) entry_data[5];
uint8_t field_level = (uint8_t) entry_data[6];
pds->add_pokemon(name_id, field_id, field_level);
// cout << "Name: " << pkmn_names[name_id] << " (ID: " << name_id << ", " << field_moves[field_id] << " " << dec << field_level << ")" << endl;
}
bin_file.close();
}
void read_pokeID_bin (vector<string> pkmn_names, vector<string> field_moves) {
ifstream bin_file;
bin_file.open("PokeID.bin");
// size of file
char total_size_bytes[4];
bin_file.read(&total_size_bytes[0], 4);
uint32_t total_size = byte_array_to_int(&total_size_bytes[0]);
cout << "File size: " << dec << total_size << " bytes" << endl;
// other header stuff
char unique_size_bytes[4];
char data_size_bytes[4];
char unknown_bytes[4];
bin_file.read(&unique_size_bytes[0], 4);
bin_file.read(&data_size_bytes[0], 4);
bin_file.read(&unknown_bytes[0], 4);
uint32_t unique_size = byte_array_to_int(&unique_size_bytes[0]);
uint32_t data_size = byte_array_to_int(&data_size_bytes[0]);
cout << "Unique size: " << hex << unique_size << endl;
cout << "Data size: " << hex << data_size << endl;
assert(unique_size == UNIQUE_SIZE);
char unique[UNIQUE_SIZE];
bin_file.read(&unique[0], UNIQUE_SIZE);
for (int i = 0; i < UNIQUE_SIZE; i ++) {
assert(unique[i] == UNIQUE[i]);
}
uint32_t num_entries = data_size / ENTRY_SIZE;
cout << "Number of entries: " << dec << num_entries << endl;
for (uint32_t i = 0; i < num_entries; i ++) {
char entry_data[ENTRY_SIZE];
bin_file.read(&entry_data[0], ENTRY_SIZE);
uint16_t name_id = byte_array_to_short(&entry_data[0]);
uint32_t field_id = (uint32_t) (unsigned char) entry_data[5];
uint32_t field_level = (uint32_t) (unsigned char) entry_data[6];
cout << "Name: " << pkmn_names[name_id] << " (ID: " << name_id << ", " << field_moves[field_id] << " " << dec << field_level << ")" << endl;
}
bin_file.close();
}
vector<string> get_pokemon_names_rom (const char *path, uint32_t offset) {
return read_mes_from_rom(path, offset);
}
vector<string> get_field_moves_rom (const char *path, uint32_t offset) {
return read_mes_from_rom(path, offset);
}
vector<string> get_pokemon_names () {
return read_mes_file("pokemon_name_us.mes");
}
vector<string> get_field_moves () {
return read_mes_file("fieldwaza_name_us.mes");
}
// int main () {
// vector<string> pkmn_names = get_pokemon_names ();
// vector<string> field_moves = get_field_moves ();
// read_pokeID_bin (pkmn_names, field_moves);
// return 0;
// }

25
pokeid.h Normal file
View File

@@ -0,0 +1,25 @@
#ifndef POKEID_H
#define POKEID_H
#include <iostream>
#include <fstream>
#include <iomanip>
#include <cassert>
#include <vector>
#include <map>
#include <string>
#include <stdint.h>
#include "util.h"
#include "pokedatastructure.h"
std::vector<std::string> read_mes_from_rom (const char* path, uint32_t offset);
std::vector<std::string> get_pokemon_names_rom (const char *path, uint32_t offset);
std::vector<std::string> get_field_moves_rom (const char *path, uint32_t offset);
void read_pokeID_rom (PokeDataStructure *pds, const char* rom_path,
uint32_t offset);
#endif

BIN
pokemon_name_us.mes Normal file
View File

Binary file not shown.

BIN
reference.txt Normal file
View File

Binary file not shown.

BIN
rom.nds Normal file
View File

Binary file not shown.

362
test.cpp Normal file
View File

@@ -0,0 +1,362 @@
#include <iostream>
#include <fstream>
#include <iomanip>
#include <map>
#include <vector>
#include <string.h>
#include <stdint.h>
#include "util.h"
#include "lcs.h"
#include "pokeid.h"
#include "pokedatastructure.h"
using namespace std;
const char* rom_file_path = "rom.nds";
// DFS through subtable, return a map from filepath to FAT offset
void explore_subtable (uint32_t subtable_offset, uint32_t fnt_offset, string path, uint16_t *file_id, map<string,uint16_t> *file_locations) {
ifstream rom_file;
rom_file.open(rom_file_path);
rom_file.seekg(fnt_offset + subtable_offset, ios::beg);
// get first byte
char length_type_byte = 0;
rom_file.read(&length_type_byte, 1);
uint32_t length_type = (uint32_t) (unsigned char)length_type_byte;
while (length_type != 0) {
if (length_type & 0x80) {
// this is a directory
// get name, appended to end of path
char *new_path_bytes = new char[path.length() + length_type - 0x80 + 2];
strcpy(new_path_bytes, path.c_str());
rom_file.read(new_path_bytes + path.length(), length_type - 0x80);
new_path_bytes [path.length() + length_type - 0x80] = '/';
new_path_bytes[path.length() + length_type - 0x80+1] = '\0';
string new_path(new_path_bytes);
// after name, offset for subdirectory from start of fnt
uint32_t subdir_id = (uint32_t) read_short(&rom_file);
int pos_in_subtable = rom_file.tellg();
rom_file.seekg(fnt_offset + (subdir_id & 0xFFF)*8, ios::beg);
uint32_t new_subtable = read_int(&rom_file);
uint16_t first_file_id = read_short(&rom_file);
// now, explore this subtable
explore_subtable (new_subtable, fnt_offset, new_path, &first_file_id, file_locations);
// move filestream back to continue while loop
rom_file.seekg(pos_in_subtable, ios::beg);
delete [] new_path_bytes;
} else {
char *filename_bytes = new char[length_type + 1];
rom_file.read(filename_bytes, length_type);
filename_bytes[length_type] = '\0';
string filename_str(filename_bytes);
string full_path_str(path);
full_path_str += filename_str;
// cout << full_path_str << endl;
file_locations->insert(pair<string,uint16_t>(full_path_str, *file_id));
*file_id += 1;
}
rom_file.read(&length_type_byte, 1);
length_type = (uint32_t) (unsigned char)length_type_byte;
}
rom_file.close();
}
map<string,uint16_t> get_file_locations () {
// https://web.archive.org/web/20110718184246/http://nocash.emubase.de/gbatek.htm#dsmemorymaps
// mapping of filenames (strings) to FAT offsets
map<string,uint16_t> file_locations;
ifstream rom_file;
rom_file.open(rom_file_path);
rom_file.seekg(0x40, ios::beg);
uint32_t fnt_offset = read_int(&rom_file);
rom_file.seekg(0x48, ios::beg);
uint32_t fat_offset = read_int(&rom_file);
cout << "FNT offset: " << hex << fnt_offset << endl;
cout << "FAT offset: " << hex << fat_offset << endl;
// Go to start of fnt
rom_file.seekg(fnt_offset, ios::beg);
uint32_t subtable_offset = read_int(&rom_file);
uint16_t first_file_id = read_short(&rom_file);
cout << "First subtable offset: " << hex << subtable_offset << endl;
cout << "First file id: " << hex << first_file_id << endl;
explore_subtable(subtable_offset, fnt_offset, "", &first_file_id, &file_locations);
rom_file.close();
return file_locations;
}
// https://en.wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Storer%E2%80%93Szymanski
// https://magikos.livejournal.com/7375.html?
// https://github.com/SciresM/FEAT/blob/master/FEAT/DSDecmp/Formats/Nitro/LZ10.cs#L83
char* decompress_LZ10 (char *compressed_bytes, uint32_t compressed_size) {
assert(compressed_size >= 4);
// First byte 0x10 means file is LZ10 compressed
assert(*compressed_bytes == 0x10);
// next three bytes are size of uncompressed file
uint32_t uncompressed_size = (uint8_t) *(compressed_bytes + 3) << 16 |
(uint8_t) *(compressed_bytes + 2) << 8 |
(uint8_t) *(compressed_bytes + 1);
char* uncompressed_file = new char[uncompressed_size];
uint32_t input_pos = 4;
uint32_t output_pos = 0;
while (input_pos < compressed_size) {
uint8_t flag_bytes = (uint8_t) *(compressed_bytes + (input_pos++));
for (int i = 0; i < 8; i ++) {
bool flag_bit = (flag_bytes >> i) & 1;
if (flag_bit) {
// Dictionary entry
uint8_t token1 = (uint8_t) *(compressed_bytes + (input_pos++));
uint8_t token2 = (uint8_t) *(compressed_bytes + (input_pos++));
uint16_t disp = ((token1 & 0xF) << 8) | token2;
uint8_t length = (token1 >> 4) + 3; // plus 3 for some reason?
uint16_t read_start = output_pos - disp - 1;
for (int j = 0; j < length; j++) {
*(uncompressed_file + (output_pos++)) = *(uncompressed_file + read_start + i);
}
} else {
// Raw byte
*(uncompressed_file + (output_pos++)) = *(compressed_bytes + (input_pos++));
}
}
}
return uncompressed_file;
}
uint32_t compress_LZ10 (char *uncompressed_data, uint32_t uncompressed_size, unsigned char **compressed_data) {
vector<uint8_t> buffer;
buffer.push_back(0x10);
buffer.push_back(0);
buffer.push_back(0);
buffer.push_back(0);
// int max_disp = 0xFFFF;
// int max_length = 0xFFF + 3;
uint32_t cur_position = 4;
while (cur_position < uncompressed_size) {
// https://en.wikipedia.org/wiki/Longest_common_substring
// get longest common substring (up to max length of dicitionary entry)
// if passes threshold (length >= 3), make it a dictionary entry
// otherwise just copy raw bytes
buffer.push_back(0); // flag byte
int flag_index = buffer.size() - 1;
uint8_t flag_byte = 0;
// int current_compress_idx = buffer.size() - 1;
for (int i = 0; i < 8; i++) {
// get longest common substring
// int start_idx = get_max(0, cur_position - max_disp - 1);
// int end_idx = get_min(cur_position + max_length, uncompressed_size);
// vector<uint8_t> lcs = longest_common_substring(start_idx, cur_position, cur_position, end_idx, uncompressed_data);
// if (lcs.size() < 3) {
// // store as raw byte
// cur_position ++;
// } else {
// // store as dictionary entry
// flag_byte |= (1 << 8-i);
// }
}
buffer[flag_index] = flag_byte;
}
uint32_t compressed_size = buffer.size();
// load uncompressed size into buffer
buffer[1] = (uint8_t) (uncompressed_size & 0xFF);
buffer[2] = (uint8_t) ((uncompressed_size >> 8) & 0xFF);
buffer[3] = (uint8_t) ((uncompressed_size >> 16) & 0xFF);
*compressed_data = buffer.data();
return compressed_size;
}
void process_map_files (PokeDataStructure* pds, map<string, uint16_t> file_locations, uint32_t fat_offset) {
map<string, uint16_t> map_dat_files;
ifstream rom_file(rom_file_path);
for (auto it = file_locations.begin(); it != file_locations.end(); ++it) {
if (str_ends_with(it->first, ".map.dat.lz")) {
map_dat_files.insert(*it);
// cout << it->first << " " << hex << it->second << endl;
}
uint32_t file_in_fat = fat_offset + it->second*8;
rom_file.seekg(file_in_fat, ios::beg);
uint32_t file_start = read_int(&rom_file);
uint32_t file_end = read_int(&rom_file);
assert(file_start % 512 == 0);
if (file_end % 512 == 0) {
cout << "check if no padding after " << it->first << ": " << hex << file_start << "," << file_end << endl;
}
}
// uint32_t map_file_in_fat = fat_offset + file_locations["data/field/map/m038_022.map.dat.lz"]*8;
// rom_file.seekg(map_file_in_fat, ios::beg);
// uint32_t map_file_start = read_int(&rom_file);
// uint32_t map_file_end = read_int(&rom_file);
// uint32_t file_size = map_file_end - map_file_start;
// char* map_file_bytes = new char[file_size];
// rom_file.seekg(map_file_start, ios::beg);
// rom_file.read(map_file_bytes, file_size);
// char* uncompressed_file = decompress_LZ10(map_file_bytes, file_size);
// delete[] map_file_bytes;
// delete[] uncompressed_file;
// TODO decompress .map.dat.lz files (LZ10 compression)
// https://github.com/SunakazeKun/AlmiaE/blob/master/src/com/aurum/almia/game/Compression.java
// https://ndspy.readthedocs.io/en/latest/_modules/ndspy/lz10.html#decompress
rom_file.close();
}
int main () {
// map<string,uint16_t> file_locations = get_file_locations ();
// ifstream rom_file(rom_file_path);
// rom_file.seekg(0x48, ios::beg);
// uint32_t fat_offset = read_int(&rom_file);
// uint32_t pkmn_name_in_fat = fat_offset + file_locations["data/message/etc/pokemon_name_us.mes"]*8;
// uint32_t field_move_in_fat = fat_offset + file_locations["data/message/etc/fieldwaza_name_us.mes"]*8;
// uint32_t pokeid_bin_in_fat = fat_offset + file_locations["data/param/PokeID.bin"]*8;
// rom_file.seekg(pkmn_name_in_fat, ios::beg);
// uint32_t pkmn_name_offset = read_int(&rom_file);
// rom_file.seekg(field_move_in_fat, ios::beg);
// uint32_t field_move_offset = read_int(&rom_file);
// rom_file.seekg(pokeid_bin_in_fat, ios::beg);
// uint32_t pokeid_bin_offset = read_int(&rom_file);
// cout << "Pokemon name offset: " << pkmn_name_offset << endl;
// cout << "Field name offset: " << field_move_offset << endl;
// cout << "PokeID bin offset: " << pokeid_bin_offset << endl;
// vector<string> pkmn_names = get_pokemon_names_rom (rom_file_path,
// pkmn_name_offset);
// vector<string> field_moves = get_field_moves_rom (rom_file_path,
// field_move_offset);
// // cout << pkmn_names.size() << endl;
// PokeDataStructure pds;
// // TODO: make this a PDS member function
// read_pokeID_rom (&pds, rom_file_path, pokeid_bin_offset);
// // uint16_t poke_id = 411;
// // vector<uint16_t> res = pds.get_pokemon_with_geq_field_move(poke_id);
// // pair<uint8_t, uint8_t> start_field_info = pds.get_field_move(poke_id);
// // cout << "Pokemon that can replace " << pkmn_names[poke_id] << " (" <<
// // field_moves[start_field_info.first] << " " << dec
// // << (uint32_t) start_field_info.second << "):" << endl;
// // for (auto it = res.begin(); it != res.end(); ++it) {
// // pair<uint8_t, uint8_t> field_info = pds.get_field_move(*it);
// // cout << pkmn_names[*it] << " (" << field_moves[field_info.first] << " "
// // << dec << (uint32_t) field_info.second << ")" << endl;
// // }
// // cout << endl;
// // pds.print_level_order();
// // pds.self_test();
// process_map_files(&pds, file_locations, fat_offset);
// rom_file.close();
unsigned char thing[] = "papaya";
int *s = new int[6];
for (int i = 0; i < 6; i++) {
s[i] = (int) (unsigned char) thing[i];
}
int *sa = new int[6];
int *LCP = new int[6];
suffix_array(s, sa, LCP, 6, 255);
for (int i = 0; i < 6; i++)
cout << sa[i] << " ";
cout << endl << "LCP: ";
for (int i = 0; i < 6-1; i++)
cout << LCP[i] << " ";
cout << endl;
delete [] s; delete [] sa; delete [] LCP;
return 0;
}

BIN
test.out Executable file
View File

Binary file not shown.

7669
thing1.txt Normal file
View File

File diff suppressed because it is too large Load Diff

56
util.cpp Normal file
View File

@@ -0,0 +1,56 @@
#include "util.h"
using namespace std;
// little endian
uint32_t byte_array_to_int (char* bytes) {
uint32_t res = 0;
for (int i = 0; i < 4; i ++) {
res |= (uint32_t) *(unsigned char *)(bytes + i) << 8*i;
}
return res;
}
uint16_t byte_array_to_short (char* bytes) {
uint16_t res = 0;
for (int i = 0; i < 2; i ++) {
res |= (uint16_t) *(unsigned char *)(bytes + i) << 8*i;
}
return res;
}
uint32_t read_int (ifstream *file) {
char bytes[4];
file->read(&bytes[0], 4);
return byte_array_to_int(&bytes[0]);
}
uint16_t read_short (ifstream *file) {
char bytes[2];
file->read(&bytes[0], 2);
return byte_array_to_short(&bytes[0]);
}
bool str_ends_with (std::string str, std::string suffix) {
if (str.length() < suffix.length()) return false;
for (unsigned int i = 1; i <= suffix.length(); i++) {
if (suffix[suffix.length() - i] != str[str.length() - i]) return false;
}
return true;
}
int get_max (int a, int b) {
return (a > b)? a : b;
}
int get_min (int a, int b) {
return (a > b)? b : a;
}

19
util.h Normal file
View File

@@ -0,0 +1,19 @@
#ifndef UTIL_H
#define UTIL_H
#include <fstream>
#include <stdint.h>
#include <string>
uint32_t byte_array_to_int (char* bytes);
uint16_t byte_array_to_short (char* bytes);
uint32_t read_int (std::ifstream *file);
uint16_t read_short (std::ifstream *file);
bool str_ends_with (std::string str, std::string suffix);
int get_max (int a, int b);
int get_min (int a, int b);
#endif