From 91cd790c866d8e6d7c4af0bbe3493531f4b5cfcb Mon Sep 17 00:00:00 2001 From: cristian Date: Wed, 1 Jul 2026 23:52:13 -0400 Subject: [PATCH] Add Rust matching tools (ps-status, ps-firstdiff, ps-fdiff) Introduce a cargo workspace under tools/rust/ with a shared ps-core library and three binaries for the byte-for-byte matching effort: - ps-status: status of functions pending decompilation. Build-free mode scans GLOBAL_ASM in src/ (validated: 190 funcs across 68 files, matching grep); build-aware mode reports byte progress per folder from the linker .map, porting the progress.py metric. - ps-firstdiff: first difference(s) between built and expected ROM, with function naming and jal-target resolution (like tools/first_diff.py). - ps-fdiff: non-interactive per-function asm diff of built ROM vs baserom (like ./diff.py -mwo ). ps-core provides parsers for GLOBAL_ASM, symbol_addrs, and GNU ld .map files (with vram/vrom lookups), a .z64 reader with md5 verification, and a rabbitizer wrapper for disassembly. All parsers are unit-tested. Wired into tools/Makefile (cargo build, skipped with a warning if cargo is absent), target/ gitignored, and documented in README.md and tools/rust/README.md. The Python scripts (progress.py, tools/first_diff.py, diff.py) are kept in place: only the build-free ps-status path can be validated without a ROM and toolchain. Parity of the build-dependent tools must be confirmed in a full build environment before removing the Python equivalents. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + README.md | 42 ++ tools/Makefile | 16 +- tools/rust/Cargo.lock | 389 ++++++++++++++++++ tools/rust/Cargo.toml | 22 + tools/rust/README.md | 75 ++++ tools/rust/crates/ps-core/Cargo.toml | 15 + tools/rust/crates/ps-core/src/disasm.rs | 59 +++ tools/rust/crates/ps-core/src/globalasm.rs | 114 +++++ tools/rust/crates/ps-core/src/lib.rs | 27 ++ tools/rust/crates/ps-core/src/mapfile.rs | 300 ++++++++++++++ tools/rust/crates/ps-core/src/rom.rs | 78 ++++ tools/rust/crates/ps-core/src/symbols.rs | 97 +++++ tools/rust/crates/ps-fdiff/Cargo.toml | 15 + tools/rust/crates/ps-fdiff/src/main.rs | 187 +++++++++ tools/rust/crates/ps-firstdiff/Cargo.toml | 15 + tools/rust/crates/ps-firstdiff/src/main.rs | 171 ++++++++ tools/rust/crates/ps-status/Cargo.toml | 17 + tools/rust/crates/ps-status/src/buildaware.rs | 186 +++++++++ tools/rust/crates/ps-status/src/main.rs | 227 ++++++++++ 20 files changed, 2051 insertions(+), 2 deletions(-) create mode 100644 tools/rust/Cargo.lock create mode 100644 tools/rust/Cargo.toml create mode 100644 tools/rust/README.md create mode 100644 tools/rust/crates/ps-core/Cargo.toml create mode 100644 tools/rust/crates/ps-core/src/disasm.rs create mode 100644 tools/rust/crates/ps-core/src/globalasm.rs create mode 100644 tools/rust/crates/ps-core/src/lib.rs create mode 100644 tools/rust/crates/ps-core/src/mapfile.rs create mode 100644 tools/rust/crates/ps-core/src/rom.rs create mode 100644 tools/rust/crates/ps-core/src/symbols.rs create mode 100644 tools/rust/crates/ps-fdiff/Cargo.toml create mode 100644 tools/rust/crates/ps-fdiff/src/main.rs create mode 100644 tools/rust/crates/ps-firstdiff/Cargo.toml create mode 100644 tools/rust/crates/ps-firstdiff/src/main.rs create mode 100644 tools/rust/crates/ps-status/Cargo.toml create mode 100644 tools/rust/crates/ps-status/src/buildaware.rs create mode 100644 tools/rust/crates/ps-status/src/main.rs diff --git a/.gitignore b/.gitignore index 1179299..dd6a239 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ ctx.c.m2c tools/vtxdis tools/permuter tools/n64 +tools/rust/target/ .DS_Store diff --git a/README.md b/README.md index 155eea8..a35f7e8 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,46 @@ python3 -m pip install -r requirements.txt 2. Set up tools and extract the rom: `make init` 3. Re-assemble the rom: `make` +# Matching tools (Rust) + +A Rust workspace under [`tools/rust/`](tools/rust/) provides the tooling for the +byte-for-byte matching effort. It is built automatically by `make setup` +(`make -C tools`) when `cargo` is available; you can also build it directly: + +```bash +cargo build --release --manifest-path tools/rust/Cargo.toml +``` + +The binaries land in `tools/rust/target/release/`: + +* **`ps-status`** — status of functions still pending decompilation. Runs + **build-free** (no ROM needed), scanning `#pragma GLOBAL_ASM` in `src/`: + + ```bash + ./tools/rust/target/release/ps-status # summary + per-file counts + ./tools/rust/target/release/ps-status --list # every function + address + ./tools/rust/target/release/ps-status --json # machine-readable + ``` + + After `make` it also runs **build-aware** (byte progress per folder, like + `progress.py`) when `build/pokestadium-us.map` exists, or with `--build-aware`. + +* **`ps-firstdiff`** — first difference(s) between the built and expected ROM + (like `tools/first_diff.py`). Requires `make` + `make diff-init`: + + ```bash + ./tools/rust/target/release/ps-firstdiff -c 5 + ``` + +* **`ps-fdiff`** — non-interactive per-function asm diff of the built ROM vs the + baserom (like `./diff.py -mwo `). Requires `make`: + + ```bash + ./tools/rust/target/release/ps-fdiff func_80030010 + ``` + +Run any tool with `--help` for options. The equivalent Python scripts +(`progress.py`, `tools/first_diff.py`, `diff.py`) are kept alongside for now; see +[`tools/rust/README.md`](tools/rust/README.md) for parity-validation status. + For contacts and other pret projects, see [pret.github.io](https://pret.github.io/). diff --git a/tools/Makefile b/tools/Makefile index 5f9e242..8c69be2 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -13,10 +13,22 @@ default: all n64crc_SOURCES := vtxdis.c -all: $(PROGRAMS) +all: $(PROGRAMS) rust + +# Build the Rust matching tools (ps-status, ps-firstdiff, ps-fdiff). Skipped +# with a warning when cargo is unavailable so the C-only workflow still works. +rust: + @if command -v cargo >/dev/null 2>&1; then \ + echo "Building Rust matching tools..."; \ + cargo build --release --manifest-path rust/Cargo.toml; \ + else \ + echo "WARNING: cargo not found; skipping Rust tools (ps-status, ps-firstdiff, ps-fdiff)."; \ + echo " Install Rust from https://rustup.rs to build them."; \ + fi clean: $(V)rm -Rf $(PROGRAMS) $(BUILD_DIR) + $(V)command -v cargo >/dev/null 2>&1 && cargo clean --manifest-path rust/Cargo.toml || true distclean: clean @@ -28,4 +40,4 @@ $(BUILD_DIR)/%.o: %.c $(BUILD_DIR)/%.o: %.cpp $(CXX) -c $< -o $@ $(CXXFLAGS) -.PHONY: all clean distclean default +.PHONY: all clean distclean default rust diff --git a/tools/rust/Cargo.lock b/tools/rust/Cargo.lock new file mode 100644 index 0000000..e4fb54c --- /dev/null +++ b/tools/rust/Cargo.lock @@ -0,0 +1,389 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "ps-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "libc", + "md-5", + "rabbitizer", + "regex", + "serde", +] + +[[package]] +name = "ps-fdiff" +version = "0.1.0" +dependencies = [ + "anyhow", + "ps-core", +] + +[[package]] +name = "ps-firstdiff" +version = "0.1.0" +dependencies = [ + "anyhow", + "ps-core", +] + +[[package]] +name = "ps-status" +version = "0.1.0" +dependencies = [ + "anyhow", + "ps-core", + "serde", + "serde_json", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rabbitizer" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db257a0b8e4a9620d33e34a3635ee9af7643793b5febc7c47baa137952557f9" +dependencies = [ + "cc", + "glob", + "num_enum", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "winnow" +version = "0.5.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" +dependencies = [ + "memchr", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/tools/rust/Cargo.toml b/tools/rust/Cargo.toml new file mode 100644 index 0000000..7d6bcd0 --- /dev/null +++ b/tools/rust/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +resolver = "2" +members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "MIT" +rust-version = "1.74" + +[workspace.dependencies] +ps-core = { path = "crates/ps-core" } +anyhow = "1" +regex = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +md-5 = "0.10" +rabbitizer = "1" +libc = "0.2" + +[profile.release] +opt-level = 3 diff --git a/tools/rust/README.md b/tools/rust/README.md new file mode 100644 index 0000000..acc1aea --- /dev/null +++ b/tools/rust/README.md @@ -0,0 +1,75 @@ +# Pokémon Stadium matching tools (Rust) + +A cargo workspace with the tooling for the byte-for-byte matching effort. It has +no Python dependencies; MIPS disassembly uses the [`rabbitizer`] crate — the same +disassembler the Python scripts use — for output parity. + +## Crates + +| Crate | Binary | Replaces (eventually) | +| -------------- | -------------- | ------------------------- | +| `ps-core` | *(library)* | shared parsers / models | +| `ps-status` | `ps-status` | `progress.py` | +| `ps-firstdiff` | `ps-firstdiff` | `tools/first_diff.py` | +| `ps-fdiff` | `ps-fdiff` | `./diff.py -mwo ` | + +`ps-core` contains the reusable pieces: + +* `globalasm` — scan `src/**/*.c` for `#pragma GLOBAL_ASM(...)` (build-free list + of pending functions). +* `symbols` — parse `linker_scripts//symbol_addrs*.txt`. +* `mapfile` — minimal GNU `ld` `.map` parser: `.text` symbols with sizes, plus + vram/vrom lookups (vrom derived from each section's `load address`). +* `rom` — big-endian `.z64` reader + md5 verification. +* `disasm` — `rabbitizer` wrapper; resolves `jal` targets to symbol names. + +## Build & test + +```bash +cargo build --release # binaries in target/release/ +cargo test # unit tests +cargo clippy --all-targets +``` + +## Modes that need build artifacts + +`ps-status` (default), works with only the committed repo. Everything else needs +artifacts produced by `make`: + +* `ps-status --build-aware` needs `build/pokestadium-.map`. +* `ps-firstdiff` needs `build/…z64` + `.map` and `expected/build/…` (`make diff-init`). +* `ps-fdiff` needs `build/…z64` + `.map` and `baseroms//baserom.z64`. + +## Parity-validation status + +The Python scripts are **kept in place**. Only the parts validatable without a +ROM/toolchain have been checked so far: + +* `ps-status` build-free — **validated**: total (190) and per-file counts match + `grep -rho 'GLOBAL_ASM("[^"]*")' src/ | wc -l` and `grep -rln GLOBAL_ASM src/`. + +The build-dependent tools (`ps-status --build-aware`, `ps-firstdiff`, `ps-fdiff`) +are covered by unit tests on synthetic fixtures but have **not** yet been +compared against the Python tools on real build output. Before removing +`progress.py`, `tools/first_diff.py` or `diff.py`, confirm parity in a full build +environment: + +```bash +# build-aware vs progress.py (compare bytes / percentages) +make && make diff-init +python3 progress.py +./tools/rust/target/release/ps-status --build-aware + +# first-diff vs first_diff.py (introduce a deliberate regression first) +python3 tools/first_diff.py +./tools/rust/target/release/ps-firstdiff + +# per-function diff vs diff.py +./diff.py -mwo +./tools/rust/target/release/ps-fdiff +``` + +Note: `ps-fdiff` v1 is non-interactive (word-by-word alignment); `diff.py`'s +interactive TUI, scoring and insert/delete alignment are not reimplemented. + +[`rabbitizer`]: https://crates.io/crates/rabbitizer diff --git a/tools/rust/crates/ps-core/Cargo.toml b/tools/rust/crates/ps-core/Cargo.toml new file mode 100644 index 0000000..d432b23 --- /dev/null +++ b/tools/rust/crates/ps-core/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ps-core" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Shared parsers and domain models for the Pokémon Stadium matching tools" + +[dependencies] +anyhow.workspace = true +regex.workspace = true +serde.workspace = true +md-5.workspace = true +rabbitizer.workspace = true +libc.workspace = true diff --git a/tools/rust/crates/ps-core/src/disasm.rs b/tools/rust/crates/ps-core/src/disasm.rs new file mode 100644 index 0000000..459ee39 --- /dev/null +++ b/tools/rust/crates/ps-core/src/disasm.rs @@ -0,0 +1,59 @@ +//! Thin wrapper over the `rabbitizer` MIPS disassembler. +//! +//! Mirrors the behaviour of `tools/first_diff.py::decodeInstruction`: decode a +//! 32-bit word and, when it is a function call (`jal`) with an embedded address, +//! resolve that address against the map file so the disassembly shows the symbol +//! name instead of a raw value. + +use rabbitizer::{InstrCategory, Instruction}; + +use crate::mapfile::MapFile; + +/// Disassemble one instruction word located at `vram`. +/// +/// When the instruction is a `jal`/jump-with-address, its target is looked up in +/// `map` and substituted as an immediate override. +pub fn disassemble_word(word: u32, vram: u32, map: &MapFile) -> String { + let instr = Instruction::new(word, vram, InstrCategory::CPU); + + let mut imm_override: Option = None; + if instr.is_jump_with_address() { + let target = instr.instr_index_as_vram() as u64; + if let Some(sym) = map.symbol_by_vram_or_vrom(target) { + imm_override = Some(sym.name.clone()); + } + } + + instr.disassemble(imm_override.as_deref(), -20) +} + +/// Disassemble a word without map-based symbol resolution. +pub fn disassemble_word_plain(word: u32, vram: u32) -> String { + let instr = Instruction::new(word, vram, InstrCategory::CPU); + instr.disassemble(None, 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_nop() { + // 0x00000000 is `nop`. + let s = disassemble_word_plain(0x00000000, 0x80000000); + assert!(s.contains("nop"), "got: {s}"); + } + + #[test] + fn decodes_jal_and_resolves_symbol() { + // jal to 0x80000530 == 0x0C000000 | (0x80000530 >> 2 & 0x3FFFFFF) + let target = 0x80000530u32; + let word = 0x0C000000 | ((target >> 2) & 0x03FF_FFFF); + let mf = crate::mapfile::parse_str( + "\n .text 0x0000000080000530 0x40 build/src/us/main.o\n 0x0000000080000530 Main\n", + ); + let s = disassemble_word(word, 0x80001000, &mf); + assert!(s.contains("jal"), "got: {s}"); + assert!(s.contains("Main"), "expected symbol name, got: {s}"); + } +} diff --git a/tools/rust/crates/ps-core/src/globalasm.rs b/tools/rust/crates/ps-core/src/globalasm.rs new file mode 100644 index 0000000..3eb1333 --- /dev/null +++ b/tools/rust/crates/ps-core/src/globalasm.rs @@ -0,0 +1,114 @@ +//! Parser for `#pragma GLOBAL_ASM(...)` directives in `src/**/*.c`. +//! +//! Each directive references a non-matching assembly file of the form +//! `asm//nonmatchings//.s`, which is the build-free +//! source of truth for the functions still pending decompilation. + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use regex::Regex; + +/// A single `GLOBAL_ASM` reference: a function still included as raw assembly. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GlobalAsmEntry { + /// The C source file that contains the directive. + pub c_file: PathBuf, + /// The `` component of the asm path (e.g. `30640`, `2D340`). + pub file_stem: String, + /// The function name (asm file name without extension). + pub func: String, + /// The raw asm path as written in the directive. + pub asm_path: String, +} + +/// Scan a source root recursively for every `GLOBAL_ASM` directive. +/// +/// Results are sorted by `(c_file, func)` for deterministic output. +pub fn scan(src_root: &Path) -> Result> { + let re = Regex::new(r#"GLOBAL_ASM\("([^"]+)"\)"#).expect("valid regex"); + let mut c_files = Vec::new(); + collect_c_files(src_root, &mut c_files) + .with_context(|| format!("scanning {} for .c files", src_root.display()))?; + c_files.sort(); + + let mut out = Vec::new(); + for c_file in c_files { + let text = + fs::read_to_string(&c_file).with_context(|| format!("reading {}", c_file.display()))?; + for line in text.lines() { + // Skip lines that are commented out (leading `//`). + let trimmed = line.trim_start(); + if trimmed.starts_with("//") { + continue; + } + if let Some(caps) = re.captures(line) { + let asm_path = caps[1].to_string(); + if let Some((file_stem, func)) = split_asm_path(&asm_path) { + out.push(GlobalAsmEntry { + c_file: c_file.clone(), + file_stem, + func, + asm_path, + }); + } + } + } + } + out.sort_by_key(|a| (a.c_file.clone(), a.func.clone())); + Ok(out) +} + +/// Derive `(file_stem, func)` from an asm path. +/// +/// Handles both `asm/us/nonmatchings//.s` and any path that ends in +/// `...//.s` by taking the last two path components. +fn split_asm_path(asm_path: &str) -> Option<(String, String)> { + let path = Path::new(asm_path); + let func = path.file_stem()?.to_str()?.to_string(); + let stem = path.parent()?.file_name()?.to_str()?.to_string(); + Some((stem, func)) +} + +fn collect_c_files(dir: &Path, out: &mut Vec) -> Result<()> { + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + collect_c_files(&path, out)?; + } else if path.extension().and_then(|e| e.to_str()) == Some("c") { + out.push(path); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn splits_standard_path() { + let (stem, func) = split_asm_path("asm/us/nonmatchings/30640/func_80030010.s").unwrap(); + assert_eq!(stem, "30640"); + assert_eq!(func, "func_80030010"); + } + + #[test] + fn scans_directory() -> Result<()> { + let dir = std::env::temp_dir().join(format!("psga-test-{}", std::process::id())); + let sub = dir.join("sub"); + fs::create_dir_all(&sub)?; + fs::write( + sub.join("a.c"), + "int x;\n#pragma GLOBAL_ASM(\"asm/us/nonmatchings/AAAA/func_1.s\")\n// #pragma GLOBAL_ASM(\"asm/us/nonmatchings/AAAA/commented.s\")\n", + )?; + let entries = scan(&dir)?; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].file_stem, "AAAA"); + assert_eq!(entries[0].func, "func_1"); + fs::remove_dir_all(&dir)?; + Ok(()) + } +} diff --git a/tools/rust/crates/ps-core/src/lib.rs b/tools/rust/crates/ps-core/src/lib.rs new file mode 100644 index 0000000..6e12352 --- /dev/null +++ b/tools/rust/crates/ps-core/src/lib.rs @@ -0,0 +1,27 @@ +//! Shared parsers and domain models for the Pokémon Stadium (US) matching tools. +//! +//! This crate is intentionally build-free at its core: the [`globalasm`] and +//! [`symbols`] parsers only need files that are committed to the repo (`src/` +//! and `linker_scripts/`). The `mapfile`, `rom` and `disasm` modules (added in +//! later phases) support the build-aware workflows (progress by bytes, +//! first-diff and per-function diff), which require `make` artifacts. + +pub mod disasm; +pub mod globalasm; +pub mod mapfile; +pub mod rom; +pub mod symbols; + +/// Default game version handled by these tools. +pub const DEFAULT_VERSION: &str = "us"; + +/// Restore the default `SIGPIPE` behaviour so these CLIs terminate silently when +/// their output is closed early (e.g. piped into `head`), like standard Unix +/// tools, instead of panicking on a broken pipe. Call once at the start of +/// `main`. +pub fn reset_sigpipe() { + // Safety: setting a signal disposition to the default handler is sound. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } +} diff --git a/tools/rust/crates/ps-core/src/mapfile.rs b/tools/rust/crates/ps-core/src/mapfile.rs new file mode 100644 index 0000000..859ad54 --- /dev/null +++ b/tools/rust/crates/ps-core/src/mapfile.rs @@ -0,0 +1,300 @@ +//! Minimal parser for GNU `ld` map files (`build/pokestadium-.map`). +//! +//! This reimplements only the parts of the Python `mapfile_parser` that the +//! matching tools use: `.text` symbols with their sizes, grouped by object +//! file, plus a vram → containing-symbol lookup used to resolve `jal` targets. +//! +//! Symbol sizes are derived the usual way: within a single section +//! contribution, each symbol's size is the distance to the next symbol, and the +//! last symbol runs to the end of the contribution. + +use std::fs; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +/// A function/symbol read from the map, with a computed size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MapSymbol { + pub name: String, + pub vram: u64, + pub size: u64, + /// ROM offset (LMA-derived), when the section had a `load address`. + pub vrom: Option, + /// Output section type, e.g. `.text`, `.data`, `.rodata`. + pub section_type: String, + /// Object file this symbol was contributed by. + pub filepath: PathBuf, +} + +/// A parsed map file: a flat, address-sorted list of symbols. +#[derive(Debug, Clone, Default)] +pub struct MapFile { + pub symbols: Vec, +} + +/// One raw section contribution line: ` .text 0xVRAM 0xSIZE path.o`. +struct Contribution { + section_type: String, + vram: u64, + size: u64, + filepath: PathBuf, + /// Enclosing output section's vram and load address (LMA), when known. + sec_vram: Option, + sec_lma: Option, + // (vram, name) for each symbol under this contribution, in file order. + syms: Vec<(u64, String)>, +} + +fn section_type_of(section: &str) -> String { + // ".text.func" -> ".text"; ".text" -> ".text". + let mut parts = section.trim_start_matches('.').splitn(2, '.'); + match parts.next() { + Some(first) if !first.is_empty() => format!(".{first}"), + _ => section.to_string(), + } +} + +fn parse_hex(tok: &str) -> Option { + let t = tok.strip_prefix("0x").or_else(|| tok.strip_prefix("0X"))?; + u64::from_str_radix(t, 16).ok() +} + +/// Parse the textual contents of a GNU ld map file. +pub fn parse_str(text: &str) -> MapFile { + let mut contributions: Vec = Vec::new(); + // Current output section's vram/lma, taken from a `load address` line. Used + // to derive each symbol's vrom (ROM offset). + let mut sec_vram: Option = None; + let mut sec_lma: Option = None; + + for raw in text.lines() { + if raw.is_empty() { + continue; + } + let line = raw.trim_start(); + if line.starts_with("*fill*") || line.starts_with('*') { + continue; + } + let tokens: Vec<&str> = line.split_whitespace().collect(); + + // Output-section header carrying a load address, in either the inline + // form `.text 0xVRAM 0xSIZE load address 0xLMA` or the wrapped form + // `0xVRAM 0xSIZE load address 0xLMA`. Resets the current section. + if let Some(pos) = tokens.iter().position(|t| *t == "address") { + if pos > 0 && tokens.get(pos - 1) == Some(&"load") { + let first_hex = tokens.iter().find_map(|t| parse_hex(t)); + let lma = tokens.get(pos + 1).and_then(|t| parse_hex(t)); + sec_vram = first_hex; + sec_lma = lma; + continue; + } + } + + // A column-0 output-section header without a load address (e.g. `.bss`) + // resets the section context so stale LMAs don't leak across sections. + if !raw.starts_with(' ') { + if tokens.first().map(|t| t.starts_with('.')).unwrap_or(false) { + sec_vram = None; + sec_lma = None; + } + continue; + } + + // Contribution: `
0xVRAM 0xSIZE ` where
starts + // with '.', the two numbers are hex, and the last token looks like a file. + if tokens.len() >= 4 && tokens[0].starts_with('.') { + if let (Some(vram), Some(size)) = (parse_hex(tokens[1]), parse_hex(tokens[2])) { + let filepath = tokens[tokens.len() - 1]; + if looks_like_object(filepath) { + contributions.push(Contribution { + section_type: section_type_of(tokens[0]), + vram, + size, + filepath: PathBuf::from(filepath), + sec_vram, + sec_lma, + syms: Vec::new(), + }); + continue; + } + } + } + + // Symbol: `0xVRAM ` (exactly two tokens, first is hex). + if tokens.len() == 2 { + if let Some(vram) = parse_hex(tokens[0]) { + let name = tokens[1]; + if is_symbol_name(name) { + if let Some(cur) = contributions.last_mut() { + cur.syms.push((vram, name.to_string())); + } + } + } + } + } + + MapFile { + symbols: finalize(contributions), + } +} + +fn looks_like_object(path: &str) -> bool { + path.ends_with(".o") || path.contains(".a(") || path.ends_with(".a") +} + +fn is_symbol_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' || c == '$' => {} + _ => return false, + } + name.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '$' || c == '.') +} + +/// Turn contributions into sized symbols. +fn finalize(contributions: Vec) -> Vec { + let mut out = Vec::new(); + for c in contributions { + let end = c.vram.saturating_add(c.size); + for (i, (vram, name)) in c.syms.iter().enumerate() { + let next = c.syms.get(i + 1).map(|(v, _)| *v).unwrap_or(end); + let size = next.saturating_sub(*vram); + // vrom = section_lma + (vram - section_vram), when the section had a + // load address. Only meaningful for allocated-in-ROM sections. + let vrom = match (c.sec_vram, c.sec_lma) { + (Some(sv), Some(lma)) if *vram >= sv => Some(lma + (*vram - sv)), + _ => None, + }; + out.push(MapSymbol { + name: name.clone(), + vram: *vram, + size, + vrom, + section_type: c.section_type.clone(), + filepath: c.filepath.clone(), + }); + } + } + out.sort_by_key(|s| s.vram); + out +} + +impl MapFile { + /// Read and parse a map file from disk. + pub fn read(path: &Path) -> Result { + let text = + fs::read_to_string(path).with_context(|| format!("reading map {}", path.display()))?; + Ok(parse_str(&text)) + } + + /// Symbols in the `.text` section only. + pub fn text_symbols(&self) -> impl Iterator { + self.symbols.iter().filter(|s| s.section_type == ".text") + } + + /// Find the closest symbol at or before `addr` (the one whose range most + /// likely contains it). Symbols are sorted by vram. + pub fn symbol_containing_vram(&self, addr: u64) -> Option<&MapSymbol> { + let idx = match self.symbols.binary_search_by_key(&addr, |s| s.vram) { + Ok(i) => i, + Err(0) => return None, + Err(i) => i - 1, + }; + Some(&self.symbols[idx]) + } + + /// Find a symbol by exact vram start. + pub fn symbol_at_vram(&self, addr: u64) -> Option<&MapSymbol> { + self.symbols.iter().find(|s| s.vram == addr) + } + + /// Find a symbol by name. + pub fn symbol_by_name(&self, name: &str) -> Option<&MapSymbol> { + self.symbols.iter().find(|s| s.name == name) + } + + /// Find the `.text` symbol whose `[vrom, vrom+size)` range contains a ROM + /// byte offset. Used by first-diff to name the function at a diffing offset. + pub fn symbol_containing_vrom(&self, offset: u64) -> Option<&MapSymbol> { + self.symbols + .iter() + .filter(|s| s.section_type == ".text") + .filter_map(|s| s.vrom.map(|v| (v, s))) + .filter(|(v, s)| offset >= *v && offset < v + s.size.max(1)) + .max_by_key(|(v, _)| *v) + .map(|(_, s)| s) + } + + /// Resolve a `jal`/jump target the way `first_diff.py` does: match a symbol + /// by vram first, then by vrom (the Python constructs instructions with a + /// zero vram, so targets land in the low/vrom range). + pub fn symbol_by_vram_or_vrom(&self, addr: u64) -> Option<&MapSymbol> { + self.symbol_at_vram(addr) + .or_else(|| self.symbols.iter().find(|s| s.vrom == Some(addr))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE: &str = "\ +Linker script and memory map + +.text 0x0000000080000000 0x120 load address 0x0000000000001000 + .text 0x0000000080000000 0x80 build/src/us/main.o + 0x0000000080000000 Main + 0x0000000080000040 func_80000040 + .text 0x0000000080000080 0xa0 build/asm/us/entry.o + 0x0000000080000080 entrypoint +.data 0x0000000080000120 0x10 load address 0x0000000000001120 + .data 0x0000000080000120 0x10 build/src/us/main.o + 0x0000000080000120 some_data +"; + + #[test] + fn parses_sizes_and_sections() { + let mf = parse_str(SAMPLE); + let text: Vec<_> = mf.text_symbols().collect(); + assert_eq!(text.len(), 3); + + let main = mf.symbol_at_vram(0x80000000).unwrap(); + assert_eq!(main.name, "Main"); + assert_eq!(main.size, 0x40); // next symbol at +0x40 + + let f = mf.symbol_at_vram(0x80000040).unwrap(); + assert_eq!(f.size, 0x40); // runs to end of the 0x80-byte contribution + + let entry = mf.symbol_at_vram(0x80000080).unwrap(); + assert_eq!(entry.size, 0xa0); // whole contribution + + // .data symbol is excluded from text_symbols + assert!(mf.symbol_at_vram(0x80000120).unwrap().section_type == ".data"); + } + + #[test] + fn containing_vram_lookup() { + let mf = parse_str(SAMPLE); + assert_eq!(mf.symbol_containing_vram(0x80000010).unwrap().name, "Main"); + assert_eq!( + mf.symbol_containing_vram(0x80000044).unwrap().name, + "func_80000040" + ); + } + + #[test] + fn vrom_from_load_address() { + let mf = parse_str(SAMPLE); + // vram 0x80000000 -> lma 0x1000; +0x40 for func_80000040. + assert_eq!(mf.symbol_at_vram(0x80000000).unwrap().vrom, Some(0x1000)); + assert_eq!(mf.symbol_at_vram(0x80000040).unwrap().vrom, Some(0x1040)); + assert_eq!(mf.symbol_at_vram(0x80000080).unwrap().vrom, Some(0x1080)); + // ROM offset 0x1044 lands inside func_80000040 (0x1040..0x1080). + assert_eq!( + mf.symbol_containing_vrom(0x1044).unwrap().name, + "func_80000040" + ); + } +} diff --git a/tools/rust/crates/ps-core/src/rom.rs b/tools/rust/crates/ps-core/src/rom.rs new file mode 100644 index 0000000..45a08c0 --- /dev/null +++ b/tools/rust/crates/ps-core/src/rom.rs @@ -0,0 +1,78 @@ +//! Big-endian `.z64` ROM reader with optional md5 verification. + +use std::fs; +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use md5::{Digest, Md5}; + +/// A ROM image loaded into memory. +pub struct Rom { + pub bytes: Vec, +} + +impl Rom { + pub fn read(path: &Path) -> Result { + let bytes = fs::read(path).with_context(|| format!("reading ROM {}", path.display()))?; + Ok(Rom { bytes }) + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Lowercase hex md5 of the whole ROM. + pub fn md5_hex(&self) -> String { + let mut hasher = Md5::new(); + hasher.update(&self.bytes); + let digest = hasher.finalize(); + digest.iter().map(|b| format!("{b:02x}")).collect() + } + + /// Read a big-endian 32-bit word at a byte offset. + pub fn word_be(&self, offset: usize) -> Option { + let b = self.bytes.get(offset..offset + 4)?; + Some(u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } +} + +/// Verify a ROM against an `md5sum`-style checksum file (` `). +/// +/// Returns `Ok(())` when the ROM's md5 matches any hash listed in the file. +pub fn verify_against_md5_file(rom: &Rom, checksum_path: &Path) -> Result<()> { + let text = fs::read_to_string(checksum_path) + .with_context(|| format!("reading {}", checksum_path.display()))?; + let actual = rom.md5_hex(); + for line in text.lines() { + if let Some(expected) = line.split_whitespace().next() { + if expected.eq_ignore_ascii_case(&actual) { + return Ok(()); + } + } + } + bail!( + "md5 mismatch: ROM is {actual}, not listed in {}", + checksum_path.display() + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn md5_and_word() { + let rom = Rom { + bytes: vec![0x80, 0x00, 0x05, 0x30, 0xde, 0xad, 0xbe, 0xef], + }; + assert_eq!(rom.word_be(0), Some(0x80000530)); + assert_eq!(rom.word_be(4), Some(0xdeadbeef)); + assert_eq!(rom.word_be(6), None); + // md5 of these 8 bytes + assert_eq!(rom.md5_hex().len(), 32); + } +} diff --git a/tools/rust/crates/ps-core/src/symbols.rs b/tools/rust/crates/ps-core/src/symbols.rs new file mode 100644 index 0000000..32f0cc0 --- /dev/null +++ b/tools/rust/crates/ps-core/src/symbols.rs @@ -0,0 +1,97 @@ +//! Parser for `linker_scripts//symbol_addrs*.txt`. +//! +//! Lines look like `name = 0xADDR;` with an optional trailing comment carrying +//! space-separated `key:value` attributes, e.g. `// type:func size:0x40`. + +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use anyhow::{Context, Result}; +use regex::Regex; + +/// A single symbol from a `symbol_addrs` file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Symbol { + pub name: String, + pub addr: u64, + /// Attributes parsed from the trailing `// key:value ...` comment. + pub attrs: HashMap, +} + +impl Symbol { + /// Returns true when the symbol's `type` attribute is `func`. + pub fn is_func(&self) -> bool { + self.attrs.get("type").map(|s| s == "func").unwrap_or(false) + } +} + +/// Parse a single `symbol_addrs` file into a name → [`Symbol`] map. +pub fn parse_file(path: &Path) -> Result> { + let text = fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?; + Ok(parse_str(&text)) +} + +/// Parse several `symbol_addrs` files, merging them into one map. +/// +/// Later files override earlier ones on name collision. Missing files are +/// skipped silently so callers can pass an optimistic list. +pub fn parse_files(paths: &[&Path]) -> Result> { + let mut out = HashMap::new(); + for path in paths { + if !path.exists() { + continue; + } + out.extend(parse_file(path)?); + } + Ok(out) +} + +/// Parse the textual contents of a `symbol_addrs` file. +pub fn parse_str(text: &str) -> HashMap { + // name = 0xADDR; [// comment] + let re = Regex::new(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*0x([0-9A-Fa-f]+)\s*;(.*)$") + .expect("valid regex"); + let mut out = HashMap::new(); + for line in text.lines() { + if let Some(caps) = re.captures(line) { + let name = caps[1].to_string(); + let addr = u64::from_str_radix(&caps[2], 16).unwrap_or(0); + let attrs = parse_attrs(&caps[3]); + out.insert(name.clone(), Symbol { name, addr, attrs }); + } + } + out +} + +fn parse_attrs(rest: &str) -> HashMap { + let mut attrs = HashMap::new(); + if let Some(idx) = rest.find("//") { + for tok in rest[idx + 2..].split_whitespace() { + if let Some((k, v)) = tok.split_once(':') { + attrs.insert(k.to_string(), v.to_string()); + } + } + } + attrs +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_plain_and_commented() { + let text = "\ +Main = 0x80000530; // type:func +LEOcommand_que = 0x80100638; // allow_duplicated:True +mseq_tbl = 0x80101090; +"; + let map = parse_str(text); + assert_eq!(map.len(), 3); + assert_eq!(map["Main"].addr, 0x80000530); + assert!(map["Main"].is_func()); + assert!(!map["mseq_tbl"].is_func()); + assert_eq!(map["LEOcommand_que"].attrs["allow_duplicated"], "True"); + } +} diff --git a/tools/rust/crates/ps-fdiff/Cargo.toml b/tools/rust/crates/ps-fdiff/Cargo.toml new file mode 100644 index 0000000..3581cf0 --- /dev/null +++ b/tools/rust/crates/ps-fdiff/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ps-fdiff" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Non-interactive per-function assembly diff (built ROM vs baserom)" + +[[bin]] +name = "ps-fdiff" +path = "src/main.rs" + +[dependencies] +ps-core.workspace = true +anyhow.workspace = true diff --git a/tools/rust/crates/ps-fdiff/src/main.rs b/tools/rust/crates/ps-fdiff/src/main.rs new file mode 100644 index 0000000..122fb1b --- /dev/null +++ b/tools/rust/crates/ps-fdiff/src/main.rs @@ -0,0 +1,187 @@ +//! `ps-fdiff` — non-interactive per-function assembly diff. +//! +//! Rust equivalent of the non-interactive `./diff.py -mwo ` flow: locate a +//! function in the built `.map`, read its bytes from both the built ROM (`myimg`) +//! and the baserom (`baseimg`), disassemble each, and show a side-by-side diff +//! with differing lines highlighted. +//! +//! Scope (v1): word-by-word alignment (correct when both sides have the same +//! length, the common case while iterating toward a match). The interactive TUI, +//! scoring and insertion/deletion alignment of `diff.py` are out of scope. + +use std::io::IsTerminal; +use std::path::PathBuf; +use std::process::ExitCode; + +use anyhow::{bail, Context, Result}; +use ps_core::disasm; +use ps_core::mapfile::MapFile; +use ps_core::rom::Rom; +use ps_core::DEFAULT_VERSION; + +const RED: &str = "\x1b[31m"; +const GREEN: &str = "\x1b[32m"; +const RESET: &str = "\x1b[0m"; + +enum ColorMode { + Auto, + Always, + Never, +} + +struct Args { + func: String, + version: String, + root: PathBuf, + color: ColorMode, +} + +fn parse_args() -> Result { + let mut func: Option = None; + let mut version = DEFAULT_VERSION.to_string(); + let mut root = PathBuf::from("."); + let mut color = ColorMode::Auto; + + let mut it = std::env::args().skip(1); + while let Some(arg) = it.next() { + match arg.as_str() { + "-v" | "--version" => { + version = it + .next() + .ok_or_else(|| anyhow::anyhow!("--version needs a value"))?; + } + "--root" => { + root = PathBuf::from( + it.next() + .ok_or_else(|| anyhow::anyhow!("--root needs a value"))?, + ); + } + "--color" => { + color = match it.next().as_deref() { + Some("always") => ColorMode::Always, + Some("never") => ColorMode::Never, + Some("auto") | None => ColorMode::Auto, + Some(o) => bail!("invalid --color value: {o}"), + }; + } + "-h" | "--help" => { + println!( + "ps-fdiff — per-function asm diff (built ROM vs baserom)\n\n\ +USAGE:\n ps-fdiff [OPTIONS] \n\n\ +OPTIONS:\n\ + -v, --version Game version (default: us)\n\ + --root Repository root (default: .)\n\ + --color always | never | auto (default: auto)\n\ + -h, --help Show this help" + ); + std::process::exit(0); + } + other if !other.starts_with('-') => func = Some(other.to_string()), + other => bail!("unknown argument: {other}"), + } + } + + let func = func.ok_or_else(|| anyhow::anyhow!("a function name is required (see --help)"))?; + Ok(Args { + func, + version, + root, + color, + }) +} + +fn main() -> ExitCode { + ps_core::reset_sigpipe(); + match run() { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(1), // functions differ + Err(e) => { + eprintln!("ps-fdiff: {e:#}"); + ExitCode::from(2) // trouble (function not found, missing files, ...) + } + } +} + +/// Returns Ok(true) when the function matches, Ok(false) when it differs. +fn run() -> Result { + let args = parse_args()?; + let build = args.root.join("build"); + let myimg = build.join(format!("pokestadium-{}.z64", args.version)); + let map_path = build.join(format!("pokestadium-{}.map", args.version)); + let baseimg = args + .root + .join("baseroms") + .join(&args.version) + .join("baserom.z64"); + + for p in [&myimg, &map_path, &baseimg] { + if !p.exists() { + bail!("missing {} (run `make` first)", p.display()); + } + } + + let map = MapFile::read(&map_path)?; + let sym = map + .symbol_by_name(&args.func) + .with_context(|| format!("function {} not found in {}", args.func, map_path.display()))?; + let vrom = sym + .vrom + .ok_or_else(|| anyhow::anyhow!("no ROM offset (vrom) for {} in the map", args.func))?; + let size = sym.size; + if size == 0 { + bail!("function {} has zero size in the map", args.func); + } + + let my = Rom::read(&myimg)?; + let base = Rom::read(&baseimg)?; + + let use_color = match args.color { + ColorMode::Always => true, + ColorMode::Never => false, + ColorMode::Auto => std::io::stdout().is_terminal(), + }; + + println!( + "{} @ vram 0x{:08X}, rom 0x{:X}, size 0x{:X}", + sym.name, sym.vram, vrom, size + ); + println!("{:<44}CURRENT (build)", "TARGET (baserom)"); + + let words = (size / 4) as usize; + let mut all_match = true; + for i in 0..words { + let off = vrom as usize + i * 4; + let vram = (sym.vram + (i as u64) * 4) as u32; + let base_word = base.word_be(off); + let my_word = my.word_be(off); + let base_txt = base_word + .map(|w| disasm::disassemble_word(w, vram, &map)) + .unwrap_or_else(|| "".to_string()); + let my_txt = my_word + .map(|w| disasm::disassemble_word(w, vram, &map)) + .unwrap_or_else(|| "".to_string()); + let differ = base_word != my_word; + if differ { + all_match = false; + } + print_row(vram, &base_txt, &my_txt, differ, use_color); + } + + println!(); + if all_match { + println!("MATCH: {} is byte-identical.", sym.name); + } else { + println!("DIFF: {} does not match.", sym.name); + } + Ok(all_match) +} + +fn print_row(vram: u32, base: &str, cur: &str, differ: bool, color: bool) { + let marker = if differ { "|" } else { " " }; + let left = format!("{base:<40}"); + if color && differ { + println!("0x{vram:08X} {RED}{left}{RESET} {marker} {GREEN}{cur}{RESET}"); + } else { + println!("0x{vram:08X} {left} {marker} {cur}"); + } +} diff --git a/tools/rust/crates/ps-firstdiff/Cargo.toml b/tools/rust/crates/ps-firstdiff/Cargo.toml new file mode 100644 index 0000000..3c29509 --- /dev/null +++ b/tools/rust/crates/ps-firstdiff/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ps-firstdiff" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Find the first difference(s) between the built ROM and the expected ROM" + +[[bin]] +name = "ps-firstdiff" +path = "src/main.rs" + +[dependencies] +ps-core.workspace = true +anyhow.workspace = true diff --git a/tools/rust/crates/ps-firstdiff/src/main.rs b/tools/rust/crates/ps-firstdiff/src/main.rs new file mode 100644 index 0000000..818157d --- /dev/null +++ b/tools/rust/crates/ps-firstdiff/src/main.rs @@ -0,0 +1,171 @@ +//! `ps-firstdiff` — find the first difference(s) between the built ROM and the +//! expected ROM, a Rust port of `tools/first_diff.py`. +//! +//! For each differing 4-byte word it reports the ROM offset, the containing +//! function (resolved from the built `.map`), and the disassembly of both the +//! built and expected instruction, with `jal` targets resolved to symbol names. + +use std::path::PathBuf; +use std::process::ExitCode; + +use anyhow::{bail, Context, Result}; +use ps_core::disasm; +use ps_core::mapfile::MapFile; +use ps_core::rom::Rom; +use ps_core::DEFAULT_VERSION; + +struct Args { + version: String, + root: PathBuf, + count: usize, + add_colons: bool, +} + +fn parse_args() -> Result { + let mut version = DEFAULT_VERSION.to_string(); + let mut root = PathBuf::from("."); + let mut count = 5usize; + let mut add_colons = false; + + let mut it = std::env::args().skip(1); + while let Some(arg) = it.next() { + match arg.as_str() { + "-c" | "--count" => { + count = it + .next() + .ok_or_else(|| anyhow::anyhow!("--count needs a value"))? + .parse() + .context("parsing --count")?; + } + "-v" | "--version" => { + version = it + .next() + .ok_or_else(|| anyhow::anyhow!("--version needs a value"))?; + } + "--root" => { + root = PathBuf::from( + it.next() + .ok_or_else(|| anyhow::anyhow!("--root needs a value"))?, + ); + } + "-a" | "--add-colons" => add_colons = true, + "-h" | "--help" => { + println!( + "ps-firstdiff — first difference between built and expected ROM\n\n\ +USAGE:\n ps-firstdiff [OPTIONS]\n\n\ +OPTIONS:\n\ + -c, --count Report up to N differing instructions (default: 5)\n\ + -v, --version Game version (default: us)\n\ + -a, --add-colons Separate bytes with colons\n\ + --root Repository root (default: .)\n\ + -h, --help Show this help" + ); + std::process::exit(0); + } + other => bail!("unknown argument: {other}"), + } + } + Ok(Args { + version, + root, + count, + add_colons, + }) +} + +fn main() -> ExitCode { + ps_core::reset_sigpipe(); + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("ps-firstdiff: {e:#}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<()> { + let args = parse_args()?; + let rel_rom = PathBuf::from("build").join(format!("pokestadium-{}.z64", args.version)); + let rel_map = PathBuf::from("build").join(format!("pokestadium-{}.map", args.version)); + let built_rom = args.root.join(&rel_rom); + let built_map = args.root.join(&rel_map); + let expected_rom = args.root.join("expected").join(&rel_rom); + let expected_map = args.root.join("expected").join(&rel_map); + + for p in [&built_rom, &built_map, &expected_rom, &expected_map] { + if !p.exists() { + bail!( + "missing {} (run `make` and `make diff-init` first)", + p.display() + ); + } + } + + let built = Rom::read(&built_rom)?; + let expected = Rom::read(&expected_rom)?; + let map = MapFile::read(&built_map)?; + + if built.len() != expected.len() { + println!( + "ROM size mismatch: built {} bytes, expected {} bytes", + built.len(), + expected.len() + ); + } + + let common = built.len().min(expected.len()); + let mut found = 0usize; + let mut offset = 0usize; + while offset + 4 <= common { + let a = built.word_be(offset).unwrap(); + let b = expected.word_be(offset).unwrap(); + if a != b { + report_diff(offset as u64, a, b, &map, args.add_colons); + found += 1; + if found >= args.count { + break; + } + } + offset += 4; + } + + if found == 0 && built.len() == expected.len() { + println!("No differences found — ROMs match."); + } + Ok(()) +} + +fn report_diff(offset: u64, built_word: u32, expected_word: u32, map: &MapFile, add_colons: bool) { + // Name the containing function and compute the vram of this word. + let (loc, vram) = match map.symbol_containing_vrom(offset) { + Some(sym) => { + let vrom = sym.vrom.unwrap_or(offset); + let delta = offset.saturating_sub(vrom); + let vram = sym.vram + delta; + (format!("{}+0x{:X}", sym.name, delta), vram as u32) + } + None => ("".to_string(), 0u32), + }; + + println!("ROM 0x{offset:06X} ({loc}):"); + println!( + " built: {} {}", + fmt_word(built_word, add_colons), + disasm::disassemble_word(built_word, vram, map) + ); + println!( + " expected: {} {}", + fmt_word(expected_word, add_colons), + disasm::disassemble_word(expected_word, vram, map) + ); +} + +fn fmt_word(word: u32, add_colons: bool) -> String { + let b = word.to_be_bytes(); + if add_colons { + format!("{:02X}:{:02X}:{:02X}:{:02X}", b[0], b[1], b[2], b[3]) + } else { + format!("{:02X}{:02X}{:02X}{:02X}", b[0], b[1], b[2], b[3]) + } +} diff --git a/tools/rust/crates/ps-status/Cargo.toml b/tools/rust/crates/ps-status/Cargo.toml new file mode 100644 index 0000000..8998df0 --- /dev/null +++ b/tools/rust/crates/ps-status/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "ps-status" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +description = "Report the status of functions still pending decompilation" + +[[bin]] +name = "ps-status" +path = "src/main.rs" + +[dependencies] +ps-core.workspace = true +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/tools/rust/crates/ps-status/src/buildaware.rs b/tools/rust/crates/ps-status/src/buildaware.rs new file mode 100644 index 0000000..e3d259b --- /dev/null +++ b/tools/rust/crates/ps-status/src/buildaware.rs @@ -0,0 +1,186 @@ +//! Build-aware mode: per-folder byte sizes and match percentages derived from +//! the linker `.map`, a direct port of the algorithm in `progress.py`. +//! +//! For each `.text` symbol the map lists, the function is considered +//! *undecomped* when either the whole-file asm (`asm//.s`) or the +//! per-function asm (`asm//nonmatchings//.s`) still exists on +//! disk; otherwise it is *decomped*. Sizes come from the map. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use ps_core::mapfile::MapFile; + +#[derive(Default, Clone)] +struct Stats { + decomped: u64, + undecomped: u64, +} + +impl Stats { + fn total(&self) -> u64 { + self.decomped + self.undecomped + } + fn percent(&self) -> f64 { + let t = self.total(); + if t == 0 { + 0.0 + } else { + self.decomped as f64 / t as f64 * 100.0 + } + } +} + +pub fn run(root: &Path, version: &str, map_path: &Path, json: bool) -> Result<()> { + let mf = MapFile::read(map_path)?; + let asm_path = root.join("asm").join(version); + let nonmatchings = asm_path.join("nonmatchings"); + + let mut total = Stats::default(); + let mut per_folder: BTreeMap = BTreeMap::new(); + + for sym in mf.text_symbols() { + // progress.py removes the version component from the path, then uses + // parts[2] as the folder and parts[2:] as the file path. + let parts: Vec = sym + .filepath + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .filter(|p| p != version) + .collect(); + if parts.len() <= 2 { + continue; + } + + let mut folder = parts[2].clone(); + if folder == "fragments" && parts.len() > 3 { + folder = format!("fragments/{}", parts[3]); + } + if let Some(idx) = folder.find(".a") { + folder = folder[..idx].to_string(); + } + if folder == "ultralib" { + folder = "libultra".to_string(); + } + + // originalFilePath = parts[2:], with every suffix stripped. + let mut file_path = PathBuf::new(); + for p in &parts[2..] { + file_path.push(p); + } + let extensionless = strip_all_suffixes(&file_path); + + let whole_file_asm = asm_path.join(&extensionless).with_extension("s"); + let whole_undecomped = whole_file_asm.exists(); + let func_asm = nonmatchings + .join(&extensionless) + .join(format!("{}.s", sym.name)); + + let entry = per_folder.entry(folder).or_default(); + if whole_undecomped || func_asm.exists() { + total.undecomped += sym.size; + entry.undecomped += sym.size; + } else { + total.decomped += sym.size; + entry.decomped += sym.size; + } + } + + if json { + print_json(&total, &per_folder); + } else { + print_table(&total, &per_folder); + } + Ok(()) +} + +/// Repeatedly drop the file extension (`a.b.o` -> `a`). +fn strip_all_suffixes(path: &Path) -> PathBuf { + let mut p = path.to_path_buf(); + while p.extension().is_some() { + p.set_extension(""); + } + p +} + +fn print_table(total: &Stats, per_folder: &BTreeMap) { + // Priority: most undecomped bytes first. + let mut rows: Vec<(&String, &Stats)> = per_folder.iter().collect(); + rows.sort_by(|a, b| { + b.1.undecomped + .cmp(&a.1.undecomped) + .then_with(|| a.0.cmp(b.0)) + }); + + println!("Progress by folder (build-aware, bytes):"); + println!( + " {:<24} {:>10} {:>10} {:>10} {:>8}", + "folder", "decomp", "undecomp", "total", "%" + ); + for (folder, s) in rows { + println!( + " {:<24} {:>10} {:>10} {:>10} {:>7.2}%", + folder, + s.decomped, + s.undecomped, + s.total(), + s.percent() + ); + } + println!(); + println!( + "TOTAL: {} / {} bytes decompiled ({:.4}%)", + total.decomped, + total.total(), + total.percent() + ); +} + +fn print_json(total: &Stats, per_folder: &BTreeMap) { + use serde::Serialize; + + #[derive(Serialize)] + struct FolderOut { + folder: String, + decomped_bytes: u64, + undecomped_bytes: u64, + total_bytes: u64, + percent: f64, + } + #[derive(Serialize)] + struct Root { + mode: &'static str, + decomped_bytes: u64, + undecomped_bytes: u64, + total_bytes: u64, + percent: f64, + folders: Vec, + } + + let mut folders: Vec = per_folder + .iter() + .map(|(folder, s)| FolderOut { + folder: folder.clone(), + decomped_bytes: s.decomped, + undecomped_bytes: s.undecomped, + total_bytes: s.total(), + percent: s.percent(), + }) + .collect(); + folders.sort_by(|a, b| { + b.undecomped_bytes + .cmp(&a.undecomped_bytes) + .then_with(|| a.folder.cmp(&b.folder)) + }); + + let root = Root { + mode: "build-aware", + decomped_bytes: total.decomped, + undecomped_bytes: total.undecomped, + total_bytes: total.total(), + percent: total.percent(), + folders, + }; + println!("{}", serde_json::to_string_pretty(&root).unwrap()); +} diff --git a/tools/rust/crates/ps-status/src/main.rs b/tools/rust/crates/ps-status/src/main.rs new file mode 100644 index 0000000..3adede1 --- /dev/null +++ b/tools/rust/crates/ps-status/src/main.rs @@ -0,0 +1,227 @@ +//! `ps-status` — report the status of functions still pending decompilation. +//! +//! Two modes share one binary: +//! +//! * **build-free** (default): enumerate pending functions by scanning +//! `#pragma GLOBAL_ASM(...)` in `src/`, cross-referencing +//! `linker_scripts//symbol_addrs_code.txt` for addresses. Needs no +//! build. +//! * **build-aware** (when `build/pokestadium-.map` exists, or forced +//! with `--build-aware`): add per-folder byte sizes and match percentages, +//! replicating the metric of `progress.py`. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use anyhow::{bail, Result}; +use ps_core::{globalasm, symbols, DEFAULT_VERSION}; + +mod buildaware; + +struct Args { + version: String, + root: PathBuf, + json: bool, + list: bool, + build_aware: Option, +} + +fn parse_args() -> Result { + let mut version = DEFAULT_VERSION.to_string(); + let mut root = PathBuf::from("."); + let mut json = false; + let mut list = false; + let mut build_aware = None; + + let mut it = std::env::args().skip(1); + while let Some(arg) = it.next() { + match arg.as_str() { + "-v" | "--version" => { + version = it + .next() + .ok_or_else(|| anyhow::anyhow!("--version needs a value"))?; + } + "--root" => { + root = PathBuf::from( + it.next() + .ok_or_else(|| anyhow::anyhow!("--root needs a value"))?, + ); + } + "--json" => json = true, + "--list" => list = true, + "--build-aware" => build_aware = Some(true), + "--build-free" => build_aware = Some(false), + "-h" | "--help" => { + print_help(); + std::process::exit(0); + } + other => bail!("unknown argument: {other}"), + } + } + Ok(Args { + version, + root, + json, + list, + build_aware, + }) +} + +fn print_help() { + println!( + "ps-status — status of functions pending decompilation\n\n\ +USAGE:\n ps-status [OPTIONS]\n\n\ +OPTIONS:\n\ + -v, --version Game version (default: us)\n\ + --root Repository root (default: .)\n\ + --build-aware Force build-aware mode (requires build/*.map)\n\ + --build-free Force build-free mode\n\ + --list List every pending function with its address\n\ + --json Emit JSON instead of a table\n\ + -h, --help Show this help" + ); +} + +fn main() -> ExitCode { + ps_core::reset_sigpipe(); + match run() { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + eprintln!("ps-status: {e:#}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result<()> { + let args = parse_args()?; + let map_path = args + .root + .join("build") + .join(format!("pokestadium-{}.map", args.version)); + + let want_build_aware = match args.build_aware { + Some(v) => v, + None => map_path.exists(), + }; + + if want_build_aware { + if !map_path.exists() { + bail!( + "build-aware mode requested but {} does not exist (run `make` and `make diff-init` first)", + map_path.display() + ); + } + buildaware::run(&args.root, &args.version, &map_path, args.json) + } else { + run_build_free(&args, &map_path) + } +} + +fn run_build_free(args: &Args, map_path: &Path) -> Result<()> { + let src_root = args.root.join("src"); + let entries = globalasm::scan(&src_root)?; + + let sym_path = args + .root + .join("linker_scripts") + .join(&args.version) + .join("symbol_addrs_code.txt"); + let syms = symbols::parse_files(&[sym_path.as_path()])?; + + // Group by file_stem, preserving deterministic order. + let mut by_file: BTreeMap)>> = BTreeMap::new(); + for e in &entries { + let addr = syms.get(&e.func).map(|s| s.addr); + by_file + .entry(e.file_stem.clone()) + .or_default() + .push((e.func.clone(), addr)); + } + + let total = entries.len(); + let file_count = by_file.len(); + + if args.json { + print_build_free_json(total, &by_file); + return Ok(()); + } + + println!("Pending functions (build-free): {total} across {file_count} files"); + if !map_path.exists() { + println!("(no build/*.map found — run with `--build-aware` after `make` for byte metrics)"); + } + println!(); + + // Per-file table, most functions first (ties broken by name for determinism). + let mut rows: Vec<(&String, usize)> = by_file.iter().map(|(k, v)| (k, v.len())).collect(); + rows.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + + println!("By file (most functions first):"); + for (file, n) in &rows { + println!(" {file:<20} {n:>4}"); + } + println!(); + println!("Total pending functions: {total}"); + + if args.list { + println!(); + println!("Functions:"); + for (file, funcs) in &by_file { + for (func, addr) in funcs { + match addr { + Some(a) => println!(" {file:<16} {func:<32} 0x{a:08X}"), + None => println!(" {file:<16} {func:<32} (no address)"), + } + } + } + } + + Ok(()) +} + +fn print_build_free_json(total: usize, by_file: &BTreeMap)>>) { + use serde::Serialize; + + #[derive(Serialize)] + struct FuncOut { + func: String, + address: Option, + } + #[derive(Serialize)] + struct FileOut { + file: String, + pending: usize, + functions: Vec, + } + #[derive(Serialize)] + struct Root { + mode: &'static str, + total_pending: usize, + files: Vec, + } + + let mut files: Vec = by_file + .iter() + .map(|(file, funcs)| FileOut { + file: file.clone(), + pending: funcs.len(), + functions: funcs + .iter() + .map(|(func, addr)| FuncOut { + func: func.clone(), + address: addr.map(|a| format!("0x{a:08X}")), + }) + .collect(), + }) + .collect(); + files.sort_by(|a, b| b.pending.cmp(&a.pending).then_with(|| a.file.cmp(&b.file))); + + let root = Root { + mode: "build-free", + total_pending: total, + files, + }; + println!("{}", serde_json::to_string_pretty(&root).unwrap()); +}