Added pmdsky-debug sync script

asm/main_0200C4CC.s

@@ -45913,13 +45913,13 @@ sub_020318D0: ; 0x020318D0
 	ldmia sp!, {r3, r4, r5, pc}
 	arm_func_end sub_020318D0
 
-	arm_func_start KeyWaitInit__02031908
-KeyWaitInit__02031908: ; 0x02031908
+	arm_func_start sub_02031908
+sub_02031908: ; 0x02031908
 	ldr ip, _02031910 ; =KeyWaitInit__02006DA4
 	bx ip
 	.align 2, 0
 _02031910: .word KeyWaitInit__02006DA4
-	arm_func_end KeyWaitInit__02031908
+	arm_func_end sub_02031908
 
 	arm_func_start sub_02031914
 sub_02031914: ; 0x02031914
@@ -46160,7 +46160,7 @@ _02031C34:
 	mov r0, r7
 	bl sub_02032684
 _02031C4C:
-	bl KeyWaitInit__02031908
+	bl sub_02031908
 	mov r1, #0
 	strb r1, [r7, #0xf4]
 	str r1, [r7, #0xd8]
@@ -47394,7 +47394,7 @@ _02032B7C:
 	mov r0, r7
 	bl sub_020330EC
 _02032B94:
-	bl KeyWaitInit__02031908
+	bl sub_02031908
 	mov r1, #0
 	str r1, [r7, #0xd8]
 	sub r0, r1, #1

tools/sync_pmdsky_debug/.gitignore

@@ -0,0 +1 @@
+pmdsky_debug_location.txt

tools/sync_pmdsky_debug/containing_folder.py

@@ -0,0 +1,5 @@
+import os
+
+CONTAINING_FOLDER = 'sync_pmdsky_debug'
+if os.getcwd().endswith(CONTAINING_FOLDER):
+    os.chdir(os.path.join('..', '..'))

tools/sync_pmdsky_debug/pmdsky_debug_reader.py

@@ -0,0 +1,59 @@
+import os
+import yaml
+from containing_folder import CONTAINING_FOLDER
+from typing import Dict
+from symbol_details import SymbolDetails
+
+SYMBOLS_FOLDER = 'symbols'
+
+# In the returned dictionary, outer key = region, inner key = symbol address, value = symbol details.
+def read_pmdsky_debug_symbols() -> Dict[str, Dict[int, SymbolDetails]]:
+    pmdsky_debug_symbols: Dict[str, Dict[int, SymbolDetails]] = {}
+
+    debug_location_path = os.path.join('tools', CONTAINING_FOLDER, 'pmdsky_debug_location.txt')
+    if not os.path.exists(debug_location_path):
+        print('Missing file path to pmdsky-debug in pmdsky_debug_location.txt.')
+        exit(1)
+
+    with open(debug_location_path, 'r') as debug_location_file:
+        pmdsky_debug_path = debug_location_file.read().strip()
+
+    if not os.path.exists(pmdsky_debug_path) or not os.path.exists(os.path.join(pmdsky_debug_path, 'check_and_format.sh')):
+        print('pmdsky-debug not found at', pmdsky_debug_path)
+        exit(1)
+
+    def read_yaml_symbols(file_path: str, symbols: Dict[int, SymbolDetails] = None, address_suffix = '') -> Dict[int, SymbolDetails]:
+        if symbols is None:
+            symbols = {}
+        full_file_path = os.path.join(pmdsky_debug_path, SYMBOLS_FOLDER, file_path)
+        with open(full_file_path, 'r') as symbols_file:
+            symbols_yaml = yaml.safe_load(symbols_file)
+
+        for function in symbols_yaml[list(symbols_yaml.keys())[0]]['functions']:
+            if 'NA' not in function['address']:
+                continue
+            addresses = function['address'][f'NA{address_suffix}']
+            function_name = function['name']
+            if isinstance(addresses, list):
+                for address in addresses:
+                    symbols[address] = SymbolDetails(f'{function_name}__{address:08X}', full_file_path)
+            else:
+                symbols[addresses] = SymbolDetails(function_name, full_file_path)
+
+        return symbols
+
+    itcm_file = os.path.join('arm9', 'itcm.yml')
+
+    pmdsky_debug_symbols['main'] = read_yaml_symbols('arm9.yml')
+    pmdsky_debug_symbols['main'] = read_yaml_symbols(itcm_file, pmdsky_debug_symbols['main'])
+    pmdsky_debug_symbols['ITCM'] = read_yaml_symbols(itcm_file, address_suffix='-ITCM')
+    for i in range(0, 36):
+        overlay_name = f'overlay{i:02d}'
+        pmdsky_debug_symbols[str(i)] = read_yaml_symbols(f'{overlay_name}.yml')
+        overlay_folder = os.path.join(pmdsky_debug_path, SYMBOLS_FOLDER, overlay_name)
+        if os.path.exists(overlay_folder):
+            for file in os.listdir(overlay_folder):
+                if file.endswith('.yml'):
+                    read_yaml_symbols(os.path.join(overlay_name, file), pmdsky_debug_symbols[str(i)])
+
+    return pmdsky_debug_symbols

tools/sync_pmdsky_debug/requirements.txt

@@ -0,0 +1,2 @@
+PyYAML==6.0.1
+requests==2.31.0

tools/sync_pmdsky_debug/symbol_details.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+@dataclass
+class SymbolDetails:
+    name: str
+    file_path: str

tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py

@@ -0,0 +1,74 @@
+from pmdsky_debug_reader import *
+from xmap_reader import *
+from typing import List
+
+# Syncs symbols from pmdsky-debug (https://github.com/UsernameFodder/pmdsky-debug) to the decomp.
+# To use this script, you will need:
+#   - A file named pmdsky_debug_location.txt with the file path to your local clone of pmdsky-debug.
+#   - Python dependencies in requirements.txt.
+# Make sure there are no uncommitted changes when running this, in case you need to revert.
+
+pmdsky_debug_symbols = read_pmdsky_debug_symbols()
+xmap_symbols = read_xmap_symbols()
+
+asm_files = []
+def add_files_with_extensions(folder: str, extensions: List[str]) -> List[str]:
+    found_files = []
+    for root, _, files in os.walk(folder):
+        for file in files:
+            for extension in extensions:
+                if file.endswith(extension):
+                    found_files.append(os.path.join(root, file))
+                    break
+    return found_files
+
+asm_files = add_files_with_extensions('asm', ['.s', '.inc'])
+src_files = add_files_with_extensions('include', ['.h'])
+src_files.extend(add_files_with_extensions('src', ['.c']))
+
+for section_name, pmdsky_debug_section in pmdsky_debug_symbols.items():
+    if section_name in xmap_symbols:
+        xmap_section = xmap_symbols[section_name]
+    else:
+        xmap_section = {}
+
+    for address, symbol in pmdsky_debug_section.items():
+        if address in xmap_section and xmap_section[address].name != symbol.name:
+            old_symbol = xmap_section[address]
+            print(f'Replacing {old_symbol.name} with {symbol.name}')
+            asm_search_string_bases = [
+                f'arm_func_start {old_symbol.name}\n',
+                f'arm_func_end {old_symbol.name}\n',
+                f'\n{old_symbol.name}: ',
+                f'thumb_func_start {old_symbol.name}\n',
+                f'thumb_func_end {old_symbol.name}\n',
+                f'.word {old_symbol.name}\n',
+                f'b {old_symbol.name} ; case',
+                f'bl {old_symbol.name}\n',
+                f'blx {old_symbol.name}\n',
+                f'beq {old_symbol.name}\n',
+                f'bne {old_symbol.name}\n',
+                f'; ={old_symbol.name}\n',
+                f'.public {old_symbol.name}\n',
+            ]
+            asm_search_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in asm_search_string_bases]
+            for file_path in asm_files:
+                with open(file_path, 'r') as asm_file:
+                    asm_contents = asm_file.read()
+                for search_string in asm_search_strings:
+                    asm_contents = asm_contents.replace(search_string[0], search_string[1])
+                with open(file_path, 'w') as asm_file:
+                    asm_file.write(asm_contents)
+
+            src_search_string_bases = [
+                f' {old_symbol.name}(',
+                f'({old_symbol.name}(',
+            ]
+            src_search_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in src_search_string_bases]
+            for file_path in src_files:
+                with open(file_path, 'r') as src_file:
+                    src_contents = src_file.read()
+                for search_string in src_search_strings:
+                    src_contents = src_contents.replace(search_string[0], search_string[1])
+                with open(file_path, 'w') as src_file:
+                    src_file.write(src_contents)

tools/sync_pmdsky_debug/xmap_reader.py

@@ -0,0 +1,54 @@
+import os
+import requests
+from typing import Dict
+from symbol_details import SymbolDetails
+
+XMAP_PATH = os.path.join('build', 'pmdsky.us', 'main.nef.xMAP')
+MAIN_LSF_PATH = 'main.lsf'
+REMOTE_XMAP_URL = 'https://raw.githubusercontent.com/pret/pmd-sky/xmap/pmdskyus.xMAP'
+
+# In the returned dictionary, outer key = region, inner key = symbol address, value = symbol details.
+def read_xmap_symbols() -> Dict[str, Dict[int, SymbolDetails]]:
+    xmap_symbols: Dict[str, Dict[int, SymbolDetails]] = {}
+    overlay_names: Dict[str, int] = {}
+    OVERLAY_START = 'Overlay '
+    with open(MAIN_LSF_PATH, 'r') as main_lsf_file:
+        for line in main_lsf_file.readlines():
+            if line.startswith(OVERLAY_START):
+                overlay_names[line[len(OVERLAY_START) : -1]] = len(overlay_names)
+
+    SECTION_START = '# .'
+    current_section = None
+
+    if os.path.exists(XMAP_PATH):
+        print('Using local xMAP file.')
+        with open(XMAP_PATH, 'r') as xmap_file:
+            xmap_lines = xmap_file.readlines()
+    else:
+        print('No local xMAP file found. Using remote xMAP file from upstream repo.')
+        remote_xmap_response = requests.get(REMOTE_XMAP_URL)
+        if remote_xmap_response.status_code == 200:
+            xmap_lines = remote_xmap_response.text.split('\n')
+        else:
+            raise Exception(f'Received error {remote_xmap_response.status_code} from {REMOTE_XMAP_URL}')
+
+    NON_FUNCTION_SYMBOLS = set(['$', '.'])
+    for line in xmap_lines:
+        if line.startswith(SECTION_START):
+            section_name = line[len(SECTION_START) : -1]
+            if section_name in overlay_names:
+                current_section = str(overlay_names[section_name])
+            elif section_name == 'main' or section_name == 'ITCM':
+                current_section = section_name
+            else:
+                current_section = None
+            if current_section is not None:
+                xmap_symbols[current_section]: Dict[str, int] = {}
+
+        elif current_section is not None and line.startswith('  ') and len(line) > 28 and line[28] not in NON_FUNCTION_SYMBOLS:
+            symbol_split = line[28:-1].split('\t')
+            symbol_name = symbol_split[0]
+            symbol_address = int(line[2:10], 16)
+            xmap_symbols[current_section][symbol_address] = SymbolDetails(symbol_name, symbol_split[1][1:-1])
+
+    return xmap_symbols