From 8304f6ecd33770b3eb266b1b0b7b9a2ccfaad8b4 Mon Sep 17 00:00:00 2001 From: AnonymousRandomPerson Date: Sat, 6 Jan 2024 14:18:12 -0600 Subject: [PATCH] Supported pmdsky-debug aliases for sync --- .../sync_pmdsky_debug/pmdsky_debug_reader.py | 13 +- tools/sync_pmdsky_debug/symbol_details.py | 41 +---- .../sync_from_pmdsky_debug.py | 154 +++++++++--------- .../sync_pmdsky_debug/sync_to_pmdsky_debug.py | 28 +--- 4 files changed, 106 insertions(+), 130 deletions(-) diff --git a/tools/sync_pmdsky_debug/pmdsky_debug_reader.py b/tools/sync_pmdsky_debug/pmdsky_debug_reader.py index be8160c4..e7c01fd1 100644 --- a/tools/sync_pmdsky_debug/pmdsky_debug_reader.py +++ b/tools/sync_pmdsky_debug/pmdsky_debug_reader.py @@ -17,7 +17,6 @@ LANGUAGE_KEYS_XMAP_TO_PMDSKY_DEBUG = {value: key for key, value in LANGUAGE_KEYS # Symbols with duplicate addresses that should be ignored. SYMBOL_BLACKLIST = set([ - 'EXCLUSIVE_ITEM_STAT_BOOST_DATA', 'GAME_STATE_VALUES', ]) @@ -83,6 +82,11 @@ def read_pmdsky_debug_symbols() -> Dict[str, Dict[str, Dict[int, SymbolDetails]] if symbol_name in SYMBOL_BLACKLIST: continue + if 'aliases' in symbol: + aliases = symbol['aliases'] + else: + aliases = [] + def add_symbol_address(address: int, symbol_details: SymbolDetails): if address in symbols: print(f'Warning: Duplicate symbols found for address {hex(address)}: {symbols[address].name}, {symbol_details.name}') @@ -91,11 +95,12 @@ def read_pmdsky_debug_symbols() -> Dict[str, Dict[str, Dict[int, SymbolDetails]] if isinstance(addresses, list): if len(addresses) > 1: for address in addresses: - add_symbol_address(address, SymbolDetails(f'{symbol_name}__{address:08X}', full_file_path, is_data)) + aliases = [f'{alias}__{address:08X}' for alias in aliases] + add_symbol_address(address, SymbolDetails(f'{symbol_name}__{address:08X}', full_file_path, is_data, aliases)) else: - add_symbol_address(addresses[0], SymbolDetails(symbol_name, full_file_path, is_data)) + add_symbol_address(addresses[0], SymbolDetails(symbol_name, full_file_path, is_data, aliases)) else: - add_symbol_address(addresses, SymbolDetails(symbol_name, full_file_path, is_data)) + add_symbol_address(addresses, SymbolDetails(symbol_name, full_file_path, is_data, aliases)) read_symbols_from_array('functions', False) read_symbols_from_array('data', True) diff --git a/tools/sync_pmdsky_debug/symbol_details.py b/tools/sync_pmdsky_debug/symbol_details.py index fdf0f62f..631e903c 100644 --- a/tools/sync_pmdsky_debug/symbol_details.py +++ b/tools/sync_pmdsky_debug/symbol_details.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import List # Some symbol names in the decomp do not match pmdsky-debug because of naming convention differences. # Map these symbol names between the two projects to avoid changes when syncing the projects. @@ -7,39 +8,7 @@ NONMATCHING_SYMBOLS_ARM9 = { '_start_AutoloadDoneCallback': 'StartAutoloadDoneCallback', '_start_ModuleParams': 'START_MODULE_PARAMS', 'CRYPTO_RC4Init': 'Crypto_RC4Init', - 'Debug_FatalError': 'FatalError', - 'Debug_GetDebugFlag': 'GetDebugFlag', - 'Debug_GetLogFlag': 'GetDebugLogFlag', - 'Debug_Init': 'InitDebug', - 'Debug_InitDebugFlag': 'InitDebugFlag', - 'Debug_InitLogFlag': 'InitDebugLogFlag', - 'Debug_Print': 'DebugPrint', - 'Debug_Print0': 'DebugPrint0', - 'Debug_PrintTrace': 'DebugPrintTrace', - 'Debug_SetDebugFlag': 'SetDebugFlag', - 'Debug_SetLogFlag': 'SetDebugLogFlag', - 'Debug_Stripped1': 'InitDebugStripped1', - 'Debug_Stripped2': 'InitDebugStripped2', - 'Debug_Stripped3': 'InitDebugStripped3', - 'Debug_Stripped4': 'InitDebugStripped4', - 'Debug_Stripped5': 'InitDebugStripped5', - 'Debug_Stripped6': 'InitDebugStripped6', - 'DIRECTORY_FILE_TABLE': 'PACK_FILE_PATHS_TABLE', - 'DIRECTORY_FILES_EXTRACTED': 'PACK_FILES_OPENED', - 'DirectoryFile_ExtractDirectoryFile': 'OpenPackFile', - 'DirectoryFile_GetDirectoryFileSize': 'GetFileLengthInPack', - 'DirectoryFile_LoadDirectoryFile': 'LoadFileInPack', - 'DirectoryFileMngr_ExtractAllDirectoryFiles': 'OpenAllPackFiles', - 'DirectoryFileMngr_GetDirectoryFileSize': 'GetFileLengthInPackWithPackNb', - 'DirectoryFileMngr_LoadDirectoryFile': 'LoadFileInPackWithPackId', - 'DirectoryFileMngr_OpenDirectoryFile': 'AllocAndLoadFileInPack', 'disableBankForX_': 'DisableBankForX', - 'FileRom_HandleOpen': 'FileOpen', - 'FileRom_HandleRead': 'FileRead', - 'FileRom_HandleSeek': 'FileSeek', - 'FileRom_InitDataTransfer': 'DataTransferInit', - 'FileRom_StopDataTransfer': 'DataTransferStop', - 'FileRom_Veneer_FileInit': 'FileInitVeneer', 'FX_AtanIdxTable_': 'FX_ATAN_IDX_TABLE', 'G2x_ChangeBlendBrightness_': 'G2x_ChangeBlendBrightness', 'G2x_SetBlendAlpha_': 'G2x_SetBlendAlpha', @@ -137,3 +106,9 @@ class SymbolDetails: name: str file_path: str is_data: bool + aliases: List[str] = field(default_factory=list) + + def get_all_names(self) -> List[str]: + all_names = [self.name] + all_names.extend(self.aliases) + return all_names diff --git a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py index e05dcba1..808aba59 100644 --- a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py @@ -55,81 +55,87 @@ for language, pmdsky_debug_language_symbols in pmdsky_debug_symbols.items(): for address, symbol in pmdsky_debug_section.items(): if section_name == 'arm7' and address < 0x27E0000: - # Shift ARM 7 WRAM to its RAM location. + # Shift ARM7 WRAM to its RAM location. address += WRAM_OFFSET - if address in xmap_section and xmap_section[address].name != symbol.name and xmap_section[address].name not in NONMATCHING_SYMBOLS_ARM9 and xmap_section[address].name not in NONMATCHING_SYMBOLS_ARM7 and xmap_section[address].name not in replaced_symbols: - old_symbol = xmap_section[address] + if address not in xmap_section: + continue + old_symbol = xmap_section[address] + if old_symbol.name in symbol.get_all_names(): + continue + if old_symbol.name in NONMATCHING_SYMBOLS_ARM9 or old_symbol.name in NONMATCHING_SYMBOLS_ARM7: + continue + if old_symbol.name in replaced_symbols: + continue + if '__' in old_symbol.name: + # When a symbol is duplicated in multiple places and has the address appended to the symbol name, + # don't flag the EU name/address combination as a different symbol. + base_symbol = old_symbol.name.split('__')[0] + if any(symbol_name.startswith(f'{base_symbol}__') for symbol_name in symbol.get_all_names()): + continue - if '__' in old_symbol.name: - # When a symbol is duplicated in multiple places and has the address appended to the symbol name, - # don't flag the EU name/address combination as a different symbol. - base_symbol = old_symbol.name.split('__')[0] - if symbol.name.startswith(f'{base_symbol}__'): - continue + print(f'Replacing {old_symbol.name} with {symbol.name}') + replaced_symbols.add(old_symbol.name) - print(f'Replacing {old_symbol.name} with {symbol.name}') - replaced_symbols.add(old_symbol.name) - - # Replace symbol occurrences in ASM files. - if symbol.is_data: - asm_search_string_bases = [ - f'\n{old_symbol.name}:\n', - f'.word {old_symbol.name}\n', - f'; ={old_symbol.name}\n', - f'.global {old_symbol.name}\n', - f'.public {old_symbol.name}\n', - ] - else: - asm_search_string_bases = [ - f'arm_func_start {old_symbol.name}\n', - f'arm_func_end {old_symbol.name}\n', - f'\n{old_symbol.name}: ', - f'thumb_func_start {old_symbol.name}\n', - f'thumb_func_end {old_symbol.name}\n', - f'.word {old_symbol.name}\n', - f'b {old_symbol.name} ; case', - f'bl {old_symbol.name}\n', - f'blx {old_symbol.name}\n', - f'beq {old_symbol.name}\n', - f'bne {old_symbol.name}\n', - f'; ={old_symbol.name}\n', - f'.public {old_symbol.name}\n', - ] - asm_search_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in asm_search_string_bases] - candidate_asm_files = asm_files - if section_name == 'arm7': - candidate_asm_files = asm_arm7_files - for file_path in candidate_asm_files: - with open(file_path, 'r') as asm_file: - asm_contents = asm_file.read() - for search_string in asm_search_strings: - asm_contents = asm_contents.replace(search_string[0], search_string[1]) - if file_path.endswith('.inc') and 'macros' not in file_path and 'syscall' not in file_path: - asm_contents_split = asm_contents.split('\n') - sorted_imports = sorted(asm_contents_split[1:], key=str.casefold) - if len(sorted_imports) and sorted_imports[0] == '': - sorted_imports = sorted_imports[1:] - asm_contents = '#pragma once\n' + '\n'.join(sorted_imports) + '\n' - - with open(file_path, 'w') as asm_file: - asm_file.write(asm_contents) - - # Replace symbol occurrences in C files. - src_search_string_data_regex = re.compile(fr'([ &*(]){old_symbol.name}([,); [])') - src_search_string_data_regex_replace = fr'\1{symbol.name}\2' - - src_search_string_bases = [ - f' {old_symbol.name}(', - f'({old_symbol.name}(', + # Replace symbol occurrences in ASM files. + if symbol.is_data: + asm_search_string_bases = [ + f'\n{old_symbol.name}:\n', + f'.word {old_symbol.name}\n', + f'; ={old_symbol.name}\n', + f'.global {old_symbol.name}\n', + f'.public {old_symbol.name}\n', ] - src_search_function_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in src_search_string_bases] - for file_path in src_files: - with open(file_path, 'r') as src_file: - src_contents = src_file.read() - if symbol.is_data: - src_contents = src_search_string_data_regex.sub(src_search_string_data_regex_replace, src_contents) - else: - for search_string in src_search_function_strings: - src_contents = src_contents.replace(search_string[0], search_string[1]) - with open(file_path, 'w') as src_file: - src_file.write(src_contents) + else: + asm_search_string_bases = [ + f'arm_func_start {old_symbol.name}\n', + f'arm_func_end {old_symbol.name}\n', + f'\n{old_symbol.name}: ', + f'thumb_func_start {old_symbol.name}\n', + f'thumb_func_end {old_symbol.name}\n', + f'.word {old_symbol.name}\n', + f'b {old_symbol.name} ; case', + f'bl {old_symbol.name}\n', + f'blx {old_symbol.name}\n', + f'beq {old_symbol.name}\n', + f'bne {old_symbol.name}\n', + f'; ={old_symbol.name}\n', + f'.public {old_symbol.name}\n', + ] + asm_search_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in asm_search_string_bases] + candidate_asm_files = asm_files + if section_name == 'arm7': + candidate_asm_files = asm_arm7_files + for file_path in candidate_asm_files: + with open(file_path, 'r') as asm_file: + asm_contents = asm_file.read() + for search_string in asm_search_strings: + asm_contents = asm_contents.replace(search_string[0], search_string[1]) + if file_path.endswith('.inc') and 'macros' not in file_path and 'syscall' not in file_path: + asm_contents_split = asm_contents.split('\n') + sorted_imports = sorted(asm_contents_split[1:], key=str.casefold) + if len(sorted_imports) and sorted_imports[0] == '': + sorted_imports = sorted_imports[1:] + asm_contents = '#pragma once\n' + '\n'.join(sorted_imports) + '\n' + + with open(file_path, 'w') as asm_file: + asm_file.write(asm_contents) + + # Replace symbol occurrences in C files. + src_search_string_data_regex = re.compile(fr'([ &*(]){old_symbol.name}([,); [])') + src_search_string_data_regex_replace = fr'\1{symbol.name}\2' + + src_search_string_bases = [ + f' {old_symbol.name}(', + f'({old_symbol.name}(', + ] + src_search_function_strings = [(base, base.replace(old_symbol.name, symbol.name)) for base in src_search_string_bases] + for file_path in src_files: + with open(file_path, 'r') as src_file: + src_contents = src_file.read() + if symbol.is_data: + src_contents = src_search_string_data_regex.sub(src_search_string_data_regex_replace, src_contents) + else: + for search_string in src_search_function_strings: + src_contents = src_contents.replace(search_string[0], search_string[1]) + with open(file_path, 'w') as src_file: + src_file.write(src_contents) diff --git a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py index 4144b923..2e237bcd 100644 --- a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py @@ -83,30 +83,20 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma symbol_type_key = 'functions' if address in pmdsky_debug_section: - # If the address is already defined in pmdsky-debug, replace the old symbol name with the new one in the YAML and header files. + # If the address is already defined in pmdsky-debug, add an alias with the new symbol name from the decomp. old_symbol = pmdsky_debug_section[address] - base_old_symbol_name = get_base_symbol_name(old_symbol.name) - if base_old_symbol_name != base_symbol_name: - print(f'Replacing {base_old_symbol_name} with {base_symbol_name}') + base_old_symbol_names = [get_base_symbol_name(symbol_name) for symbol_name in old_symbol.get_all_names()] + base_old_symbol_name = base_old_symbol_names[0] + if base_symbol_name not in base_old_symbol_names: + print(f'Adding alias for {base_old_symbol_name}: {base_symbol_name}') symbol_array = read_symbol_array(symbol_path, symbol_type_key, yaml_manager) for yaml_symbol in symbol_array: if yaml_symbol['name'] == base_old_symbol_name: - yaml_symbol['name'] = base_symbol_name + if 'aliases' in yaml_symbol: + yaml_symbol['aliases'].append(base_symbol_name) + else: + yaml_symbol['aliases'] = [base_symbol_name] break - - header_path = old_symbol.file_path.replace(SYMBOLS_FOLDER, os.path.join('headers', symbol_type_key)).replace('.yml', '.h') - with open(header_path, 'r') as header_file: - header_contents = header_file.read() - - if symbol.is_data: - # Match data symbols by looking for either the end-of-line semicolon or array start bracket. - header_contents = re.sub(fr' {base_old_symbol_name}([\[;])', fr' {base_symbol_name}\1', header_contents) - else: - # Match function symbols by looking for the open parentheses syntax. - header_contents = header_contents.replace(f' {base_old_symbol_name}(', f' {base_symbol_name}(') - - with open(header_path, 'w') as header_file: - header_file.write(header_contents) return matching_symbol_entry = None