diff --git a/.gitignore b/.gitignore index 26dc792..4b8c8a3 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ config/**/tmp_*/ config/**/objdiff.json # ignore elf files config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9] +config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9].rom .idea/ .vscode/ SLUS_210.07 diff --git a/Makefile b/Makefile index 260e6df..81cd220 100644 --- a/Makefile +++ b/Makefile @@ -111,9 +111,6 @@ build: ## Build project extract-data: ## Extract variables from .data in config directory @python3 tools/python/parse_data.py - -make-asm: ## Create expected asm folder in config directory - @python3 configure.py config/kuon.yaml --make-asm map-mismatch: ## Check for mismatches in mapfile @python3 tools/python/map_mismatch.py --language diff --git a/config/checksum.sha1 b/config/checksum.sha1 index f58e64e..3278653 100644 --- a/config/checksum.sha1 +++ b/config/checksum.sha1 @@ -1 +1 @@ -7124b436f61ebbe5944e4b50b7d4c04475711c11 build/SLUS_210.07 +294a1d49a96589c9caf68837b8d914b7a1ba2ce1 build/SLUS_210.07 diff --git a/config/kuon.yaml b/config/kuon.yaml index 63770ca..564ff60 100644 --- a/config/kuon.yaml +++ b/config/kuon.yaml @@ -28,14 +28,6 @@ options: make_full_disasm_for_code: true asm_nonmatching_label_macro: "" create_bss_pads: False -# custom field used by fix_linkerscript.py to fix linkerscript alignments -_section_subalign: - elf_header: 8 - main: 2 - main_bss: 1 - reg_info: 4 - mips_debug: 4 - rest: 4 segments: - [0, databin, elf_header] - name: main @@ -43,12 +35,6 @@ segments: start: 0x80 vram: 0x100000 bss_size: 0x80BB00 # size of .sbss + size of .bss - align: 1 - subalign: 2 # safe subalign value. proper values are applied by fix_linkerscript.py where necessary - # text has align 8 but other sections may have a different alignment ... - # explicit align correction is performed by fix_linkerscript.py during - # project configuration. vram address is used, when reported, in non text - # sections to calculate the correct align value. subsegments: - { start: 0x000080, type: asmtu, name: sdk/sce/ee/crt0, ld_align_segment_start: 0x8 } - [0x0002B0, asmtu, sdk/sce/ee/libgraph/graph001] @@ -249,10 +235,10 @@ segments: - [0x032110, asmtu, cs4/Seimei/src/Camera] - [0x034930, asmtu, cs4/Seimei/src/Debug] - [0x034D10, asmtu, cs4/Seimei/src/Draw] - - [0x038710, asmtu, cs4/Seimei/src/File] + - [0x038710, c, cs4/Seimei/src/File] - [0x038D00, asmtu, cs4/Seimei/src/Game] - [0x039A00, asmtu, cs4/Seimei/src/Load] - - [0x03A380, asmtu, cs4/Seimei/src/Main] + - [0x03A380, c, cs4/Seimei/src/Main] - [0x03AFE0, asmtu, cs4/Seimei/src/MathUtil] - [0x03BCE0, asmtu, cs4/Seimei/src/MdlLink] - [0x03BDC0, asmtu, cs4/Seimei/src/Mic] @@ -283,7 +269,7 @@ segments: - [0x059C60, asmtu, cs4/Seimei/src/Dss] - [0x05ACB0, asmtu, cs4/Seimei/src/Damage] - [0x05B3B0, asmtu, cs4/Seimei/src/Hmd] - - [0x05E360, asmtu, cs4/Seimei/src/Binder] + - [0x05E360, c, cs4/Seimei/src/Binder] - [0x05E980, asmtu, cs4/Seimei/src/Ene] - [0x060DF0, asmtu, cs4/Seimei/src/EneAct] - [0x062F40, asmtu, cs4/Seimei/src/EneThk] @@ -421,7 +407,7 @@ segments: - [0x105150, asmtu, cs4/Seimei/src/E044Act] - [0x105560, asmtu, cs4/Seimei/src/sfx/SfxFog] - [0x105A00, asmtu, cs4/Seimei/fslib/lib/fsspummng] - - [0x106520, asmtu, cs4/Seimei/src/Event] + - [0x106520, c, cs4/Seimei/src/Event] - [0x106A90, asmtu, cs4/Seimei/src/Event0031] - [0x107380, asmtu, cs4/Seimei/src/Event0051] - [0x107E90, asmtu, cs4/Seimei/src/MiniGame/sgh/sgh] diff --git a/configure.py b/configure.py index cce9d9f..9d31656 100755 --- a/configure.py +++ b/configure.py @@ -5,23 +5,15 @@ from __future__ import annotations import os -import re import sys import json -import yaml -import copy import splat import shutil import argparse -import tempfile import subprocess import ninja_syntax -from dataclasses import dataclass, asdict -#from utils import ensure_path_and_write, normalize_object_path, to_expected_path - -import splat.util.options as splat_options -from splat.segtypes.linker_entry import LinkerEntry, clean_up_path +from splat.segtypes.linker_entry import LinkerEntry from spimdisasm.common.CompilerConfig import compilerOptions from typing import Any, Union, Protocol, Literal, cast @@ -30,16 +22,6 @@ from contextlib import contextmanager from splat.scripts import split from splat.util.conf import load as splat_load_yaml -from splat.segtypes.linker_entry import LinkerEntry - -# from tools.python.fix_gp import fix_gp -from tools.python.fix_assets import fix_assets -from tools.python.fix_linkerscript import fix_linkerscript - -from sys import stdout -from subprocess import run -from dataclasses import dataclass -from struct import unpack @contextmanager def suppress_stdout_stderr(): @@ -110,7 +92,7 @@ def make_compiler_cmd(config_dir: Path, src_path: Path, language: str): def exec_shell(command: list[str]) -> str: - ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + ret = subprocess.run(command, capture_output=True, text=True, check=False) return ret.stdout @@ -196,6 +178,21 @@ def build_stuff( object_strs = [str(obj) for obj in object_paths] + expected_strs = [] + for obj in object_paths: + if obj.name.endswith(".c.o"): + obj = obj.with_name(obj.name.removesuffix(".c.o") + ".o") + elif obj.name.endswith(".s.o"): + obj = obj.with_name(obj.name.removesuffix(".s.o") + ".o") + + if obj.parts[:2] == ("build", "asm"): + p = obj.relative_to("build/asm") + elif obj.parts[:2] == ("build", "src"): + p = obj.relative_to("build/src") + else: + p = obj + expected_strs.append(str(Path("build/expected") / p)) + for object_path in object_paths: if object_path.name == "elf_header.s.o": continue @@ -211,6 +208,14 @@ def build_stuff( implicit_outputs=implicit_outputs, ) + ninja.build( + outputs=expected_strs, + rule="as", + inputs=[str((Path("asm") / s.relative_to("../src") if s.parts[:2] == ("..", "src") else s).with_suffix(".s")) for s in src_paths], + variables=variables, + implicit_outputs=implicit_outputs, + ) + ninja = ninja_syntax.Writer(open(str(ROOT / config_dir / "build.ninja"), "w"), width=9999) # Rules @@ -295,16 +300,15 @@ def build_stuff( else: build(entry.object_path, entry.src_paths, "cc") - elif isinstance(seg, splat.segtypes.common.databin.CommonSegDatabin): - build(entry.object_path, entry.src_paths, "as") - - elif isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin): - build(entry.object_path, entry.src_paths, "as") - - elif isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin): - build(entry.object_path, entry.src_paths, "as") - - elif isinstance(seg, splat.segtypes.common.sbss.CommonSegSbss): + elif isinstance( + seg, + ( + splat.segtypes.common.databin.CommonSegDatabin, + splat.segtypes.common.rodatabin.CommonSegRodatabin, + splat.segtypes.common.textbin.CommonSegTextbin, + splat.segtypes.common.sbss.CommonSegSbss, + ), + ): build(entry.object_path, entry.src_paths, "as") else: @@ -338,288 +342,6 @@ def build_stuff( implicit=[elf_path], ) - -def get_line_file_path(args: AnnotationArgs): - if args.line_file_path is not None: - return args.line_file_path - - # if args.elf_path.name == SH2_SERIAL and "Event/stage" in args.asm_path.as_posix(): -# return Path(f"{TOOLS}/alessatool/dwarf") / Path(args.asm_path.name).with_suffix(".line") - - return None - - -def line_has_vram_addr(line: str, addr_str: str) -> bool: - if addr_str not in line or "*/" not in line: - return False - - return line.index("*/") > line.index(addr_str) - - -def annotate_asm(args: AnnotationArgs): - with open(args.asm_path, "r") as asm_file: - asm_contents = asm_file.read() - - asm_lines = asm_contents.splitlines() - asm_line_index = 0 - - vram_start = args.vram_start - vram_end = args.vram_end - - if vram_start is None or vram_end is None: - vram_start, vram_end = find_vram_bounds(asm_lines) - - line_file_path = get_line_file_path(args) - - if line_file_path is None or not line_file_path.exists(): - addresses = (f"0x{v:X}" for v in range(vram_start, vram_end, 0x4)) - proc = run([args.addr2line_path, "-e", args.elf_path, *addresses], capture_output=True, encoding=args.encoding) - addr2line_output_lines = proc.stdout.splitlines() - else: - # parse the binary line number file. - - # the format is a list of u_shorts, one per line number. - # there should be one line number per vram address, and each vram - # address should be exactly 4 bytes apart, mirroring how the addr2line - # output is formatted - - with open(line_file_path, "rb") as line_file: - line_data = line_file.read() - line_numbers = unpack(f"<{len(line_data) // 2}H", line_data) - compile_unit = args.asm_path.with_suffix(".c").name - addr2line_output_lines = list(map(lambda n : to_addr2line_format(compile_unit, n), line_numbers)) - - main_tu_name = None - prev_tu_name = None - prev_line_number = -1 - function_count = 0 - is_in_function_label = False - current_vram_addr = vram_start - annotated_asm_lines = [] - - for addr_index in range(0, len(addr2line_output_lines) - 1): - line = addr2line_output_lines[addr_index] - - if line.startswith("?"): - current_vram_addr += 0x4 - continue - - separator_index = line.rfind(":") - current_line_number = int(line[separator_index+1:]) - - if current_line_number == prev_line_number: - current_vram_addr += 0x4 - continue - - vram_addr_str = f"{current_vram_addr:X}" - - current_tu_name = line[0:separator_index] - if current_tu_name != main_tu_name and current_tu_name != prev_tu_name: - if main_tu_name: - print(f"[warn] alessatool/annotate: address at {vram_addr_str} belongs to {current_tu_name}") - else: - main_tu_name = current_tu_name - prev_tu_name = current_tu_name - - while True: - if asm_line_index >= len(asm_lines): - raise AssertionError( - f"address 0x{current_vram_addr:X} not found in asm" - ) - - asm_line = asm_lines[asm_line_index] - - if line_has_vram_addr(asm_line, vram_addr_str): - break - - should_append_asm_line = True - if args.tu: - asm_line_trimmed = asm_line.strip() - - # track when we go in & out of function symbols - if asm_line_trimmed.startswith(FUNCTION_SYMBOL_LABEL): - function_count += 1 - is_in_function_label = True - annotated_asm_lines.append(f"{UNIQUE_TEXT_SECTION_DIRECTIVE}{function_count}") - annotated_asm_lines.append("") - elif asm_line_trimmed.startswith(END_FUNCTION_SYMBOL_LABEL): - is_in_function_label = False - - # remove `nop`s - if not is_in_function_label and asm_line_trimmed.endswith("nop"): - should_append_asm_line = False - - # remove `macro.inc` include directive - elif asm_line_trimmed == INCLUDE_MACRO_INC_DIRECTIVE: - should_append_asm_line = False - - if should_append_asm_line: - annotated_asm_lines.append(asm_line) - - asm_line_index += 1 - - annotated_asm_lines.append(f"\t.loc 1 {current_line_number}") - annotated_asm_lines.append(asm_line) - asm_line_index += 1 - - prev_line_number = current_line_number - current_vram_addr += 0x4 - - while asm_line_index < len(asm_lines) - 1: - annotated_asm_lines.append(asm_lines[asm_line_index]) - asm_line_index += 1 - - assert main_tu_name, "no valid compilation unit found" - annotated_asm_lines = [ - ".section .debug", - ".previous", - ".text", - f".file 1 \"{main_tu_name}\"", - *annotated_asm_lines - ] - - append_final_new_line(annotated_asm_lines) - annotated_asm_contents = "\n".join(annotated_asm_lines) - - if not args.stdout and args.out_path: - with open(args.out_path, "w") as out_file: - out_file.write(annotated_asm_contents) - if args.verbose: - print(f"alessatool/annotate: wrote asm to {args.out_path}") - else: - stdout.write(annotated_asm_contents) - - -def append_final_new_line(lines: list[str]): - if lines[-1] != "": - lines.append("") - - -def make_asm(config_path: Path, config: dict[str, Any]): - """ - Extracts assembly for each function into 'expected/asm/' subfolder. - - The extraction is done following these steps: - 1. create temporary directory in project folder - 2. copy yaml to temporary directory - 3. modify yaml to have c subsegments instead of asm subsegments - 4. remove data, bss, etc., subsegments - 5. run splat on the new config yaml - 6. splat extracts assembly for each function into '/asm/nonmatchings/' - 7. copy '/asm/nonmatchings/' to '/expected/asm/' - """ - with tempfile.TemporaryDirectory(dir=config_path, prefix="tmp_") as tmp_dir: - tmp_path = Path(tmp_dir) - - yaml_path = tmp_path / "config.yaml" - asm_path = tmp_path / "asm" / "nonmatchings" - dst_path = tmp_path / ".." / "expected" / "asm" - dst_path = dst_path.resolve().relative_to(ROOT) - - if dst_path.exists(): - print(f"expected asm dir '{dst_path}' already exists") - return - - config = copy.deepcopy(config) - config["options"]["target_path"] = "../" + config["options"]["target_path"] - config["options"]["asm_path"] = "asm" - config["options"]["src_path"] = "src" - config["options"]["build_path"] = "build" - config["options"]["asset_path"] = "assets" - config["options"]["undefined_funcs_auto_path"] = "../" + config["options"]["undefined_funcs_auto_path"] - config["options"]["undefined_syms_auto_path"] = "../" + config["options"]["undefined_syms_auto_path"] - config["options"]["symbol_addrs_path"] = "../" + config["options"]["symbol_addrs_path"] - config["options"]["extensions_path"] = "../" + config["options"]["extensions_path"] - - new_segments: list[Any] = [] - segments: list[Any] = config["segments"] - for segment in segments: - if isinstance(segment, list): - new_segments.append(segment) - elif isinstance(segment, dict) and segment["name"] == "main": - new_subsegments: list[Any] = [] - subsegments = cast(list[Any], segment["subsegments"]) - for subsegment in subsegments: - if isinstance(subsegment, list): - if subsegment[1] == "asm": - subsegment[1] = "c" - new_subsegments.append(subsegment) - elif isinstance(subsegment, dict): - subsegment["type"] = subsegment["type"].strip(".") - if subsegment["type"] == "rodata": - # splat now requires rodata to be always prefixed with a dot (.) - # for the rodata migration to work properly - subsegment["type"] = ".rodata" - new_subsegments.append(subsegment) - segment["subsegments"] = new_subsegments - new_segments.append(segment) - config["segments"] = new_segments - - def rename_locals(base_path: Path): - for asm_file in base_path.rglob("*.s"): - data = asm_file.read_text() - data = re.sub(r"__local_\d+", "", data) - asm_file.write_text(data) - - with yaml_path.open(mode="w") as yaml_file: - yaml.dump(config, yaml_file, default_flow_style=False) - - with suppress_stdout_stderr(): - split.main([yaml_path], modes=["all"], verbose=False) - - # remove '__local_#' from asm - rename_locals(asm_path) - - dst_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(asm_path, dst_path, dirs_exist_ok=True) - - print(f"expected asm extracted to '{dst_path}'") - - # make expected objs - - for subseg in new_segments[1]["subsegments"]: - if isinstance(subseg, list) and subseg[1] == "c": - subseg[1] = "asm" - subseg[2] += ".c" - - config["options"]["asm_jtbl_label_macro"] = "llabel" - - with yaml_path.open(mode="w") as yaml_file: - yaml.dump(config, yaml_file, default_flow_style=False) - - shutil.rmtree(tmp_path / "asm") - (tmp_path / ".splache").unlink() - - with suppress_stdout_stderr(): - split.main([yaml_path], modes=["all"], verbose=False) - - # remove '__local_#' from asm - rename_locals(asm_path) - - dst_path = dst_path.parent / "obj" - tmp_obj_path = tmp_path / "obj" - tmp_asm_dir = tmp_path / "asm" - - cpp = Path("..", "..", "..", (Path("tools") / "cc" / COMPILER)) - - for asm_file in tmp_asm_dir.rglob("*.c.s"): - asm_file_rel = asm_file.relative_to(tmp_path) - obj_file_rel = Path("obj") / asm_file.relative_to(tmp_asm_dir).with_suffix(".o") - obj_file = tmp_obj_path / obj_file_rel.relative_to("obj") - obj_file.parent.mkdir(parents=True, exist_ok=True) - subprocess.run( - f"{cpp} -I../src -I../include -Iinclude -Iinclude/sdk/ee -I../.. '{asm_file_rel}' -o - | " - #f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | " - f"mips-linux-gnu-as -no-pad-sections -EL -march=5900 -mabi=eabi -I../include -o {obj_file_rel} {asm_file_rel}", - shell=True, - cwd=tmp_path, - ) - - shutil.copytree(tmp_obj_path, dst_path, dirs_exist_ok=True) - - print(f"expected obj built to '{dst_path}'") - - def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], language: str): """ Generate `objdiff.json` configuration from splat YAML config. @@ -633,7 +355,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la """ segments: list[Any] = config["segments"] - tu_to_diff: list[tuple[Literal["asm", "c"], str]] = [] + tu_to_diff: list[tuple[Literal["asmtu", "c"], str]] = [] for segment in segments: if not (isinstance(segment, dict) and segment["name"] == "main"): @@ -653,7 +375,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la else: raise RuntimeError("invalid subsegment type") - if subs_type in ("asm", "c"): + if subs_type in ("asmtu", "c"): if subs_name in ( "crt0", "main/glob", @@ -678,7 +400,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la units: list[dict[str, Any]] = [] for tu_type, tu_name in tu_to_diff: - target_path = Path("expected", "obj", tu_name).with_suffix(".c.o") + target_path = Path("build", "expected", tu_name).with_suffix(".o") # since we only compile fully decompiled TUs, the # "c" type implies that the TU is complete @@ -695,7 +417,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la "name": tu_name, "target_path": str(target_path), "base_path": str(base_path) if base_path else None, - "metadata": {"progress_categories": [language]}, + "metadata": { + "source_path": str(Path("..", "src", tu_name).with_suffix(".c")), + "progress_categories": [language] + }, } if not base_path: @@ -717,10 +442,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la objdiff_json: dict[str, Any] = { "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json", - "custom_make": "true", + "custom_make": "ninja", "custom_args": [], - "build_target": False, - "build_base": False, + "build_target": True, + "build_base": True, "watch_patterns": [], "units": units, "progress_categories": progress_categories, @@ -858,7 +583,7 @@ def main(): cwd = Path(os.getcwd()).resolve() if not ROOT.samefile(cwd): print("ERROR: this script must be run from it's directory!") - exit(1) + sys.exit(1) compilerOptions["MWCCPS2"].value.bigAddendWorkaroundForMigratedFunctions = False @@ -876,9 +601,9 @@ def main(): if basename not in LANGUAGES: - supported_elfs = f"{set(f'{elf} ({lang})' for elf, lang in LANGUAGES.items())}".replace("'", "") + supported_elfs = f"{ {f'{elf} ({lang})' for elf, lang in LANGUAGES.items()} }".replace("'", "") print(f"unsupported game ELF. Supported versions are: {supported_elfs}") - exit(1) + sys.exit(1) language = LANGUAGES[basename] @@ -886,10 +611,6 @@ def main(): clean(config_dir, config) return - if args.make_asm: - make_asm(config_dir, config) - return - if args.clean: clean(config_dir, config) @@ -906,36 +627,12 @@ def main(): generate_lcf() - - # fix asset .incbin path from relative to root to relative to config dir: - # e.g., change: - # .incbin "config/assets/name.section.bin" - # to: - # .incbin "assets/name.section.bin" - fix_assets(asm_data_path, asset_rel_path) - linker_entries = split.linker_writer.entries build_stuff(config_dir, split.config, linker_entries, language) write_permuter_settings(config_dir, src_path, language) - # # replace gp_rel assembler macro with explicit offset as the gcc used - # # to compile the code does not support it - # gp_value = split.config["options"]["gp_value"] - # symbol_addrs_path = Path(split.config["options"]["symbol_addrs_path"]) - # asm_rel_path = (config_dir / asm_path).resolve().relative_to(ROOT) - # symbol_addrs_rel_path = (config_dir / symbol_addrs_path).resolve().relative_to(ROOT) - # assert asm_rel_path.is_dir(), f"{asm_rel_path} not found or not a directory" - # assert symbol_addrs_rel_path.is_file(), f"{symbol_addrs_rel_path} not found" - # fix_gp(asm_rel_path, gp_value, symbol_addrs_rel_path) - - # fix linkerscript by applying explicit alignments as - # specified in the config yaml - linkerscript_path = (config_dir / f"{basename}.ld").resolve().relative_to(ROOT) - assert linkerscript_path.is_file(), f"{linkerscript_path} not found" - fix_linkerscript(split.config, linkerscript_path) - generate_objdiff_configuration(config_dir, split.config, language) diff --git a/objdiff.patch b/objdiff.patch new file mode 100644 index 0000000..d3c0706 --- /dev/null +++ b/objdiff.patch @@ -0,0 +1,634 @@ +diff --git a/.gitignore b/.gitignore +index 26dc792..4b8c8a3 100644 +--- a/.gitignore ++++ b/.gitignore +@@ -17,6 +17,7 @@ config/**/tmp_*/ + config/**/objdiff.json + # ignore elf files + config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9] ++config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9].rom + .idea/ + .vscode/ + SLUS_210.07 +diff --git a/Makefile b/Makefile +index 260e6df..4a4a30d 100644 +--- a/Makefile ++++ b/Makefile +@@ -96,7 +96,7 @@ help: $(WITH_ENV) ## Show this help + ## + ## Commands: + configure: ## Configure project (needs SLUS_210.07) +- @python3 configure.py config/kuon.yaml -c ++ @uv run configure.py config/kuon.yaml -c + + .build-only: + @cd config/; \ +@@ -111,9 +111,6 @@ build: ## Build project + + extract-data: ## Extract variables from .data in config directory + @python3 tools/python/parse_data.py +- +-make-asm: ## Create expected asm folder in config directory +- @python3 configure.py config/kuon.yaml --make-asm + + map-mismatch: ## Check for mismatches in mapfile + @python3 tools/python/map_mismatch.py --language +diff --git a/config/checksum.sha1 b/config/checksum.sha1 +index f58e64e..3278653 100644 +--- a/config/checksum.sha1 ++++ b/config/checksum.sha1 +@@ -1 +1 @@ +-7124b436f61ebbe5944e4b50b7d4c04475711c11 build/SLUS_210.07 ++294a1d49a96589c9caf68837b8d914b7a1ba2ce1 build/SLUS_210.07 +diff --git a/config/kuon.yaml b/config/kuon.yaml +index 63770ca..564ff60 100644 +--- a/config/kuon.yaml ++++ b/config/kuon.yaml +@@ -28,14 +28,6 @@ options: + make_full_disasm_for_code: true + asm_nonmatching_label_macro: "" + create_bss_pads: False +-# custom field used by fix_linkerscript.py to fix linkerscript alignments +-_section_subalign: +- elf_header: 8 +- main: 2 +- main_bss: 1 +- reg_info: 4 +- mips_debug: 4 +- rest: 4 + segments: + - [0, databin, elf_header] + - name: main +@@ -43,12 +35,6 @@ segments: + start: 0x80 + vram: 0x100000 + bss_size: 0x80BB00 # size of .sbss + size of .bss +- align: 1 +- subalign: 2 # safe subalign value. proper values are applied by fix_linkerscript.py where necessary +- # text has align 8 but other sections may have a different alignment ... +- # explicit align correction is performed by fix_linkerscript.py during +- # project configuration. vram address is used, when reported, in non text +- # sections to calculate the correct align value. + subsegments: + - { start: 0x000080, type: asmtu, name: sdk/sce/ee/crt0, ld_align_segment_start: 0x8 } + - [0x0002B0, asmtu, sdk/sce/ee/libgraph/graph001] +@@ -249,10 +235,10 @@ segments: + - [0x032110, asmtu, cs4/Seimei/src/Camera] + - [0x034930, asmtu, cs4/Seimei/src/Debug] + - [0x034D10, asmtu, cs4/Seimei/src/Draw] +- - [0x038710, asmtu, cs4/Seimei/src/File] ++ - [0x038710, c, cs4/Seimei/src/File] + - [0x038D00, asmtu, cs4/Seimei/src/Game] + - [0x039A00, asmtu, cs4/Seimei/src/Load] +- - [0x03A380, asmtu, cs4/Seimei/src/Main] ++ - [0x03A380, c, cs4/Seimei/src/Main] + - [0x03AFE0, asmtu, cs4/Seimei/src/MathUtil] + - [0x03BCE0, asmtu, cs4/Seimei/src/MdlLink] + - [0x03BDC0, asmtu, cs4/Seimei/src/Mic] +@@ -283,7 +269,7 @@ segments: + - [0x059C60, asmtu, cs4/Seimei/src/Dss] + - [0x05ACB0, asmtu, cs4/Seimei/src/Damage] + - [0x05B3B0, asmtu, cs4/Seimei/src/Hmd] +- - [0x05E360, asmtu, cs4/Seimei/src/Binder] ++ - [0x05E360, c, cs4/Seimei/src/Binder] + - [0x05E980, asmtu, cs4/Seimei/src/Ene] + - [0x060DF0, asmtu, cs4/Seimei/src/EneAct] + - [0x062F40, asmtu, cs4/Seimei/src/EneThk] +@@ -421,7 +407,7 @@ segments: + - [0x105150, asmtu, cs4/Seimei/src/E044Act] + - [0x105560, asmtu, cs4/Seimei/src/sfx/SfxFog] + - [0x105A00, asmtu, cs4/Seimei/fslib/lib/fsspummng] +- - [0x106520, asmtu, cs4/Seimei/src/Event] ++ - [0x106520, c, cs4/Seimei/src/Event] + - [0x106A90, asmtu, cs4/Seimei/src/Event0031] + - [0x107380, asmtu, cs4/Seimei/src/Event0051] + - [0x107E90, asmtu, cs4/Seimei/src/MiniGame/sgh/sgh] +diff --git a/configure.py b/configure.py +index cce9d9f..9d31656 100755 +--- a/configure.py ++++ b/configure.py +@@ -5,23 +5,15 @@ + from __future__ import annotations + + import os +-import re + import sys + import json +-import yaml +-import copy + import splat + import shutil + import argparse +-import tempfile + import subprocess + import ninja_syntax + +-from dataclasses import dataclass, asdict +-#from utils import ensure_path_and_write, normalize_object_path, to_expected_path +- +-import splat.util.options as splat_options +-from splat.segtypes.linker_entry import LinkerEntry, clean_up_path ++from splat.segtypes.linker_entry import LinkerEntry + from spimdisasm.common.CompilerConfig import compilerOptions + + from typing import Any, Union, Protocol, Literal, cast +@@ -30,16 +22,6 @@ from contextlib import contextmanager + + from splat.scripts import split + from splat.util.conf import load as splat_load_yaml +-from splat.segtypes.linker_entry import LinkerEntry +- +-# from tools.python.fix_gp import fix_gp +-from tools.python.fix_assets import fix_assets +-from tools.python.fix_linkerscript import fix_linkerscript +- +-from sys import stdout +-from subprocess import run +-from dataclasses import dataclass +-from struct import unpack + + @contextmanager + def suppress_stdout_stderr(): +@@ -110,7 +92,7 @@ def make_compiler_cmd(config_dir: Path, src_path: Path, language: str): + + + def exec_shell(command: list[str]) -> str: +- ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) ++ ret = subprocess.run(command, capture_output=True, text=True, check=False) + return ret.stdout + + +@@ -196,6 +178,21 @@ def build_stuff( + + object_strs = [str(obj) for obj in object_paths] + ++ expected_strs = [] ++ for obj in object_paths: ++ if obj.name.endswith(".c.o"): ++ obj = obj.with_name(obj.name.removesuffix(".c.o") + ".o") ++ elif obj.name.endswith(".s.o"): ++ obj = obj.with_name(obj.name.removesuffix(".s.o") + ".o") ++ ++ if obj.parts[:2] == ("build", "asm"): ++ p = obj.relative_to("build/asm") ++ elif obj.parts[:2] == ("build", "src"): ++ p = obj.relative_to("build/src") ++ else: ++ p = obj ++ expected_strs.append(str(Path("build/expected") / p)) ++ + for object_path in object_paths: + if object_path.name == "elf_header.s.o": + continue +@@ -211,6 +208,14 @@ def build_stuff( + implicit_outputs=implicit_outputs, + ) + ++ ninja.build( ++ outputs=expected_strs, ++ rule="as", ++ inputs=[str((Path("asm") / s.relative_to("../src") if s.parts[:2] == ("..", "src") else s).with_suffix(".s")) for s in src_paths], ++ variables=variables, ++ implicit_outputs=implicit_outputs, ++ ) ++ + ninja = ninja_syntax.Writer(open(str(ROOT / config_dir / "build.ninja"), "w"), width=9999) + + # Rules +@@ -295,16 +300,15 @@ def build_stuff( + else: + build(entry.object_path, entry.src_paths, "cc") + +- elif isinstance(seg, splat.segtypes.common.databin.CommonSegDatabin): +- build(entry.object_path, entry.src_paths, "as") +- +- elif isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin): +- build(entry.object_path, entry.src_paths, "as") +- +- elif isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin): +- build(entry.object_path, entry.src_paths, "as") +- +- elif isinstance(seg, splat.segtypes.common.sbss.CommonSegSbss): ++ elif isinstance( ++ seg, ++ ( ++ splat.segtypes.common.databin.CommonSegDatabin, ++ splat.segtypes.common.rodatabin.CommonSegRodatabin, ++ splat.segtypes.common.textbin.CommonSegTextbin, ++ splat.segtypes.common.sbss.CommonSegSbss, ++ ), ++ ): + build(entry.object_path, entry.src_paths, "as") + + else: +@@ -338,288 +342,6 @@ def build_stuff( + implicit=[elf_path], + ) + +- +-def get_line_file_path(args: AnnotationArgs): +- if args.line_file_path is not None: +- return args.line_file_path +- +- # if args.elf_path.name == SH2_SERIAL and "Event/stage" in args.asm_path.as_posix(): +-# return Path(f"{TOOLS}/alessatool/dwarf") / Path(args.asm_path.name).with_suffix(".line") +- +- return None +- +- +-def line_has_vram_addr(line: str, addr_str: str) -> bool: +- if addr_str not in line or "*/" not in line: +- return False +- +- return line.index("*/") > line.index(addr_str) +- +- +-def annotate_asm(args: AnnotationArgs): +- with open(args.asm_path, "r") as asm_file: +- asm_contents = asm_file.read() +- +- asm_lines = asm_contents.splitlines() +- asm_line_index = 0 +- +- vram_start = args.vram_start +- vram_end = args.vram_end +- +- if vram_start is None or vram_end is None: +- vram_start, vram_end = find_vram_bounds(asm_lines) +- +- line_file_path = get_line_file_path(args) +- +- if line_file_path is None or not line_file_path.exists(): +- addresses = (f"0x{v:X}" for v in range(vram_start, vram_end, 0x4)) +- proc = run([args.addr2line_path, "-e", args.elf_path, *addresses], capture_output=True, encoding=args.encoding) +- addr2line_output_lines = proc.stdout.splitlines() +- else: +- # parse the binary line number file. +- +- # the format is a list of u_shorts, one per line number. +- # there should be one line number per vram address, and each vram +- # address should be exactly 4 bytes apart, mirroring how the addr2line +- # output is formatted +- +- with open(line_file_path, "rb") as line_file: +- line_data = line_file.read() +- line_numbers = unpack(f"<{len(line_data) // 2}H", line_data) +- compile_unit = args.asm_path.with_suffix(".c").name +- addr2line_output_lines = list(map(lambda n : to_addr2line_format(compile_unit, n), line_numbers)) +- +- main_tu_name = None +- prev_tu_name = None +- prev_line_number = -1 +- function_count = 0 +- is_in_function_label = False +- current_vram_addr = vram_start +- annotated_asm_lines = [] +- +- for addr_index in range(0, len(addr2line_output_lines) - 1): +- line = addr2line_output_lines[addr_index] +- +- if line.startswith("?"): +- current_vram_addr += 0x4 +- continue +- +- separator_index = line.rfind(":") +- current_line_number = int(line[separator_index+1:]) +- +- if current_line_number == prev_line_number: +- current_vram_addr += 0x4 +- continue +- +- vram_addr_str = f"{current_vram_addr:X}" +- +- current_tu_name = line[0:separator_index] +- if current_tu_name != main_tu_name and current_tu_name != prev_tu_name: +- if main_tu_name: +- print(f"[warn] alessatool/annotate: address at {vram_addr_str} belongs to {current_tu_name}") +- else: +- main_tu_name = current_tu_name +- prev_tu_name = current_tu_name +- +- while True: +- if asm_line_index >= len(asm_lines): +- raise AssertionError( +- f"address 0x{current_vram_addr:X} not found in asm" +- ) +- +- asm_line = asm_lines[asm_line_index] +- +- if line_has_vram_addr(asm_line, vram_addr_str): +- break +- +- should_append_asm_line = True +- if args.tu: +- asm_line_trimmed = asm_line.strip() +- +- # track when we go in & out of function symbols +- if asm_line_trimmed.startswith(FUNCTION_SYMBOL_LABEL): +- function_count += 1 +- is_in_function_label = True +- annotated_asm_lines.append(f"{UNIQUE_TEXT_SECTION_DIRECTIVE}{function_count}") +- annotated_asm_lines.append("") +- elif asm_line_trimmed.startswith(END_FUNCTION_SYMBOL_LABEL): +- is_in_function_label = False +- +- # remove `nop`s +- if not is_in_function_label and asm_line_trimmed.endswith("nop"): +- should_append_asm_line = False +- +- # remove `macro.inc` include directive +- elif asm_line_trimmed == INCLUDE_MACRO_INC_DIRECTIVE: +- should_append_asm_line = False +- +- if should_append_asm_line: +- annotated_asm_lines.append(asm_line) +- +- asm_line_index += 1 +- +- annotated_asm_lines.append(f"\t.loc 1 {current_line_number}") +- annotated_asm_lines.append(asm_line) +- asm_line_index += 1 +- +- prev_line_number = current_line_number +- current_vram_addr += 0x4 +- +- while asm_line_index < len(asm_lines) - 1: +- annotated_asm_lines.append(asm_lines[asm_line_index]) +- asm_line_index += 1 +- +- assert main_tu_name, "no valid compilation unit found" +- annotated_asm_lines = [ +- ".section .debug", +- ".previous", +- ".text", +- f".file 1 \"{main_tu_name}\"", +- *annotated_asm_lines +- ] +- +- append_final_new_line(annotated_asm_lines) +- annotated_asm_contents = "\n".join(annotated_asm_lines) +- +- if not args.stdout and args.out_path: +- with open(args.out_path, "w") as out_file: +- out_file.write(annotated_asm_contents) +- if args.verbose: +- print(f"alessatool/annotate: wrote asm to {args.out_path}") +- else: +- stdout.write(annotated_asm_contents) +- +- +-def append_final_new_line(lines: list[str]): +- if lines[-1] != "": +- lines.append("") +- +- +-def make_asm(config_path: Path, config: dict[str, Any]): +- """ +- Extracts assembly for each function into 'expected/asm/' subfolder. +- +- The extraction is done following these steps: +- 1. create temporary directory in project folder +- 2. copy yaml to temporary directory +- 3. modify yaml to have c subsegments instead of asm subsegments +- 4. remove data, bss, etc., subsegments +- 5. run splat on the new config yaml +- 6. splat extracts assembly for each function into '/asm/nonmatchings/' +- 7. copy '/asm/nonmatchings/' to '/expected/asm/' +- """ +- with tempfile.TemporaryDirectory(dir=config_path, prefix="tmp_") as tmp_dir: +- tmp_path = Path(tmp_dir) +- +- yaml_path = tmp_path / "config.yaml" +- asm_path = tmp_path / "asm" / "nonmatchings" +- dst_path = tmp_path / ".." / "expected" / "asm" +- dst_path = dst_path.resolve().relative_to(ROOT) +- +- if dst_path.exists(): +- print(f"expected asm dir '{dst_path}' already exists") +- return +- +- config = copy.deepcopy(config) +- config["options"]["target_path"] = "../" + config["options"]["target_path"] +- config["options"]["asm_path"] = "asm" +- config["options"]["src_path"] = "src" +- config["options"]["build_path"] = "build" +- config["options"]["asset_path"] = "assets" +- config["options"]["undefined_funcs_auto_path"] = "../" + config["options"]["undefined_funcs_auto_path"] +- config["options"]["undefined_syms_auto_path"] = "../" + config["options"]["undefined_syms_auto_path"] +- config["options"]["symbol_addrs_path"] = "../" + config["options"]["symbol_addrs_path"] +- config["options"]["extensions_path"] = "../" + config["options"]["extensions_path"] +- +- new_segments: list[Any] = [] +- segments: list[Any] = config["segments"] +- for segment in segments: +- if isinstance(segment, list): +- new_segments.append(segment) +- elif isinstance(segment, dict) and segment["name"] == "main": +- new_subsegments: list[Any] = [] +- subsegments = cast(list[Any], segment["subsegments"]) +- for subsegment in subsegments: +- if isinstance(subsegment, list): +- if subsegment[1] == "asm": +- subsegment[1] = "c" +- new_subsegments.append(subsegment) +- elif isinstance(subsegment, dict): +- subsegment["type"] = subsegment["type"].strip(".") +- if subsegment["type"] == "rodata": +- # splat now requires rodata to be always prefixed with a dot (.) +- # for the rodata migration to work properly +- subsegment["type"] = ".rodata" +- new_subsegments.append(subsegment) +- segment["subsegments"] = new_subsegments +- new_segments.append(segment) +- config["segments"] = new_segments +- +- def rename_locals(base_path: Path): +- for asm_file in base_path.rglob("*.s"): +- data = asm_file.read_text() +- data = re.sub(r"__local_\d+", "", data) +- asm_file.write_text(data) +- +- with yaml_path.open(mode="w") as yaml_file: +- yaml.dump(config, yaml_file, default_flow_style=False) +- +- with suppress_stdout_stderr(): +- split.main([yaml_path], modes=["all"], verbose=False) +- +- # remove '__local_#' from asm +- rename_locals(asm_path) +- +- dst_path.parent.mkdir(parents=True, exist_ok=True) +- shutil.copytree(asm_path, dst_path, dirs_exist_ok=True) +- +- print(f"expected asm extracted to '{dst_path}'") +- +- # make expected objs +- +- for subseg in new_segments[1]["subsegments"]: +- if isinstance(subseg, list) and subseg[1] == "c": +- subseg[1] = "asm" +- subseg[2] += ".c" +- +- config["options"]["asm_jtbl_label_macro"] = "llabel" +- +- with yaml_path.open(mode="w") as yaml_file: +- yaml.dump(config, yaml_file, default_flow_style=False) +- +- shutil.rmtree(tmp_path / "asm") +- (tmp_path / ".splache").unlink() +- +- with suppress_stdout_stderr(): +- split.main([yaml_path], modes=["all"], verbose=False) +- +- # remove '__local_#' from asm +- rename_locals(asm_path) +- +- dst_path = dst_path.parent / "obj" +- tmp_obj_path = tmp_path / "obj" +- tmp_asm_dir = tmp_path / "asm" +- +- cpp = Path("..", "..", "..", (Path("tools") / "cc" / COMPILER)) +- +- for asm_file in tmp_asm_dir.rglob("*.c.s"): +- asm_file_rel = asm_file.relative_to(tmp_path) +- obj_file_rel = Path("obj") / asm_file.relative_to(tmp_asm_dir).with_suffix(".o") +- obj_file = tmp_obj_path / obj_file_rel.relative_to("obj") +- obj_file.parent.mkdir(parents=True, exist_ok=True) +- subprocess.run( +- f"{cpp} -I../src -I../include -Iinclude -Iinclude/sdk/ee -I../.. '{asm_file_rel}' -o - | " +- #f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | " +- f"mips-linux-gnu-as -no-pad-sections -EL -march=5900 -mabi=eabi -I../include -o {obj_file_rel} {asm_file_rel}", +- shell=True, +- cwd=tmp_path, +- ) +- +- shutil.copytree(tmp_obj_path, dst_path, dirs_exist_ok=True) +- +- print(f"expected obj built to '{dst_path}'") +- +- + def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], language: str): + """ + Generate `objdiff.json` configuration from splat YAML config. +@@ -633,7 +355,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la + """ + segments: list[Any] = config["segments"] + +- tu_to_diff: list[tuple[Literal["asm", "c"], str]] = [] ++ tu_to_diff: list[tuple[Literal["asmtu", "c"], str]] = [] + + for segment in segments: + if not (isinstance(segment, dict) and segment["name"] == "main"): +@@ -653,7 +375,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la + else: + raise RuntimeError("invalid subsegment type") + +- if subs_type in ("asm", "c"): ++ if subs_type in ("asmtu", "c"): + if subs_name in ( + "crt0", + "main/glob", +@@ -678,7 +400,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la + units: list[dict[str, Any]] = [] + + for tu_type, tu_name in tu_to_diff: +- target_path = Path("expected", "obj", tu_name).with_suffix(".c.o") ++ target_path = Path("build", "expected", tu_name).with_suffix(".o") + + # since we only compile fully decompiled TUs, the + # "c" type implies that the TU is complete +@@ -695,7 +417,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la + "name": tu_name, + "target_path": str(target_path), + "base_path": str(base_path) if base_path else None, +- "metadata": {"progress_categories": [language]}, ++ "metadata": { ++ "source_path": str(Path("..", "src", tu_name).with_suffix(".c")), ++ "progress_categories": [language] ++ }, + } + + if not base_path: +@@ -717,10 +442,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la + + objdiff_json: dict[str, Any] = { + "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json", +- "custom_make": "true", ++ "custom_make": "ninja", + "custom_args": [], +- "build_target": False, +- "build_base": False, ++ "build_target": True, ++ "build_base": True, + "watch_patterns": [], + "units": units, + "progress_categories": progress_categories, +@@ -858,7 +583,7 @@ def main(): + cwd = Path(os.getcwd()).resolve() + if not ROOT.samefile(cwd): + print("ERROR: this script must be run from it's directory!") +- exit(1) ++ sys.exit(1) + + compilerOptions["MWCCPS2"].value.bigAddendWorkaroundForMigratedFunctions = False + +@@ -876,9 +601,9 @@ def main(): + + + if basename not in LANGUAGES: +- supported_elfs = f"{set(f'{elf} ({lang})' for elf, lang in LANGUAGES.items())}".replace("'", "") ++ supported_elfs = f"{ {f'{elf} ({lang})' for elf, lang in LANGUAGES.items()} }".replace("'", "") + print(f"unsupported game ELF. Supported versions are: {supported_elfs}") +- exit(1) ++ sys.exit(1) + + language = LANGUAGES[basename] + +@@ -886,10 +611,6 @@ def main(): + clean(config_dir, config) + return + +- if args.make_asm: +- make_asm(config_dir, config) +- return +- + if args.clean: + clean(config_dir, config) + +@@ -906,36 +627,12 @@ def main(): + + generate_lcf() + +- +- # fix asset .incbin path from relative to root to relative to config dir: +- # e.g., change: +- # .incbin "config/assets/name.section.bin" +- # to: +- # .incbin "assets/name.section.bin" +- fix_assets(asm_data_path, asset_rel_path) +- + linker_entries = split.linker_writer.entries + + build_stuff(config_dir, split.config, linker_entries, language) + + write_permuter_settings(config_dir, src_path, language) + +- # # replace gp_rel assembler macro with explicit offset as the gcc used +- # # to compile the code does not support it +- # gp_value = split.config["options"]["gp_value"] +- # symbol_addrs_path = Path(split.config["options"]["symbol_addrs_path"]) +- # asm_rel_path = (config_dir / asm_path).resolve().relative_to(ROOT) +- # symbol_addrs_rel_path = (config_dir / symbol_addrs_path).resolve().relative_to(ROOT) +- # assert asm_rel_path.is_dir(), f"{asm_rel_path} not found or not a directory" +- # assert symbol_addrs_rel_path.is_file(), f"{symbol_addrs_rel_path} not found" +- # fix_gp(asm_rel_path, gp_value, symbol_addrs_rel_path) +- +- # fix linkerscript by applying explicit alignments as +- # specified in the config yaml +- linkerscript_path = (config_dir / f"{basename}.ld").resolve().relative_to(ROOT) +- assert linkerscript_path.is_file(), f"{linkerscript_path} not found" +- fix_linkerscript(split.config, linkerscript_path) +- + generate_objdiff_configuration(config_dir, split.config, language) + + diff --git a/tools/python/add_line_numbers_to_asm.py b/tools/python/add_line_numbers_to_asm.py deleted file mode 100755 index e72ca7d..0000000 --- a/tools/python/add_line_numbers_to_asm.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python3 - -# SPDX-FileCopyrightText: Copyright 2025 karas84 (https://github.com/karas84) -# SPDX-License-Identifier: MIT -# -# This script inserts source line debug information (.loc directives) into assembly files -# using data extracted from a JSON "stdump" file generated by the ccc tool (version 2.1), -# available at https://github.com/chaoticgd/ccc. -# -# Original concept by Mc-muffin (https://github.com/Mc-muffin). - -from __future__ import annotations - -import re -import io -import sys -import json -import argparse - -from typing import Callable, Protocol, cast -from pathlib import Path - - -class STDUMPJson: - def __init__(self, json_path: Path): - self._json = json.loads(json_path.read_text()) - self._line_cache = self._build_line_cache() - self._validate_sub_source_files() - - def _build_line_cache(self): - line_cache: dict[int, int] = {} - functions = self._json["functions"] - - for fun in functions: - if "line_numbers" not in fun: - continue - - for addr, num in fun["line_numbers"]: - # there may be more lines for the same address. Ghidra seems to - # keep the last one, so that's what we are also doing - line_cache[addr] = num - - return line_cache - - def _validate_sub_source_files(self): - functions = self._json["functions"] - for fun in functions: - if (line_numbers := fun.get("line_numbers")) is None: - continue - - if (sub_source_files := fun.get("sub_source_files")) is None: - continue - - line_numbers = cast(list[tuple[int, int]], line_numbers) - sub_source_files = cast(list[tuple[int, str]], sub_source_files) - - # # some functions may have "unterminated" inlines, but they usually have just one line number - # if len(sub_source_files) % 2 != 0: # and len(line_numbers) != 1: - # print(fun["name"], len(line_numbers)) - - # for addr, source in sub_source_files: - - def find_function(self, name: str): - functions = self._json["functions"] - fun = next((f for f in functions if f["name"] == name), None) - return fun - - def get_line(self, addr: int): - return self._line_cache.get(addr) - - -def make_range_checker(lst: list[tuple[int, str]], delimiter: str) -> Callable[[int], bool]: - """ - Given a sorted list of (number, label) where number increases, - build a checker that returns True if x is in a valid interval. - - Interpretation: - - Each entry (n, label) marks the interval starting at n and going - up to the next entry's n (exclusive). The last entry's interval - goes to +inf. - - An interval starting at n is valid iff label == delimiter. - - If the first entry's label != delimiter, everything before the first n is valid. - """ - if not lst: - # no markers -> everything valid - return lambda x: True - - # ensure sorted by number - lst_sorted = sorted(lst, key=lambda t: t[0]) - assert lst == lst_sorted - - nums = [t[0] for t in lst_sorted] - labels = [t[1] for t in lst_sorted] - - # Precompute intervals as (start, end_exclusive, is_valid) - intervals: list[tuple[int, int | None, bool]] = [] - n_items = len(nums) - - for i in range(n_items): - start = nums[i] - end_exclusive: int | None - - if i + 1 < n_items: - end_exclusive = nums[i + 1] - else: - end_exclusive = None # means to +inf - - is_valid = labels[i] == delimiter - intervals.append((start, end_exclusive, is_valid)) - - first_before_is_valid = labels[0] != delimiter - - def is_valid_fn(x: int) -> bool: - # before first number - if x < nums[0]: - return first_before_is_valid - - # find the interval that contains x - for start, end_exclusive, valid_flag in intervals: - if end_exclusive is None: - if x >= start: - return valid_flag - else: - if start <= x < end_exclusive: - return valid_flag - - # fallback (shouldn't happen) - return False - - return is_valid_fn - - -def is_always_valid_fn(addr: int): - return True - - -def add_lines_to_asm( - asm_path: Path, - stdump_json_path: Path, - fun_start_offset: int, - keep_original_numbers: bool, - asm_out: Path | None, -): - stdump_json = STDUMPJson(stdump_json_path) - - function_name = asm_path.stem - - if (fun := stdump_json.find_function(function_name)) is None: - raise RuntimeError(f"Cannot find function '{function_name}' in ccc's JSON") - - sub_source_files = fun.get("sub_source_files") - # print(len(sub_source_files) if sub_source_files is not None else None) - - asm_lines = asm_path.read_text().splitlines() - re_instr = re.compile(r"^\s*\/\* [A-Z0-9]+ ([A-Z0-9]{8}) [A-Z0-9]{8} \*\/ .*$") - line_dict: dict[int, int] = {} - - relative_path: str = fun["relative_path"] - non_func_addrs: list[int] = [] - - if sub_source_files: - checker = make_range_checker(sub_source_files, relative_path) - else: - checker = is_always_valid_fn - - start_line_num: int = sys.maxsize - - for line in asm_lines: - if m := re_instr.match(line): - instr_addr = int(m.group(1), 16) - line_num = stdump_json.get_line(instr_addr) - - if line_num: - line_dict[instr_addr] = line_num - - if not checker(instr_addr): - non_func_addrs.append(instr_addr) - elif line_num: - start_line_num = min(start_line_num, line_num) - - if start_line_num == sys.maxsize: - start_line_num = 1 - - min_line_num = 0 if keep_original_numbers else start_line_num - 1 - - new_asm_lines: list[str] = [] - asm_n: int = 0 - - for line in asm_lines: - if m := re_instr.match(line): - asm_n += 1 - - instr_addr = int(m.group(1), 16) - line_num = line_dict.get(instr_addr) - - if line_num is not None and instr_addr in non_func_addrs: - new_asm_lines.append(f" .loc 1 {line_num} # inline") - new_asm_lines.append(line) - continue - - if asm_n == 1 and line_num is None: - # sometimes we don't have a number for the first line of assembly, - # so we reuse the first known line among the ones we have - line_num = start_line_num - - if line_num is not None: - new_line_num = (line_num - min_line_num) + fun_start_offset - new_asm_lines.append(f" .loc 1 {new_line_num} ") - - new_asm_lines.append(line) - - stream = io.StringIO() - stream.write(""".section .debug -.previous -.text -.file 1 "source.c" - -.set noat -.set noreorder - -""") - stream.write("\n".join(new_asm_lines)) - - if asm_out is not None: - asm_out.write_text(stream.getvalue()) - else: - print(stream.getvalue()) - - -def main(): - class ArgProtocol(Protocol): - asm_path: Path - stdump_json_path: Path - asm_out: Path | None - offset: int - keep_original: bool - - parser = argparse.ArgumentParser(description="Add line debug info to assembly files") - parser.add_argument("--asm-path", required=True, type=Path, help="Path to the assembly file to add lines to") - parser.add_argument("--stdump-json-path", required=True, type=Path, help="Path to ccc's json stdump") - parser.add_argument("--asm-out", type=Path, required=False, help="Path to output asm (defaults to stdout)") - parser.add_argument( - "--offset", type=int, required=False, default=0, help="Offset to apply to line numbers (default: 0)" - ) - parser.add_argument( - "--keep-original", - action="store_true", - help="Don't start line numbers from 1 (+ offset) but keep the original line numbers (+offset) instead", - ) - - args = cast(ArgProtocol, parser.parse_args()) - - fun_start_num = args.offset - - add_lines_to_asm( - args.asm_path, - args.stdump_json_path, - fun_start_num, - args.keep_original, - args.asm_out, - ) - - -if __name__ == "__main__": - main() diff --git a/tools/python/ccc_json_v7.py b/tools/python/ccc_json_v7.py deleted file mode 100644 index a4c3bad..0000000 --- a/tools/python/ccc_json_v7.py +++ /dev/null @@ -1,351 +0,0 @@ -from __future__ import annotations - -import pydantic - -from typing import Optional, Literal, Any - - -class AddressRange(pydantic.BaseModel): - low: int - high: int - - -class ValueType(pydantic.BaseModel): - descriptor: Optional[str] = None - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[ValueType] = None - return_type: Optional[ReturnType] = None - modifier: Optional[str] = None - vtable_index: Optional[int] = None - is_constructor: Optional[bool] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - - -class DeduplicatedTypeValueType(pydantic.BaseModel): - descriptor: Literal["function_type", "type_name"] - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[ValueType] = None - return_type: Optional[ReturnType] = None - modifier: Optional[str] = None - vtable_index: Optional[int] = None - is_constructor: Optional[bool] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - - -class ParameterType(pydantic.BaseModel): - descriptor: str - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[ValueType] = None - - -class Parameter(pydantic.BaseModel): - descriptor: str - name: str - class_: str = pydantic.Field(..., alias="class") - storage: Storage - block_low: int - block_high: int - type: ParameterType - - -class ReturnType(pydantic.BaseModel): - descriptor: str - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[ValueType] = None - - -class FunctionType(pydantic.BaseModel): - descriptor: str - parameters: list[Parameter] - modifier: str - vtable_index: int - is_constructor: bool - return_type: Optional[ReturnType] = None - - -class Storage(pydantic.BaseModel): - type: str - register_: Optional[str] = pydantic.Field(None, alias="register") - register_class: Optional[str] = None - dbx_register_number: Optional[int] = None - register_index: Optional[int] = None - is_by_reference: Optional[bool] = None - stack_offset: Optional[int] = None - global_location: Optional[str] = None - global_address: Optional[int] = None - - -class Constant(pydantic.BaseModel): - value: int - name: str - - -class ElementType(pydantic.BaseModel): - descriptor: Literal["array", "pointer", "type_name", "enum"] - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - value_type: Optional[ValueType] = None - constants: Optional[list[Constant]] = None - - def parsed_size(self) -> tuple[int, str]: - if self.descriptor == "pointer": - return 1, "pointer" - - elif self.descriptor == "array": - assert self.element_type - assert self.element_count is not None - if self.element_count == 0: - # implicit size array - _, type_name = self.element_type.parsed_size() - return 0, type_name - n, type_name = self.element_type.parsed_size() - return self.element_count * n, type_name - - elif self.descriptor == "enum": - return 1, "enum" - - else: # "type_name" - assert self.type_name - return 1, self.type_name - - -class Local(pydantic.BaseModel): - descriptor: str - name: str - class_: str = pydantic.Field(..., alias="class") - storage: Storage - block_low: int - block_high: int - type: ElementType - storage_class: Optional[str] = None - - @property - def parsed_size(self) -> tuple[int, str]: - return self.type.parsed_size() - - -class SubSourceFile(pydantic.BaseModel): - address: int - path: str - - -class Function(pydantic.BaseModel): - descriptor: str - name: str - address_range: AddressRange - type: FunctionType - locals: list[Local] - line_numbers: list[list[int]] - sub_source_files: list[SubSourceFile] - storage_class: Optional[str] = None - relative_path: Optional[str] = None - - -class Global(pydantic.BaseModel): - descriptor: str - name: str - class_: str = pydantic.Field(..., alias="class") - storage: Storage - block_low: int - block_high: int - type: ElementType - storage_class: Optional[str] = None - - @property - def parsed_size(self) -> tuple[int, str]: - return self.type.parsed_size() - - -class File(pydantic.BaseModel): - descriptor: str - path: str - relative_path: str - text_address: int - types: list[Any] - functions: list[Function] - globals: list[Global] - stabs_type_number_to_deduplicated_type_index: dict[str, int] - - -class UnderlyingType(pydantic.BaseModel): - descriptor: str - source: str - type_name: str - referenced_file_index: int - referenced_stabs_type_number: int - - -class Field(pydantic.BaseModel): - descriptor: str - name: str - relative_offset_bytes: int - absolute_offset_bytes: int - size_bits: int - bitfield_offset_bits: Optional[int] = None - underlying_type: Optional[UnderlyingType] = None - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[ValueType] = None - base_classes: Optional[list[Any]] = None - fields: Optional[list[Field]] = None - member_functions: Optional[list[Any]] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - - -class FieldModel(pydantic.BaseModel): - descriptor: str - name: str - relative_offset_bytes: int - absolute_offset_bytes: int - size_bits: int - value_type: Optional[ValueType] = None - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - bitfield_offset_bits: Optional[int] = None - underlying_type: Optional[UnderlyingType] = None - fields: Optional[list[Field]] = None - member_functions: Optional[list[Any]] = None - base_classes: Optional[list[Any]] = None - - -class DeduplicatedType(pydantic.BaseModel): - descriptor: Literal["array", "builtin", "enum", "pointer", "struct", "type_name", "union"] - name: Optional[str] = None - storage_class: Optional[Literal["typedef"]] = None - stabs_type_number: int - files: list[int] - class_: Optional[str] = pydantic.Field(None, alias="class") - size_bits: Optional[int] = None - base_classes: Optional[list[Any]] = None - fields: Optional[list[FieldModel]] = None - member_functions: Optional[list[Any]] = None - source: Optional[str] = None - type_name: Optional[str] = None - referenced_file_index: Optional[int] = None - referenced_stabs_type_number: Optional[int] = None - value_type: Optional[DeduplicatedTypeValueType] = None - conflict: Optional[bool] = None - element_type: Optional[ElementType] = None - element_count: Optional[int] = None - constants: Optional[list[Constant]] = None - - -class CCCJSONv7Model(pydantic.BaseModel): - version: Literal[7] - files: list[File] - deduplicated_types: list[DeduplicatedType] - - -# def test(stdump_json_path: str): -# import json -# -# with open(stdump_json_path, mode="r") as fh: -# json_data = fh.read() -# -# model = CCCJSONv7Model.model_validate_json(json_data) -# -# type_map: dict[str, int] = {} -# -# for n, dt in enumerate(model.deduplicated_types): -# if dt.descriptor == "builtin": -# assert dt.name and dt.name not in type_map -# assert dt.class_ is not None -# size_bits = int(dt.class_.split("-", maxsplit=1)[0]) -# assert size_bits % 8 == 0 -# type_map[dt.name] = size_bits // 8 -# -# elif dt.descriptor == "type_name": -# assert dt.name is not None -# assert dt.type_name is not None -# if dt.name in type_map: -# if dt.size_bits: -# assert type_map[dt.name] == dt.size_bits // 8 -# if dt.name == dt.type_name == "void": -# type_map["void"] = type_map["int"] -# continue -# assert dt.type_name in type_map, (n, dt.type_name) -# type_map[dt.name] = type_map[dt.type_name] -# -# elif dt.descriptor == "pointer": -# assert dt.name is not None -# assert dt.value_type is not None -# if dt.value_type.descriptor == "function_type": -# assert dt.name not in type_map -# type_map[dt.name] = type_map["int"] -# elif dt.value_type.descriptor == "type_name": -# assert dt.value_type.type_name -# assert dt.value_type.type_name in type_map -# type_map[dt.name] = type_map[dt.value_type.type_name] -# -# elif dt.descriptor == "struct": -# assert dt.name is not None -# if not dt.conflict: -# assert dt.name not in type_map, n -# else: -# if dt.name in type_map: -# continue -# assert dt.size_bits is not None -# assert dt.size_bits % 8 == 0 -# type_map[dt.name] = dt.size_bits // 8 -# -# elif dt.descriptor == "array": -# assert dt.name is not None -# element_count = 1 -# element_type = dt.element_type -# type_name = None -# while element_type: -# if element_type.element_count is not None: -# element_count *= element_type.element_count -# if element_type.type_name is not None: -# type_name = element_type.type_name -# assert type_name in type_map -# element_count *= type_map[type_name] -# element_type = element_type.element_type -# assert type_name and type_name in type_map, n -# type_map[dt.name] = type_map[type_name] * element_count -# -# elif dt.descriptor == "enum": -# if dt.name: -# if not dt.conflict: -# assert dt.name not in type_map, n -# type_map[dt.name] = type_map["int"] -# -# elif dt.descriptor == "union": -# assert dt.name -# assert dt.size_bits -# assert dt.size_bits % 8 == 0 -# type_map[dt.name] = dt.size_bits // 8 -# -# else: -# assert False, f"unknown {n}" -# -# print(json.dumps(type_map, indent=2)) - - -# if __name__ == "__main__": -# test(path-to-stdump-json) diff --git a/tools/python/cstruct.py b/tools/python/cstruct.py deleted file mode 100644 index c5f773d..0000000 --- a/tools/python/cstruct.py +++ /dev/null @@ -1,322 +0,0 @@ -from __future__ import annotations - -import io -import sys -import math -import ctypes -import numpy as np -import numpy.typing as npt -from collections.abc import Sized, Sequence -from typing import Type, TypeVar, Any, cast, TextIO, BinaryIO, Optional -from ctypes import LittleEndianStructure, Union, Array, c_uint32 - -_G = TypeVar("_G") - -CTypeType = ( - type[ctypes.c_int8] - | type[ctypes.c_uint8] - | type[ctypes.c_int16] - | type[ctypes.c_uint16] - | type[ctypes.c_int32] - | type[ctypes.c_uint32] - | type[ctypes.c_float] - | type[ctypes.c_double] -) - -ctypes_types: dict[str, CTypeType] = { - "char": ctypes.c_int8, - "u_char": ctypes.c_uint8, - "short": ctypes.c_int16, - "u_short": ctypes.c_uint16, - "int": ctypes.c_int32, - "u_int": ctypes.c_uint32, - "float": ctypes.c_float, - "double": ctypes.c_double, -} - - -class c_addr(c_uint32): - def __str__(self): - # if self.value == 0: - # return "NULL" - - # return f"0x{self.value:08x}" - return f"0x{self.value:x}" - - -class c_str(c_uint32): - def to_str(self, elf: BinaryIO): - if self.value == 0: - return "NULL" - elf.seek(self.value) - buf = io.BytesIO() - while (c := elf.read(1)) != b"\0": - buf.write(c) - return '"' + buf.getvalue().decode("ASCII") + '"' - - -class c_addr_ptr(c_uint32): - _addresses: dict[int, str] | None = None - - @classmethod - def set_addresses(cls, addresses: dict[int, str] | None): - cls._addresses = addresses - - def __str__(self): - if self.value == 0: - return "NULL" - - if self._addresses and self.value in self._addresses: - return self._addresses[self.value] - - print(f"warning: no address for pointer 0x{self.value:x}") - - return f"0x{self.value:x}" - - -def print_arr(arr: npt.NDArray[np.int_], lst: list[Any], file: TextIO): - if arr.ndim > 1: - for x in arr: - file.write("{") - print_arr(x, lst, file) - file.write("},") - else: - file.write(",".join(str(lst[i]) for i in arr)) - - -def print_carr(arr: Sequence[Any], file: TextIO): - if hasattr(arr[0], "_length_"): - for a in arr: - file.write("{") - print_carr(a, file) - file.write("},") - else: - v = str(str([x for x in arr])).replace("[", "").replace("]", "") - file.write(v) - - -def chunks(lst: Sequence[_G], n: int): - """Yield successive n-sized chunks from lst.""" - for i in range(0, len(lst), n): - yield lst[i : i + n] - - -def format_array(lst: Sequence[_G], dims: Sequence[int], file: TextIO): - if len(dims) > 1: - for ll in chunks(lst, len(lst) // dims[0]): - file.write("{") - format_array(ll, dims[1:], file) - file.write("},") - else: - for n, x in enumerate(lst): - sep = ", " if n < len(lst) - 1 else "" - file.write(f"{x}{sep}") - - -def resolve_annotations(namespace: dict[str, Any], annotations: dict[str, Any]): - module = sys.modules.get(namespace.get("__module__", "")) - globals_ = vars(module) if module else {} - resolved: dict[str, Any] = {} - - for k, v in annotations.items(): - if isinstance(v, str): - try: - resolved[k] = eval(v, globals_, namespace) - except Exception: - resolved[k] = v # keep as string if can't resolve - else: - resolved[k] = v - - return resolved - - -class LittleEndianStructureFieldsFromTypeHints(type(LittleEndianStructure)): # pyright: ignore - def __new__( - cls: Type[type], - name: str, - bases: tuple[type, ...], - namespace: dict[str, Any], - /, - *, - align: Optional[int] = None, - pack: Optional[int] = None, - ) -> LittleEndianStructureFieldsFromTypeHints: - annotations = namespace.get("__annotations__", {}) - annotations = resolve_annotations(namespace, annotations) - if "__elf__" in annotations: - annotations.pop("__elf__") - if align is not None: - namespace["_align_"] = align - if pack is not None: - namespace["_pack_"] = pack - namespace["_layout_"] = "ms" - if fields := list(annotations.items()): - namespace["_fields_"] = fields - return type(LittleEndianStructure).__new__(cls, name, bases, namespace) # pyright: ignore - - -class CStructure(LittleEndianStructure, metaclass=LittleEndianStructureFieldsFromTypeHints): - __elf__: BinaryIO - - @classmethod - def sizeof(cls) -> int: - align = getattr(cls, "_align_", 0) - c_size = ctypes.sizeof(cls) - if align > 0: - rem = c_size % align - if rem: - rem = align - rem - return c_size + rem - - return c_size - - @classmethod - def parse(cls, data: bytes): - cstruct_size = sizeof(cls) - assert len(data) % cstruct_size == 0, f"{len(data)}, {cstruct_size}" - cstruct_num = len(data) // cstruct_size - stream = io.BytesIO(data) - cstructs = [cls.from_buffer_copy(stream.read(cstruct_size)) for _ in range(cstruct_num)] - return cstructs - - @classmethod - def dumps( - cls, - name: str, - data: bytes, - numel: int | list[int], - static: bool = False, - nosize: bool = False, - noarray: bool = False, - ): - cstructs = cls.parse(data) - stream = io.StringIO() - if static: - stream.write("static ") - stream.write(f"{cls.__name__} {name}") # pyright: ignore - assert not (noarray and isinstance(numel, list)) - if not noarray: - if isinstance(numel, int): - assert len(cstructs) == numel, (len(cstructs), numel) - numel_str = f"{len(cstructs)}" if not nosize else "" - stream.write(f"[{numel_str}]") - else: - numel_str = "".join([f"[{num if n > 0 or not nosize else ''}]" for n, num in enumerate(numel)]) - stream.write(numel_str) - tot_numel = max(1, numel) if isinstance(numel, int) else math.prod(numel) - # nmdim = 1 if isinstance(numel, int) else len(numel) - assert len(cstructs) == tot_numel, (len(cstructs), tot_numel) - - stream.write(" = ") - if not noarray: - stream.write("{\n") - if isinstance(numel, int): - for n, s in enumerate(cstructs): - is_last = n == len(cstructs) - 1 - if is_last and noarray: - stream.write(f"{s}\n") - else: - stream.write(f"{s},\n") - - else: - idxs = np.arange(tot_numel).reshape(numel) - print_arr(idxs, cstructs, stream) - if not noarray: - stream.write("}") - stream.write(";\n\n") - return stream.getvalue() - - # def to_str(self, elf: BinaryIO): - def __str__(self): - stream = io.StringIO() - stream.write(" {\n") - for f, *_ in self._fields_: # pyright: ignore - if f.startswith("_pad_"): - continue - v = getattr(self, f) # pyright: ignore - if f in ("_in", "_pass"): - f = f[1:] # pyright: ignore - if isinstance(v, c_str): - stream.write(f" .{f} = {v.to_str(self.__elf__)},\n") - elif not isinstance(v, Array): - stream.write(f" .{f} = {v},\n") - else: - arr = cast(Sized, v) - if len(arr) and isinstance(v[0], CStructure): - arr = cast(list[CStructure], arr) - stream.write(f" .{f} = {{\n") - for elem in arr: - stream.write(f" {elem},\n") - stream.write(" },\n") - else: - arr = cast(Sequence[Any], arr) - if isinstance(arr[0], Array): - # multidimensional ctypes array - stream.write(f" .{f} = {{") - print_carr(arr, stream) - stream.write("},\n") - else: - stream.write(f" .{f} = {{") - dims = [len(arr)] - format_array(arr, dims, stream) - - stream.write("},\n") - stream.write(" }") - return stream.getvalue() - - -class UnionFieldsFromTypeHints(type(Union)): # pyright: ignore - def __new__( - cls: Type[type], - name: str, - bases: tuple[type, ...], - namespace: dict[str, Any], - /, - *, - align: Optional[int] = None, - pack: Optional[int] = None, - ) -> UnionFieldsFromTypeHints: - annotations = namespace.get("__annotations__", {}) - if align is not None: - namespace["_align_"] = align - if pack is not None: - namespace["_pack_"] = pack - if fields := list(annotations.items()): - namespace["_fields_"] = fields - return type(Union).__new__(cls, name, bases, namespace) # pyright: ignore - - -class CUnion(Union, metaclass=UnionFieldsFromTypeHints): - pass - - -_T = TypeVar("_T", bound=CStructure) - - -def sizeof(cstruct_type: Type[_T]) -> int: - align = getattr(cstruct_type, "_align_", 0) - c_size = ctypes.sizeof(cstruct_type) - if align > 0: - rem = c_size % align - if rem: - rem = align - rem - return c_size + rem - - return c_size - - -def parse_cstruct(cstruct_type: Type[_T], data: bytes) -> list[_T]: - cstruct_size = sizeof(cstruct_type) - assert len(data) % cstruct_size == 0, f"{len(data)}, {cstruct_size}" - cstruct_num = len(data) // cstruct_size - stream = io.BytesIO(data) - cstructs = [cstruct_type.from_buffer_copy(stream.read(cstruct_size)) for _ in range(cstruct_num)] - return cstructs - - -def print_cstruct(name: str, cstruct_type: Type[CStructure], data: bytes): - cstructs = parse_cstruct(cstruct_type, data) - print(f"{cstruct_type.__name__} {name}[{len(cstructs)}] = {{") # pyright: ignore - for s in cstructs: - print(s) - print("};\n") diff --git a/tools/python/fix_asm_for_m2c.py b/tools/python/fix_asm_for_m2c.py deleted file mode 100644 index b635918..0000000 --- a/tools/python/fix_asm_for_m2c.py +++ /dev/null @@ -1,56 +0,0 @@ -import argparse - - -registers = { - "$0": "zero", - "$1": "at", - "$2": "v0", - "$3": "v1", - "$4": "a0", - "$5": "a1", - "$6": "a2", - "$7": "a3", - "$8": "t0", - "$9": "t1", - "$10": "t2", - "$11": "t3", - "$12": "t4", - "$13": "t5", - "$14": "t6", - "$15": "t7", - "$16": "s0", - "$17": "s1", - "$18": "s2", - "$19": "s3", - "$20": "s4", - "$21": "s5", - "$22": "s6", - "$23": "s7", - "$24": "t8", - "$25": "t9", - "$26": "k0", - "$27": "k1", - "$28": "gp", - "$29": "sp", - "$30": "fp", - "$31": "ra", -} - - -def fix_asm(asm_file: str): - with open(asm_file, mode="r") as fh: - for line in fh: - for reg_num, reg_mnem in reversed(registers.items()): - line = line.replace(reg_num, reg_mnem) - print(line, end="") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("asm_file", help="source assembly file path") - args = parser.parse_args() - fix_asm(args.asm_file) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_asm_matching.py b/tools/python/fix_asm_matching.py deleted file mode 100644 index b8c41ab..0000000 --- a/tools/python/fix_asm_matching.py +++ /dev/null @@ -1,125 +0,0 @@ -import os -import re -import argparse -import pydantic - -from pathlib import Path -from typing import Protocol, cast - - -CONFIG_ROOT = Path(__file__).parent.parent.parent.resolve() / "config" - -# enumerate languages from config dir -LANGUAGES = [Path(f.path).name for f in os.scandir(CONFIG_ROOT) if f.is_dir()] - -RE_INSTR = re.compile(r"^\t(?:[^.]|\.p2align \d+)") - - -class InstructionPatch(pydantic.RootModel[tuple[int, str, str]]): - root: tuple[int, str, str] - - @property - def instr_no(self): - return self.root[0] - - @property - def instr_org(self): - return self.root[1] - - @property - def instr_patch(self): - return self.root[2] - - -class ASMPatch(pydantic.RootModel[dict[str, list[InstructionPatch]]]): - root: dict[str, list[InstructionPatch]] - - -class PatchDB(pydantic.RootModel[dict[str, ASMPatch]]): - root: dict[str, ASMPatch] - - -def fix_asm(asm_file: Path, asm_patch: ASMPatch): - lines = asm_file.read_text().splitlines() - - def find_func(func: str): - for i, line in enumerate(lines): - if re.match(rf"^{func}:", line): - return i - - return -1 - - def find_line(start: int, num: int): - offset = 0 - instr_no = -1 - for line in lines[start:]: - if RE_INSTR.match(line): - instr_no += 1 - - if instr_no == num: - return start + offset - - offset += 1 - - raise RuntimeError("cannot find instruction!") - - for func, patch_lst in asm_patch.root.items(): - n = find_func(func) - - if n == -1: - print(f"WARNING: cannot find function {func} in {asm_file.name}") - continue - - for patch in patch_lst: - line_no = find_line(n, patch.instr_no) - line_org = lines[line_no] - - line_org_clean = re.sub(r"\s+", " ", line_org).strip() - instr_org_clean = re.sub(r"\s+", " ", patch.instr_org).strip() - instr_patch_clean = re.sub(r"\s+", " ", patch.instr_patch).strip() - - if line_org_clean != instr_org_clean: - print(f"WARNING: wrong line: {asm_file.name}:{func}:{line_no + 1}: {line_org} != {patch.instr_org}") - continue - - lines[line_no] = f"\t{instr_patch_clean}" - - asm_file.write_text("\n".join(lines)) - - -def main(): - class ArgsProtocol(Protocol): - language: str - asm_file: Path - - parser = argparse.ArgumentParser(description="apply asm patches to assembly files") - parser.add_argument("language", type=str, choices=LANGUAGES, help="language of the asm that is being patched") - parser.add_argument("asm_file", type=Path, help="generated assembly file to patch (relative to build dir)") - - args = cast(ArgsProtocol, parser.parse_args()) - - asm_file = CONFIG_ROOT / args.language / args.asm_file - - if not asm_file.exists(): - print(f"ERROR: cannot find assembly file {asm_file}") - exit(1) - - patch_db_path = CONFIG_ROOT / args.language / "asm_patches.json" - - if not patch_db_path.exists(): - print("ERROR: patch db file doesn't exist. aborting patching asm") - exit(1) - - patch_db = PatchDB.model_validate_json(patch_db_path.read_text()) - - if asm_file.name not in patch_db.root: - print(f"ERROR: no patches found for file {asm_file.name}") - exit(1) - - asm_patch = patch_db.root[asm_file.name] - - fix_asm(asm_file, asm_patch) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_assets.py b/tools/python/fix_assets.py deleted file mode 100644 index f172c67..0000000 --- a/tools/python/fix_assets.py +++ /dev/null @@ -1,38 +0,0 @@ -import re -import glob -import tqdm -import argparse - -from typing import Protocol, cast -from pathlib import Path - - -def fix_assets(asm_data_path: Path, asset_rel_path: Path): - asm_files = glob.glob(str(asm_data_path / "**/*.s"), recursive=True) - for asm_file in tqdm.tqdm(asm_files, desc="Fixing data asm"): - n: int = 0 - with open(asm_file, mode="r") as fh: - data_asm: str = fh.read() - data_asm, n = re.subn(rf'\.incbin "{asset_rel_path}/', '.incbin "assets/', data_asm) - - if n > 0: - with open(asm_file, mode="w") as wh: - wh.write(data_asm) - - -def main(): - class ArgsProtocol(Protocol): - asm_data_path: Path - asset_rel_path: Path - - parser = argparse.ArgumentParser(description="fixes data asm include path") - parser.add_argument("asm_path", metavar="asm-path", type=Path, help="data path in assembly root to patch") - parser.add_argument("asset_rel_path", metavar="asset-path", type=Path, help="relative asset path") - - args = cast(ArgsProtocol, parser.parse_args()) - - fix_assets(args.asm_data_path, args.asset_rel_path) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_bin_matching.py b/tools/python/fix_bin_matching.py deleted file mode 100644 index 5075bb6..0000000 --- a/tools/python/fix_bin_matching.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import os -import argparse -import pydantic - -from pathlib import Path -from typing import Protocol, cast - - -CONFIG_ROOT = Path(__file__).parent.parent.parent.resolve() / "config" - -# enumerate languages from config dir -LANGUAGES = [Path(f.path).name for f in os.scandir(CONFIG_ROOT) if f.is_dir()] - - -class BytePatch(pydantic.RootModel[tuple[int, int]]): - root: tuple[int, int] - - @property - def address(self): - return self.root[0] - - @property - def size(self): - return self.root[1] - - -class BINPatch(pydantic.RootModel[dict[str, BytePatch]]): - root: dict[str, BytePatch] - - -class PatchDB(pydantic.RootModel[dict[str, BINPatch]]): - root: dict[str, BINPatch] - - -def fix_elf(orig_elf_path: Path, built_elf_path: Path, bin_patch: BytePatch): - with orig_elf_path.open("rb") as fh: - fh.seek(bin_patch.address) - data = fh.read(bin_patch.size) - - with built_elf_path.open(mode="r+b") as wh: - wh.seek(bin_patch.address) - wh.write(data) - - -def main(): - class ArgsProtocol(Protocol): - language: str - elf_file: Path - - parser = argparse.ArgumentParser(description="apply asm patches to assembly files") - parser.add_argument("language", type=str, choices=LANGUAGES, help="language of the elf that is being patched") - parser.add_argument("elf_file", type=Path, help="elf file to patch (relative to build dir)") - - args = cast(ArgsProtocol, parser.parse_args()) - - built_elf_file = CONFIG_ROOT / args.language / args.elf_file - - if not built_elf_file.exists(): - print(f"ERROR: cannot find elf file {built_elf_file}") - exit(1) - - orig_elf_path = CONFIG_ROOT / args.language / args.elf_file.name - - patch_db_path = CONFIG_ROOT / args.language / "bin_patches.json" - - if not patch_db_path.exists(): - exit(0) - - patch_db = PatchDB.model_validate_json(patch_db_path.read_text()) - - for _tu_name, bin_patch in patch_db.root.items(): - for _func_name, byte_patch in bin_patch.root.items(): - fix_elf(orig_elf_path, built_elf_file, byte_patch) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_gp.py b/tools/python/fix_gp.py deleted file mode 100644 index 907bcf6..0000000 --- a/tools/python/fix_gp.py +++ /dev/null @@ -1,75 +0,0 @@ -import re -import glob -import tqdm -import argparse -import functools - -from typing import Protocol, cast -from pathlib import Path - - -@functools.lru_cache(maxsize=1024) # pyright: ignore[reportUntypedFunctionDecorator] -def get_symbol_address(symbol_addrs: Path, symbol: str): - if match := re.match(r"^ *([^+ ]+) *\+ *(0x[0-9a-fA-F]+)$", symbol): - symbol = match.group(1) - offset = int(match.group(2), 16) - else: - offset = 0 - - with open(symbol_addrs, mode="r") as fh: - for line in fh: - if line.startswith(f"{symbol} "): - addr = int(line.split("=")[1].split(";")[0].strip(), 16) - return addr + offset - - assert False, f"{symbol} not found" - - -def fix_gp(asm_path: Path, gp_value: int, symbol_addrs: Path): - if gp_value <= 0: - return - - asm_files = glob.glob(str(asm_path / "**/*.s"), recursive=True) - for asm_file in tqdm.tqdm(asm_files, desc="Fixing gp_rel"): - lines: list[str] = [] - with open(asm_file, mode="r") as fh: - for line in fh: - if match := re.match(r"^(.*)%gp_rel\(([^)]+)\)(.*)$", line): - # ol = line - instr_pre = match.group(1) - instr_post = match.group(3) - address_str = match.group(2) - if address_str.startswith("D_"): - address_str = address_str.replace("D_", "0x") - res = eval(address_str) - address = res - else: - address = get_symbol_address(symbol_addrs, address_str) - gp_rel = address - gp_value - line = f"{instr_pre}{hex(gp_rel)}{instr_post}\n" - lines.append(line) - with open(asm_file, mode="w") as wh: - wh.writelines(lines) - - -def main(): - class ArgsProtocol(Protocol): - asm_path: Path - gp_value: int - symbol_addrs: Path - - def hex_int(x: str): - return int(x, 16) - - parser = argparse.ArgumentParser(description="fixes asm removing gp_rel macro") - parser.add_argument("asm_path", metavar="asm-path", type=Path, help="assembly root path to patch") - parser.add_argument("gp_value", metavar="gp", type=hex_int, help="gp value in hex") - parser.add_argument("symbol_addrs", metavar="symbol-addrs", type=Path, help="path of symbol_addrs.txt") - - args = cast(ArgsProtocol, parser.parse_args()) - - fix_gp(args.asm_path, args.gp_value, args.symbol_addrs) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_linkerscript.py b/tools/python/fix_linkerscript.py deleted file mode 100644 index 952f537..0000000 --- a/tools/python/fix_linkerscript.py +++ /dev/null @@ -1,139 +0,0 @@ -import re -import sys -import yaml -import tqdm - -from typing import cast, Any -from pathlib import Path - - -# e.g.: build/src/main/main.c.o(.text); -# build/asm/data/rodata_4.rodata.s.o(.rodata); -# ... -# NOTE: also account for optional '/data/' in path as chunks of data -# not belonging to a c file are put into 'build/asm/data/' folder. -re_subsegment_line = re.compile( - r"^(?P +)build/(?:asm|src)(?:/data)?/(?P.*)\.[sc]\.o\(\.(?P
.+)\);$" -) - -# e.g.: .main 0x100000 : AT(main_ROM_START) SUBALIGN(2) -# .main_bss (NOLOAD) : SUBALIGN(4) -# ... -re_section_line = re.compile(r"^(?P +)\.(?P
[^ ]+) .* SUBALIGN\((?P[0-9]+)\)$") - - -def get_align(address: int): - return ( - 128 * ((address % 128) == 0) - or 64 * ((address % 64) == 0) - or 32 * ((address % 32) == 0) - or 16 * ((address % 16) == 0) - or 8 * ((address % 8) == 0) - or 4 * ((address % 4) == 0) - or 2 * ((address % 2) == 0) - or 1 - ) - - -def make_align_map(config: dict[str, Any]): - segments = cast(list[dict[str, Any] | list[Any]] | None, config["segments"]) - assert segments - main_segment = next( - (segment for segment in segments if isinstance(segment, dict) and segment.get("name") == "main"), None - ) - assert main_segment, "cannot find main segment" - - subsegments = cast(list[dict[str, Any] | list[Any]] | None, main_segment["subsegments"]) - assert subsegments, "cannot find main subsegments" - - align_map: dict[str, int] = {} - - for subsegment in subsegments: - if not isinstance(subsegment, dict): - continue - - s_type = cast(str | None, subsegment.get("type")) - vram = cast(int | None, subsegment.get("vram")) - name = cast(str | None, subsegment.get("name")) - if not s_type or not vram or not name: - continue - - if name.endswith("bin"): - continue - - align = get_align(vram) - - if not s_type.startswith("."): - name = f"{s_type}#{name}.{s_type}" - else: - name = f"{s_type[1:]}#{name}" - - align_map[name] = align - - return align_map - - -def fix_linkerscript(config: dict[str, Any], linkerscript_path: Path): - align_map = make_align_map(config) - - section_subalign = cast(dict[str, int], config["_section_subalign"]) - - line_count = 0 - with open(linkerscript_path, mode="r") as fh: - for line in fh: - line_count += 1 - - patched_lines: list[str] = [] - - with open(linkerscript_path, mode="r") as fh: - for line in tqdm.tqdm(fh, desc="Fixing linker script", total=line_count): - if match := re_subsegment_line.match(line): - indent = cast(str, match["indent"]) - name = cast(str, match["name"]) - section = cast(str, match["section"]) - - # force each subsegment in the following sections to have align 8 - if section == "text": - patched_lines.append(f"{indent}. = ALIGN(., 8);\n") - - key = f"{section}#{name}" - if align := align_map.get(key): - patched_lines.append(f"{indent}. = ALIGN(., {align});\n") - - if match := re_section_line.match(line): - indent = cast(str, match["indent"]) - section = cast(str, match["section"]) - subalign = cast(str, match["subalign"]) - - # force each section to have the subalign specified in the yaml - if section in section_subalign: - current_subalign = f"SUBALIGN({subalign})" - fixed_subalign = f"SUBALIGN({section_subalign[section]})" - line = line.replace(current_subalign, fixed_subalign) - - patched_lines.append(line) - - with open(linkerscript_path, mode="w") as fh: - fh.writelines(patched_lines) - - -def main(): - if len(sys.argv) != 3: - print("usage: fix_linkerscript.py CONFIG_YAML_PATH LINKERSCRIPT_PATH") - exit(1) - - config_path = Path(sys.argv[1]) - linkerscript_path = Path(sys.argv[2]) - - with open(config_path, mode="r") as fh: - try: - config = cast(dict[str, Any], yaml.safe_load(fh)) - except yaml.YAMLError as e: - print(e) - raise e - - fix_linkerscript(config, linkerscript_path) - - -if __name__ == "__main__": - main() diff --git a/tools/python/fix_report.py b/tools/python/fix_report.py deleted file mode 100644 index fdae4c4..0000000 --- a/tools/python/fix_report.py +++ /dev/null @@ -1,153 +0,0 @@ -import json -import argparse - -from typing import Protocol, Iterable, Any, cast -from pathlib import Path - - -def fix_unit(unit: dict[str, Any]): - # name: str = unit["name"] - fuzzy_match_percent: float | None = unit["measures"].get("fuzzy_match_percent") - - assert fuzzy_match_percent and 0 < fuzzy_match_percent < 100 - - text_section = next(section for section in unit["sections"] if section["name"] == ".text") - # text_size: int = int(text_section["size"]) - text_section_fuzzy_match_percent: float = text_section["fuzzy_match_percent"] - - if 0 < text_section_fuzzy_match_percent < 100: - # fix text section fuzzy match percent - text_section["fuzzy_match_percent"] = 100.0 - - text_fuzzy_match_percent: float = unit["measures"]["fuzzy_match_percent"] - - assert text_fuzzy_match_percent == fuzzy_match_percent - - functions = unit["functions"] - - total_code: int = int(unit["measures"]["total_code"]) - matched_code: int = int(unit["measures"]["matched_code"]) - matched_functions = unit["measures"]["matched_functions"] - - computed_total_code: int = 0 - computed_matched_code: int = 0 - computed_matched_functions: int = 0 - - duplicate_functions = ( - "Tim2CalcBufWidth__2", - "_ftoi0__2", - "ItemGetMain__2", - "setD3_CHCR__2", - "setD4_CHCR__2", - "setD4_CHCR__3", - "_fpadd_parts__2", - ) - - for function in functions: - function_size: int = int(function["size"]) - try: - function_fuzzy_match_percent: float = function["fuzzy_match_percent"] - except KeyError: - # fix known function duplicates - if function["name"] in duplicate_functions: - function["fuzzy_match_percent"] = 100.0 - function_fuzzy_match_percent = 100.0 - matched_code += function_size - matched_functions += 1 - else: - raise - - if function_fuzzy_match_percent == 100.0: - computed_matched_code += function_size - computed_matched_functions += 1 - - # fix function fuzzy match percent - function["fuzzy_match_percent"] = 100.0 - - computed_total_code += function_size - - assert total_code == computed_total_code - assert matched_code == computed_matched_code - assert matched_functions == computed_matched_functions - - # fix unit measures - unit["measures"]["fuzzy_match_percent"] = 100.0 - unit["measures"]["matched_code"] = unit["measures"]["total_code"] - unit["measures"]["matched_code_percent"] = 100.0 - unit["measures"]["matched_functions"] = unit["measures"]["total_functions"] - unit["measures"]["matched_functions_percent"] = 100.0 - - -def fix_report(report_path: Path): - report = json.loads(report_path.read_text()) - - units: Iterable[Any] = report["units"] - - computed_total_code: int = 0 - computed_matched_code: int = 0 - - computed_total_functions: int = 0 - computed_matched_functions: int = 0 - - total_code: int = int(report["measures"]["total_code"]) - total_functions: int = report["measures"]["total_functions"] - - for unit in units: - # name: str = unit["name"] - unit_total_code: int = int(unit["measures"]["total_code"]) - unit_total_functions: int = unit["measures"]["total_functions"] - fuzzy_match_percent: float | None = unit["measures"].get("fuzzy_match_percent") - - if fuzzy_match_percent and 0 < fuzzy_match_percent < 100: - fix_unit(unit) - - if fuzzy_match_percent and fuzzy_match_percent > 0: - computed_matched_code += unit_total_code - computed_matched_functions += unit_total_functions - - computed_total_functions += unit_total_functions - computed_total_code += unit_total_code - - assert total_code == computed_total_code - assert total_functions == computed_total_functions - - # fix report measures - report["measures"]["fuzzy_match_percent"] = 100.0 * computed_matched_code / computed_total_code - report["measures"]["matched_code"] = str(computed_matched_code) - report["measures"]["matched_code_percent"] = report["measures"]["fuzzy_match_percent"] - report["measures"]["matched_functions"] = computed_matched_functions - report["measures"]["matched_functions_percent"] = 100.0 * computed_matched_functions / computed_total_functions - - categories = report["categories"] - assert len(categories) == 1 - assert categories[0]["measures"]["total_code"] == report["measures"]["total_code"] - assert categories[0]["measures"]["total_units"] == report["measures"]["total_units"] - - categories[0]["measures"]["fuzzy_match_percent"] = report["measures"]["fuzzy_match_percent"] - categories[0]["measures"]["matched_code"] = report["measures"]["matched_code"] - categories[0]["measures"]["matched_code_percent"] = report["measures"]["matched_code_percent"] - categories[0]["measures"]["matched_functions"] = report["measures"]["matched_functions"] - categories[0]["measures"]["matched_functions_percent"] = report["measures"]["matched_functions_percent"] - - # /path/to/report.json -> /path/to/report_fixed.json - # fixed_report_path = report_path.with_name(f"{report_path.stem}_fixed{report_path.suffix}") - # fixed_report_path.write_text(json.dumps(report)) - report_path.write_text(json.dumps(report)) - - print(f"Wrote fixed report to {report_path}") - - -def main(): - class ArgsProtocol(Protocol): - report_path: Path - - parser = argparse.ArgumentParser(description="fixes objdiff report") - parser.add_argument("report_path", metavar="report-path", type=Path, help="path to the report generated by objdiff") - - args = cast(ArgsProtocol, parser.parse_args()) - - fix_report(args.report_path) - - -if __name__ == "__main__": - main() diff --git a/tools/python/map_mismatch.py b/tools/python/map_mismatch.py deleted file mode 100644 index 01241de..0000000 --- a/tools/python/map_mismatch.py +++ /dev/null @@ -1,32 +0,0 @@ -import re -import argparse - -from pathlib import Path - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--language", required=True, choices=["us", "eu"]) - - args = parser.parse_args() - - if args.language == "us": - map_path = Path("config/us/build/SLUS_203.88.map") - else: - map_path = Path("config/eu/build/SLES_508.21.map") - - with open(map_path, mode="r") as fh: - for n, line in enumerate(fh): - line = line.rstrip("\n") - if match := re.match(r"^\s*0x([0-9a-fA-F]+)\s+[^ ]+?([0-9a-fA-F]{6,8})\s*$", line): - addr = match.group(1) - label = match.group(2) - if addr.upper() != label.upper(): - print(f"{map_path}:{n+1} {line}") - return - - print("no mismatches found") - - -if __name__ == "__main__": - main() diff --git a/tools/python/parse_data.py b/tools/python/parse_data.py deleted file mode 100644 index 9d03864..0000000 --- a/tools/python/parse_data.py +++ /dev/null @@ -1,438 +0,0 @@ -# pyright: reportInvalidTypeForm=false -from __future__ import annotations - -import os -import re -import io -import enum -import math -import tqdm -import argparse -import pydantic -import subprocess - -from ctypes import c_uint16, c_int16, c_uint64, c_uint32, c_int32, c_uint8, c_float, sizeof as c_sizeof -from typing import Protocol, BinaryIO, Type, Iterable, cast -from pathlib import Path - -from cstruct import CStructure, ctypes_types, CTypeType, c_addr_ptr, c_str, print_carr - - -def vram2offset(vram: int): - return vram - 0xFF000 - - -sceVu0FVECTOR = c_float * 4 -sceVu0IVECTOR = c_int32 * 4 -sceVu0FMATRIX = sceVu0FVECTOR * 4 - -qword = c_int32 * 4 - - -class ANI_CODE(c_uint16): - def __str__(self): - return f"{self.value}" - - -class GS_REG(c_uint64): - def __str__(self): - return f"0x{self.value:x}" - - -class HEX_ADDR(c_uint32): - def __str__(self): - return f"0x{self.value:x}" - - -# CTypeTypeEX = CTypeType | type[sceVu0FVECTOR] | type[sceVu0IVECTOR] - - -########################################################################### - -# struct _MNG_STATE_ST { -# // Members -# int state; // offset 0x0, size 0x4 -# u_int flag; // offset 0x4, size 0x4 -# u_int calc_total_time; // offset 0x8, size 0x4 -# u_int total_time; // offset 0xC, size 0x4 -# u_int get_time; // offset 0x10, size 0x4 -# u_int dmy[3]; // offset 0x14, size 0xC -# }; -class MNG_STATE_ST(CStructure): - state: c_int32 - flag: c_uint32 - calc_total_time: c_uint32 - total_time: c_uint32 - get_time: c_uint32 - flag: c_uint32 - dmy: c_uint32 * 3 - - -# struct _TMP_MENU_MNG_ST { -# // Members -# struct _MNG_STATE_ST mng_state; // offset 0x0, size 0x20 -# int DrawState; // offset 0x20, size 0x4 -# u_char** pTmpMenuFileBuf; // offset 0x24, size 0x4 -# float draw_up_rate; // offset 0x28, size 0x4 -# u_char dmy[12]; // offset 0x2C, size 0xC -# }; -class TMP_MENU_MNG_ST(CStructure): - mng_state: MNG_STATE_ST - DrawState: c_int32 - pTmpMenuFileBuf: c_addr_ptr - draw_up_rate: c_float - dmy: c_uint8 * 12 - -########################################################################### - -elf_names: dict[str, str] = { - "us": "SLUS_203.88", - "eu": "SLES_508.21", -} - - -class VRamElf(BinaryIO): - def __init__(self, elf: BinaryIO): - self.elf = elf - - def read(self, n: int = -1): - return self.elf.read(n) - - def seek(self, offset: int, whence: int = 0): - return self.elf.seek(vram2offset(offset), whence) - - -# matches "var_name = 0x12345678; // attr1:val1 attr2:val2 ... attrN:valN" -line_pattern = re.compile(r"^([^\s=]+)\s*=\s*(0x[0-9a-fA-F]+);\s*\/\/((?:\s*[0-9a-zA-Z_]+(:?:[0-9a-zA-Z_\*]+)?)+)\s*$") -# matches multiple attributes in the form of attr:val or attr -attr_pattern = re.compile(r"([0-9a-zA-Z_]+)(?::([0-9a-zA-Z_\*]+))?") -# match address lines: "0xaabbccdd = &var" -addr_pattern = re.compile(r"^\s*(0x[0-9a-fA-F]+)\s*=\s*(.*)\s*;\s*(\/\/.*)?$") - - -class DataVar(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") - - _elf: VRamElf = pydantic.PrivateAttr() - - address: int - name: str - type: Type[CStructure] | CTypeType | type[sceVu0FVECTOR] | type[sceVu0IVECTOR] | type[qword] | type[c_str] - numel: int | list[int] = 0 - nosize: bool = False - static: bool = False - - num_ptr: int = 0 - - @pydantic.model_validator(mode="after") - def store_num_ptr(self): - # transfer class attribute to instance - if DataVar.num_ptr > 0: - self.num_ptr = DataVar.num_ptr - return self - - @pydantic.field_validator("type", mode="before") - @classmethod - def type_from_str( # pyright: ignore - cls, v: str | Type[CStructure] | CTypeType - ) -> Type[CStructure] | CTypeType | sceVu0FVECTOR | sceVu0IVECTOR | qword | ANI_CODE | GS_REG | HEX_ADDR: - if not isinstance(v, str): - return v - cls.num_ptr = v.count("*") # temporary store num_ptr in class attribute - v = v.rstrip("*") # remove pointer(s) - if v in ctypes_types: - return ctypes_types[v] - if v == "sceVu0FVECTOR": - return sceVu0FVECTOR - if v == "sceVu0IVECTOR": - return sceVu0IVECTOR - if v == "qword": - return qword - if v == "ANI_CODE": - return ANI_CODE - if v == "GS_REG": - return GS_REG - if v == "HEX_ADDR": - return HEX_ADDR - if v == "c_str": - return c_str - class_type = globals()[v] - if issubclass(class_type, CStructure): - return class_type - raise ValueError(f"{v} is unknown/not a valid type") - - def data_var_dumps(self, addresses: dict[int, str]): - self._elf.seek(self.address, os.SEEK_SET) - - if isinstance(self.numel, list): - numel = math.prod(self.numel) - ndims = len(self.numel) - else: - numel = max(1, self.numel) - ndims = 1 - if self.num_ptr > 0: - var_data = (c_addr_ptr * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(c_addr_ptr))) - if self.type == ANI_CODE: # pyright: ignore - type_str = "ANI_CODE" + ("*" * self.num_ptr) - if self.type == GS_REG: # pyright: ignore - type_str = "u_long" + ("*" * self.num_ptr) - if self.type == HEX_ADDR: # pyright: ignore - type_str = "u_int" + ("*" * self.num_ptr) - elif issubclass(self.type, CStructure): # pyright: ignore - type_str = self.type.__name__ + ("*" * self.num_ptr) - elif self.type == sceVu0FVECTOR: # pyright: ignore - type_str = "sceVu0FVECTOR" + ("*" * self.num_ptr) - elif self.type == sceVu0IVECTOR: # pyright: ignore - type_str = "sceVu0IVECTOR" + ("*" * self.num_ptr) - else: - type_str = next( - k - for k, v in ctypes_types.items() - if getattr(v, "_type_") == getattr(self.type, "_type_") # pyright: ignore - ) + ("*" * self.num_ptr) - stream = io.StringIO() - if self.static: - stream.write("static ") - stream.write(f"{type_str} {self.name}") - if numel > 1: - numel = f"{self.numel}" if not self.nosize else "" - var_str = f"{{ {', '.join(str(var) for var in cast(Iterable[c_addr_ptr], var_data))}, }}" # pyright: ignore - stream.write(f"[{numel}]") - else: - # should have been parsed as c_float_Array_4_Array_1 - assert len(var_data) == 1 and var_data[0].__class__ is c_addr_ptr - var_str = str(var_data[0]) - stream.write(f" = {var_str};") - elif issubclass(self.type, CStructure): # pyright: ignore - data = self._elf.read(numel * self.type.sizeof()) - - return self.type.dumps( - self.name, - data, - numel=self.numel, - static=self.static, - nosize=self.nosize, - noarray=self.numel == 0, - ) - elif self.type == sceVu0FVECTOR: # pyright: ignore[reportUnknownMemberType] - var_data = (sceVu0FVECTOR * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(sceVu0FVECTOR))) - type_str = "sceVu0FVECTOR" - stream = io.StringIO() - if self.static: - stream.write("static ") - stream.write(f"{type_str} {self.name}") - - def sceVu0FVECTOR_to_str(_v: sceVu0FVECTOR): # pyright: ignore - return f"{{ {', '.join(f'{flt}f' for flt in cast(Iterable[c_float], _v))} }}" - - if numel > 1: - numel = f"{self.numel}" if not self.nosize else "" - var_str = ( - f"{{ {', '.join(sceVu0FVECTOR_to_str(var) for var in cast(Iterable[sceVu0FVECTOR], var_data))}, }}" # pyright: ignore - ) - stream.write(f"[{numel}]") - else: - # should have been parsed as c_float_Array_4_Array_1 - assert len(var_data) == 1 and var_data[0].__class__ is sceVu0FVECTOR - var_str = sceVu0FVECTOR_to_str(var_data[0]) - stream.write(f" = {var_str};") - elif self.type == sceVu0IVECTOR: # pyright: ignore[reportUnknownMemberType] - var_data = (sceVu0IVECTOR * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(sceVu0IVECTOR))) - type_str = "sceVu0IVECTOR" - stream = io.StringIO() - if self.static: - stream.write("static ") - stream.write(f"{type_str} {self.name}") - - def sceVu0IVECTOR_to_str(_v: sceVu0IVECTOR): # pyright: ignore - return f"{{ {', '.join(f'{int32}' for int32 in cast(Iterable[c_int32], _v))} }}" - - if numel > 1: - numel = f"{self.numel}" if not self.nosize else "" - var_str = ( - f"{{ {', '.join(sceVu0IVECTOR_to_str(var) for var in cast(Iterable[sceVu0IVECTOR], var_data))}, }}" # pyright: ignore - ) - stream.write(f"[{numel}]") - else: - # should have been parsed as c_int_Array_4_Array_1 - assert len(var_data) == 1 and var_data[0].__class__ is sceVu0IVECTOR - var_str = sceVu0IVECTOR_to_str(var_data[0]) - stream.write(f" = {var_str};") - elif self.type == c_str: # pyright: ignore - assert numel > 1, "we only handle string arrays here, simple strings should be embedded in the source" - var_data = (self.type * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(self.type))) # pyright: ignore - stream = io.StringIO() - if self.static: - stream.write("static ") - numel = f"{self.numel}" if not self.nosize else "" - stream.write(f"char *{self.name}[{numel}] = {{") - for var in cast(Iterable[c_str], var_data): - stream.write(f"{var.to_str(self._elf)},") - stream.write("};") - else: - typ = self.type # pyright: ignore - if ndims == 1: - typ *= numel # pyright: ignore - elif isinstance(self.numel, list): - for n in reversed(self.numel): - typ *= n # pyright: ignore - var_data = (typ).from_buffer_copy(self._elf.read(numel * c_sizeof(self.type))) # pyright: ignore - if self.type.__name__ == "ANI_CODE": # pyright: ignore - type_str = "ANI_CODE" - elif self.type.__name__ == "GS_REG": # pyright: ignore - type_str = "u_long" - elif self.type.__name__ == "HEX_ADDR": # pyright: ignore - type_str = "u_int" - else: - type_str = next( - k - for k, v in ctypes_types.items() - if getattr(v, "_type_") == getattr(self.type, "_type_") # pyright: ignore - ) - stream = io.StringIO() - if self.static: - stream.write("static ") - stream.write(f"{type_str} {self.name}") - if ndims == 1: - numel = f"{self.numel}" if not self.nosize else "" - var_str = f"{{ {', '.join(f'{var}' for var in cast(Iterable[CTypeType], var_data))} }}" - stream.write(f"[{numel}]") - elif ndims > 1: - assert isinstance(self.numel, list) - numel = "" - for n, num in enumerate(self.numel): - numel += "[]" if n == 0 and self.nosize else f"[{num}]" - stream.write(numel) - tmpbuf = io.StringIO() - tmpbuf.write("{") - print_carr(var_data, tmpbuf) # pyright: ignore - tmpbuf.write("}") - var_str = tmpbuf.getvalue() - else: - var_str = f"{var_data.value}" # pyright: ignore - stream.write(f" = {var_str};") - return stream.getvalue() - - -def parse_data_vars(data_vars_txt: Path, strict: bool = True): - data_vars: list[DataVar] = [] - addr_vals: dict[int, str] = {} - - with open(data_vars_txt, mode="r") as fh: - for n, line in enumerate(fh): - line = line.strip() - - # skip empty lines and comments - if not line or line.startswith("//"): - continue - - valid = False - - if addr_match := addr_pattern.match(line): - address = int(addr_match.group(1), 16) - value = addr_match.group(2) - addr_vals[address] = value - valid = True - - elif line_match := line_pattern.match(line): - address = int(line_match.group(2), 16) - options = line_match.group(3) - name = line_match.group(1) - if attr_match := attr_pattern.findall(options): - attrs: dict[str, str | list[str] | bool] = {} - for attr_name, attr_val in cast(list[tuple[str, str]], attr_match): - if attr_name not in attrs: - attrs[attr_name] = attr_val - elif not isinstance(attrs[attr_name], list): - attrs[attr_name] = cast(list[str], [attrs[attr_name], attr_val]) - else: - cast(list[str], attrs[attr_name]).append(attr_val) - for key, value in attrs.items(): - if not isinstance(value, bool) and len(value) == 0: - attrs[key] = True - try: - data_var = DataVar(address=address, name=name, **attrs) # pyright: ignore[reportArgumentType] - data_vars.append(data_var) - if address not in addr_vals: - if ( - isinstance(data_var.numel, list) or data_var.numel > 0 or data_var.type == sceVu0FVECTOR # type: ignore - ): - addr_vals[address] = name - else: - addr_vals[address] = f"&{name}" - valid = True - except pydantic.ValidationError as ve: - print(f"syntax error at line {n}") - raise ve - - if strict and not valid: - raise ValueError(f"syntax error at line {n}: {line}") - - return data_vars, addr_vals - - -def parse_data(lang: str): - data_vars_txt = Path("config") / lang / "data_vars.txt" - - data_vars, addr_vals = parse_data_vars(data_vars_txt) - - if not data_vars: - return - - if len(set(data_var.name for data_var in data_vars)) != len(data_vars): - raise RuntimeError("duplicate names in data_vars.txt") - - elf_path = Path("config") / lang / elf_names[lang] - - include_path = Path("config") / lang / "include" / "data" - include_path.mkdir(parents=True, exist_ok=True) - - c_addr_ptr.set_addresses(addr_vals) - - extract_filter: list[str] = [] - extract_filter_str = os.environ.get("EXTRACT_DATA_FILTER", "").strip() - if extract_filter_str: - extract_filter: list[str] = extract_filter_str.split(",") - - with open(elf_path, mode="rb") as elf: - vram_elf = VRamElf(elf) - CStructure.__elf__ = vram_elf - - for data_var in tqdm.tqdm(data_vars, desc="Extracting data"): - header_path = include_path / f"{data_var.name}.h" - - if extract_filter and data_var.name not in extract_filter: - continue - - with header_path.open(mode="w") as fw: - data_var._elf = vram_elf # pyright: ignore[reportPrivateUsage] - - fw.write(data_var.data_var_dumps(addr_vals)) - - try: - subprocess.run(["clang-format", "-i", "--style=Microsoft", header_path]) - except Exception: - # formatting is best effort for now - pass - - -def main(): - class Language(str, enum.Enum): - US = "us" - EU = "eu" - - languages = [lang.value for lang in Language] - - class ArgsProtocol(Protocol): - language: Language - - parser = argparse.ArgumentParser(description="fixes asm removing gp_rel macro") - parser.add_argument("language", type=str, choices=languages, help="language to extract data for") - - args = cast(ArgsProtocol, parser.parse_args()) - - parse_data(args.language) - - -if __name__ == "__main__": - main() diff --git a/tools/python/parse_globals.py b/tools/python/parse_globals.py deleted file mode 100644 index 927c991..0000000 --- a/tools/python/parse_globals.py +++ /dev/null @@ -1,288 +0,0 @@ -# pyright: reportUnknownMemberType=false - -import re -import argparse - -from typing import cast -from pathlib import Path -from elftools.elf.elffile import ELFFile -from elftools.elf.sections import SymbolTableSection - - -""" -matches variable declaration: -/* SECTION ADDRESS */ VAR_TYPE NAME[NUMEL]; -""" -re_glob = re.compile( - r"^/\* (?P
.*?) (?P
.*?) \*/ (?P[^(]*) (?P.*?)(?:\[(?P.*?)\])?;" -) - -""" -matches structs (or unions) with no typedef: -struct NAME { // SIZE - ... -}; -""" -re_struct = re.compile( - r"^(?:struct|union) (?P.*?) \{ // (?P0x[0-9a-f]+)\n.*?^\};", flags=re.MULTILINE | re.DOTALL -) - -""" -matches structs (or unions) with typedef: -typedef struct { // SIZE - ... -} NAME; -""" -re_typedef_struct = re.compile( - r"^typedef (?:struct|union) \{ // (?P0x[0-9a-f]+)\n.*?^\} (?P.*?);", flags=re.MULTILINE | re.DOTALL -) - -sizes: dict[str, int] = { - "char": 1, - "u_char": 1, - "short": 2, - "short int": 2, - "u_short": 2, - "int": 4, - "u_int": 4, - "float": 4, - "u_long128": 8, - "sceVu0FMATRIX": 4 * 4 * 4, - "sceVu0FVECTOR": 4 * 4, - "sceSifClientData": 0x2C, -} - -command_script_keywords = ( - "VERSION", - "SECTIONS", - "ABSOLUTE", - "LOADADDR", - "ALIGN", - "DEFINED", - "NEXT", - "SIZEOF", - "SIZEOF_HEADERS", - "MAX", - "MIN", - "PHDRS", - "CREATE_OBJECT_SYMBOLS", - "BYTE", - "SHORT", - "LONG", - "SQUAD", - "FILL", - "BLOCK", - "NOLOAD", - "AT", - "OVERLAY", - "NOCROSSREFS", - "PT_NULL", - "PT_LOAD", - "PT_DYNAMIC", - "PT_INTERP", - "PT_NOTE", - "PT_SHLIB", - "PT_PHDR", - "ENTRY", - "FLOAT", - "NOFLOAT", - "FORCE_COMMON_ALLOCATION", - "INCLUDE", - "INPUT", - "GROUP", - "OUTPUT", - "OUTPUT_ARCH", - "OUTPUT_FORMAT", - "SEARCH_DIR", - "STARTUP", - "TARGET", - "NOCROSSREFS", -) - - -class GlobalVarLineMatch: - max_section_len: int = 0 - max_address_len: int = 0 - max_var_type_len: int = 0 - max_name_len: int = 0 - max_numel_len: int = 0 - - _name_cache: list[str] = [] - - @classmethod - def _get_unique_name(cls, name: str): - i = 1 - unique_name = f"{name}__local_{i}" - while unique_name in cls._name_cache: - i += 1 - unique_name = f"{name}__local_{i}" - cls._name_cache.append(unique_name) - return unique_name - - def __init__( - self, - symtab: SymbolTableSection, - /, - *, - section: str, - address: str | int, - var_type: str, - name: str, - numel: str | int | None = None, - ): - self.symtab: SymbolTableSection = symtab - - self.section: str = section - self.address: int = address if isinstance(address, int) else int(address, 16) - self.var_type: str = var_type - self.name: str = name - self.numel: int - - while self.name.startswith("*"): - self.name = self.name[1:] - self.var_type += "*" - - # resolve function pointers (e.g., 'void (*SpecialEventInitTbl[0])...') - if match := re.match(r"\(\*(.*?)\[.*?\]\).*", self.name): - self.name = match.group(1) - - self.name = self.name.replace("*", "") - - if numel is not None: - if isinstance(numel, int): - self.numel = numel - else: - prod_numel = 1 - numels = numel.split("][") - for n in numels: - prod_numel *= int(n) - self.numel = prod_numel - - if numel is None: - self.numel = 1 - - # get symbol - symbol = self.symtab.get_symbol_by_name(self.name) - - assert isinstance(symbol, list) - - symbol = next( - (sym for sym in symbol if sym.name == self.name and sym.entry["st_value"] == self.address), - None, - ) - - assert symbol - - self.symbol = symbol - - if not self.is_global: - self.name = self._get_unique_name(self.name) - - escaped_name_len = len(str(self.name)) - if self.name in command_script_keywords: - escaped_name_len += 2 - - GlobalVarLineMatch.max_section_len = max(GlobalVarLineMatch.max_section_len, len(str(self.section))) - GlobalVarLineMatch.max_address_len = max(GlobalVarLineMatch.max_address_len, len(str(self.address))) - GlobalVarLineMatch.max_var_type_len = max(GlobalVarLineMatch.max_var_type_len, len(str(self.var_type))) - GlobalVarLineMatch.max_name_len = max(GlobalVarLineMatch.max_name_len, escaped_name_len) - GlobalVarLineMatch.max_numel_len = max(GlobalVarLineMatch.max_numel_len, len(str(self.numel))) - - @property - def is_global(self): - return cast(str, self.symbol["st_info"]["bind"]) == "STB_GLOBAL" - - @property - def is_hidden(self): - return cast(str, self.symbol["st_other"]["visibility"]) == "STV_HIDDEN" - - @property - def size(self): - size = cast(int, self.symbol["st_size"]) - if size == 0: - if "*" in self.var_type: - size = 4 - elif self.var_type in sizes: - size = sizes[self.var_type] * self.numel - return size - - def to_string(self, as_linker_command_file: bool): - name = self.name if self.name not in command_script_keywords else f'"{self.name}"' - cls_str = f"{name:{GlobalVarLineMatch.max_name_len}s} = 0x{self.address:08x};" - - if not as_linker_command_file: - if self.size > 0: - cls_str = f"{cls_str} // size:0x{self.size:x}" - else: - cls_str = f"{cls_str} //0 {self.var_type} * {self.numel}" - - # cls_str += f" bind:{'global' if self.is_global else 'local'}" - # cls_str += f" visibility:{'hidden' if self.is_hidden else 'visible'}" - - return cls_str - - def __str__(self): - return self.to_string(as_linker_command_file=False) - - -def parse_globals(elf_path: Path, globals_path: Path, types_path: Path, as_linker_command_file: bool): - with open(elf_path, mode="rb") as fh: - elf = ELFFile(fh) - - # Find the symbol table. - symtab = elf.get_section_by_name(".symtab") - assert isinstance(symtab, SymbolTableSection) - - with open(types_path, mode="r") as f: - types_data = f.read() - - for struct_type, size_hex_str in re_struct.findall(types_data): - struct_type = cast(str, struct_type) - struct_size = int(cast(str, size_hex_str), 16) - # assert not (struct_type in sizes and sizes[struct_type] != struct_size) - sizes[struct_type] = struct_size - - for size_hex_str, struct_type in re_typedef_struct.findall(types_data): - struct_type = cast(str, struct_type) - struct_size = int(cast(str, size_hex_str), 16) - # assert not (struct_type in sizes and sizes[struct_type] != struct_size), ( - # struct_type, - # sizes[struct_type], - # struct_size, - # ) - sizes[struct_type] = struct_size - - if "tagSE_WRK" in sizes: - sizes["SE_WRK"] = sizes["tagSE_WRK"] - - with open(globals_path, mode="r") as f: - lines = [line.strip() for line in f.readlines() if line.startswith("/*") and line.strip().endswith(";")] - - matches = [GlobalVarLineMatch(symtab, **match.groupdict()) for line in lines if (match := re_glob.match(line))] - matches.sort(key=lambda match: match.address) - - for match in matches: - print(match.to_string(as_linker_command_file=as_linker_command_file)) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--language", required=True, choices=["us", "eu"]) - parser.add_argument("--as-linker-command-file", action="store_true") - - args = parser.parse_args() - - if args.language == "us": - elf_path = Path("config/us/SLUS_203.88") - globals_path = Path("ccc/SLUS_203.88/globals.h") - types_path = Path("ccc/SLUS_203.88/types.h") - else: - elf_path = Path("config/eu/SLES_508.21") - globals_path = Path("ccc/SLES_508.21/globals.h") - types_path = Path("ccc/SLES_508.21/types.h") - - parse_globals(elf_path, globals_path, types_path, args.as_linker_command_file) - - -if __name__ == "__main__": - main() diff --git a/tools/python/parse_symbols.py b/tools/python/parse_symbols.py deleted file mode 100644 index dd85505..0000000 --- a/tools/python/parse_symbols.py +++ /dev/null @@ -1,378 +0,0 @@ -# pyright: reportUnknownMemberType=false - -from __future__ import annotations - -import re -import json -import argparse -import itertools - -from typing import Protocol, TextIO, cast -from pathlib import Path -from elftools.elf.elffile import ELFFile -from elftools.elf.sections import SymbolTableSection, Symbol - -import ccc_json_v7 - -Range = tuple[int, int] - -command_script_keywords = ( - "VERSION", - "SECTIONS", - "ABSOLUTE", - "LOADADDR", - "ALIGN", - "DEFINED", - "NEXT", - "SIZEOF", - "SIZEOF_HEADERS", - "MAX", - "MIN", - "PHDRS", - "CREATE_OBJECT_SYMBOLS", - "BYTE", - "SHORT", - "LONG", - "SQUAD", - "FILL", - "BLOCK", - "NOLOAD", - "AT", - "OVERLAY", - "NOCROSSREFS", - "PT_NULL", - "PT_LOAD", - "PT_DYNAMIC", - "PT_INTERP", - "PT_NOTE", - "PT_SHLIB", - "PT_PHDR", - "ENTRY", - "FLOAT", - "NOFLOAT", - "FORCE_COMMON_ALLOCATION", - "INCLUDE", - "INPUT", - "GROUP", - "OUTPUT", - "OUTPUT_ARCH", - "OUTPUT_FORMAT", - "SEARCH_DIR", - "STARTUP", - "TARGET", - "NOCROSSREFS", -) - -skip_symbols = ( - "_fbss", - "_gp", -) - -size_exceptions = { - "dorcon": 0x1C, -} - - -def parse_types(ccc_model_v7: ccc_json_v7.CCCJSONv7Model): - type_map: dict[str, int] = {} - - for n, dt in enumerate(ccc_model_v7.deduplicated_types): - if dt.descriptor == "builtin": - assert dt.name and dt.name not in type_map - assert dt.class_ is not None - size_bits = int(dt.class_.split("-", maxsplit=1)[0]) - assert size_bits % 8 == 0 - type_map[dt.name] = size_bits // 8 - - elif dt.descriptor == "type_name": - assert dt.name is not None - assert dt.type_name is not None - if dt.name in type_map: - if dt.size_bits: - assert type_map[dt.name] == dt.size_bits // 8 - if dt.name == dt.type_name == "void": - type_map["void"] = type_map["int"] - continue - assert dt.type_name in type_map, (n, dt.type_name) - type_map[dt.name] = type_map[dt.type_name] - - elif dt.descriptor == "pointer": - assert dt.name is not None - assert dt.value_type is not None - if dt.value_type.descriptor == "function_type": - assert dt.name not in type_map - type_map[dt.name] = type_map["int"] - elif dt.value_type.descriptor == "type_name": - assert dt.value_type.type_name - assert dt.value_type.type_name in type_map - type_map[dt.name] = type_map[dt.value_type.type_name] - - elif dt.descriptor == "struct": - assert dt.name is not None - if not dt.conflict: - assert dt.name not in type_map, n - else: - if dt.name in type_map: - continue - assert dt.size_bits is not None - assert dt.size_bits % 8 == 0 - type_map[dt.name] = dt.size_bits // 8 - - elif dt.descriptor == "array": - assert dt.name is not None - element_count = 1 - element_type = dt.element_type - type_name = None - while element_type: - if element_type.element_count is not None: - element_count *= element_type.element_count - if element_type.type_name is not None: - type_name = element_type.type_name - assert type_name in type_map - element_count *= type_map[type_name] - element_type = element_type.element_type - assert type_name and type_name in type_map, n - type_map[dt.name] = type_map[type_name] * element_count - - elif dt.descriptor == "enum": - if dt.name: - if not dt.conflict: - assert dt.name not in type_map, n - type_map[dt.name] = type_map["int"] - - elif dt.descriptor == "union": - assert dt.name - assert dt.size_bits - assert dt.size_bits % 8 == 0 - type_map[dt.name] = dt.size_bits // 8 - - else: - assert False, f"unknown {n}" - - return type_map - - -def in_range(ranges: list[Range], address: int): - if not ranges: - return True - - in_range = any((ra[0] <= address <= ra[1]) for ra in ranges) - - return in_range - - -def parse_ccc_model_v7(ccc_model_v7: ccc_json_v7.CCCJSONv7Model, ranges: list[Range]): - type_sizes = parse_types(ccc_model_v7) - - if "pointer" not in type_sizes: - type_sizes["pointer"] = type_sizes["int"] - - static_locals: list[ccc_json_v7.Local] = [] - global_vars: list[ccc_json_v7.Global] = [] - - for file in ccc_model_v7.files: - for global_var in file.globals: - assert global_var.storage.global_address - if in_range(ranges, global_var.storage.global_address): - global_vars.append(global_var) - - for function in file.functions: - for local in function.locals: - if local.storage_class == "static": - assert local.storage.global_address - if in_range(ranges, local.storage.global_address): - static_locals.append(local) - - return type_sizes, global_vars, static_locals - - -class SymbolWithNoNameException(Exception): ... - - -class ParsedSymbol: - _name_map: dict[str, list[ParsedSymbol]] = {} - _max_name_len: int = 0 - - def __init__(self, symtab: SymbolTableSection, symbol: Symbol) -> None: - self.symtab = symtab - self.symbol = symbol - - if not self.name: - raise SymbolWithNoNameException - - if self.name not in ParsedSymbol._name_map: - ParsedSymbol._name_map[self.name] = [] - - ParsedSymbol._name_map[self.name].append(self) - ParsedSymbol._max_name_len = max(ParsedSymbol._max_name_len, len(self.name)) - - @property - def name(self) -> str: - return re.sub(r"^(.*?)(\.\d+)?$", r"\1", self.symbol.name) - - @property - def size(self) -> int: - return cast(int, self.symbol["st_size"]) - - @property - def address(self) -> int: - return cast(int, self.symbol.entry["st_value"]) - - def in_range(self, range: Range): - return range[0] <= self.address <= range[1] - - def to_undefined_syms(self) -> str: - homonyms = ParsedSymbol._name_map[self.name] - assert len(homonyms) > 0 - if len(homonyms) == 1: - assert homonyms[0] == self - name = self.name - else: - idx = homonyms.index(self) - name = f"{self.name}__local_{idx + 1}" - - name = name if name not in command_script_keywords else f'"{name}"' - - # name_len_fmt = ParsedSymbol._max_name_len + len("__local_9999") - name_len_fmt = 40 - - return f"{name:{name_len_fmt}s} = 0x{self.address:08x};" - - def to_symbol_addrs( - self, - type_sizes: dict[str, int], - global_vars: list[ccc_json_v7.Global], - static_locals: list[ccc_json_v7.Local], - ) -> str: - homonyms = ParsedSymbol._name_map[self.name] - assert len(homonyms) > 0 - if len(homonyms) == 1: - assert homonyms[0] == self - name = self.name - else: - idx = homonyms.index(self) - name = f"{self.name}__local_{idx + 1}" - - # name_len_fmt = ParsedSymbol._max_name_len + len("__local_9999") - name_len_fmt = 40 - - size = size_exceptions.get(self.name, self.size) - if size == 0: - var = next( - (global_ for global_ in global_vars if global_.storage.global_address == self.address), None - ) or next((local for local in static_locals if local.storage.global_address == self.address), None) - - if not var: - print(f"cannot find {self.name} with size 0 in json") - else: - element_count, type_name = var.parsed_size - type_size = type_sizes.get(type_name) - assert type_size is not None, type_name - size = element_count * type_size - - if not size: - print(f"size of {self.name} is also 0 using json") - - size_str = f"size:0x{size:x}" if size else "" - - return f"{name:{name_len_fmt}s} = 0x{self.address:08x}; // {size_str}" - - -def parse_symbols_safe(elf_path: Path, dest_path: Path, json_path: Path, ranges: list[Range]): - if not dest_path.is_dir(): - raise RuntimeError(f"{dest_path} is not a directory") - - symbol_addrs_path = dest_path / "symbols_addrs.txt" - undefined_syms_path = dest_path / "undefined_syms.txt" - - if symbol_addrs_path.exists() or undefined_syms_path.exists(): - raise RuntimeError("symbols_addrs.txt or undefined_syms.txt already exist in dest folder") - - with open(elf_path, mode="rb") as elf_fh, open(json_path, mode="r") as json_fh: - elf = ELFFile(elf_fh) - - json_data = json.load(json_fh) - ccc_model_v7 = ccc_json_v7.CCCJSONv7Model.model_validate(json_data) - type_sizes, global_vars, static_locals = parse_ccc_model_v7(ccc_model_v7, ranges) - - with open(symbol_addrs_path, mode="w") as symbol_addrs, open(undefined_syms_path, mode="w") as undefined_syms: - parse_symbols(elf, symbol_addrs, undefined_syms, ranges, type_sizes, global_vars, static_locals) - - -def parse_symbols( - elf: ELFFile, - symbol_addrs: TextIO, - undefined_syms: TextIO, - ranges: list[Range], - type_sizes: dict[str, int], - global_vars: list[ccc_json_v7.Global], - static_locals: list[ccc_json_v7.Local], -): - # find symbol table - symtab = elf.get_section_by_name(".symtab") - assert isinstance(symtab, SymbolTableSection) - - parsed_symbols: list[ParsedSymbol] = [] - - for symbol in symtab.iter_symbols(): - try: - parsed_symbol = ParsedSymbol(symtab, symbol) - if parsed_symbol.name in skip_symbols: - continue - parsed_symbols.append(parsed_symbol) - except SymbolWithNoNameException: - pass - - parsed_symbols.sort(key=lambda x: x.address) - - for parsed_symbol in parsed_symbols: - if ranges: - in_range = any(parsed_symbol.in_range(ra) for ra in ranges) - if not in_range: - continue - - parsed_symbol.address - symbol_addrs.write(parsed_symbol.to_symbol_addrs(type_sizes, global_vars, static_locals)) - symbol_addrs.write("\n") - - undefined_syms.write(parsed_symbol.to_undefined_syms()) - undefined_syms.write("\n") - - -def main(): - class ArgsProtocol(Protocol): - elf_path: Path - dest_path: Path - json_path: Path - ranges: list[Range] - - def range_type(arg: str) -> Range: - if m := re.match(r"^([0-9a-f]+)-([0-9a-f]+)$", arg): - range_start, range_end = int(m.group(1), 16), int(m.group(2), 16) - if not (range_end > range_start): - raise argparse.ArgumentTypeError("non monotonic range") - return range_start, range_end - raise argparse.ArgumentTypeError("invalid range") - - parser = argparse.ArgumentParser() - parser.add_argument("elf_path", metavar="ELF", type=Path, help="path to ELF file") - parser.add_argument("--json", dest="json_path", type=Path, help="path to CCC json (v7)") - parser.add_argument( - "--dest", - dest="dest_path", - type=Path, - required=True, - help="folder where symbol_addrs.txt and undefined_syms.txt will be created " - "(existing files will not be overwritten)", - ) - parser.add_argument("--range", action="append", dest="ranges", nargs="+", type=range_type) - - args = parser.parse_args() - args.ranges = list(itertools.chain(*args.ranges)) - args = cast(ArgsProtocol, args) - - parse_symbols_safe(args.elf_path, args.dest_path, args.json_path, args.ranges) - - -if __name__ == "__main__": - main()