objdiff stuff

.gitignore

@@ -17,6 +17,7 @@ config/**/tmp_*/
 config/**/objdiff.json
 # ignore elf files
 config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9]
+config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9].rom
 .idea/
 .vscode/
 SLUS_210.07

Makefile

@@ -112,9 +112,6 @@ build: ## Build project
 extract-data:  ## Extract variables from .data in config directory
 	@python3 tools/python/parse_data.py
  
-make-asm:  ## Create expected asm folder in config directory
-	@python3 configure.py config/kuon.yaml --make-asm
- 
 map-mismatch:  ## Check for mismatches in mapfile
 	@python3 tools/python/map_mismatch.py --language 
 

config/checksum.sha1

@@ -1 +1 @@
-7124b436f61ebbe5944e4b50b7d4c04475711c11  build/SLUS_210.07
+294a1d49a96589c9caf68837b8d914b7a1ba2ce1  build/SLUS_210.07

config/kuon.yaml

@@ -28,14 +28,6 @@ options:
     make_full_disasm_for_code: true
     asm_nonmatching_label_macro: ""
     create_bss_pads: False
-# custom field used by fix_linkerscript.py to fix linkerscript alignments
-_section_subalign:
-    elf_header: 8
-    main: 2
-    main_bss: 1
-    reg_info: 4
-    mips_debug: 4
-    rest: 4
 segments:
     - [0, databin, elf_header]
     - name: main
@@ -43,12 +35,6 @@ segments:
       start: 0x80
       vram: 0x100000
       bss_size: 0x80BB00 # size of .sbss + size of .bss
-      align: 1
-      subalign: 2 # safe subalign value. proper values are applied by fix_linkerscript.py where necessary
-      # text has align 8 but other sections may have a different alignment ...
-      # explicit align correction is performed by fix_linkerscript.py during
-      # project configuration. vram address is used, when reported, in non text
-      # sections to calculate the correct align value.
       subsegments:
         - { start: 0x000080, type: asmtu, name: sdk/sce/ee/crt0, ld_align_segment_start: 0x8 }
         - [0x0002B0, asmtu, sdk/sce/ee/libgraph/graph001]
@@ -249,10 +235,10 @@ segments:
         - [0x032110, asmtu, cs4/Seimei/src/Camera]
         - [0x034930, asmtu, cs4/Seimei/src/Debug]
         - [0x034D10, asmtu, cs4/Seimei/src/Draw]
-        - [0x038710, asmtu, cs4/Seimei/src/File]
+        - [0x038710, c, cs4/Seimei/src/File]
         - [0x038D00, asmtu, cs4/Seimei/src/Game]
         - [0x039A00, asmtu, cs4/Seimei/src/Load]
-        - [0x03A380, asmtu, cs4/Seimei/src/Main]
+        - [0x03A380, c, cs4/Seimei/src/Main]
         - [0x03AFE0, asmtu, cs4/Seimei/src/MathUtil]
         - [0x03BCE0, asmtu, cs4/Seimei/src/MdlLink]
         - [0x03BDC0, asmtu, cs4/Seimei/src/Mic]
@@ -283,7 +269,7 @@ segments:
         - [0x059C60, asmtu, cs4/Seimei/src/Dss]
         - [0x05ACB0, asmtu, cs4/Seimei/src/Damage]
         - [0x05B3B0, asmtu, cs4/Seimei/src/Hmd]
-        - [0x05E360, asmtu, cs4/Seimei/src/Binder]
+        - [0x05E360, c, cs4/Seimei/src/Binder]
         - [0x05E980, asmtu, cs4/Seimei/src/Ene]
         - [0x060DF0, asmtu, cs4/Seimei/src/EneAct]
         - [0x062F40, asmtu, cs4/Seimei/src/EneThk]
@@ -421,7 +407,7 @@ segments:
         - [0x105150, asmtu, cs4/Seimei/src/E044Act]
         - [0x105560, asmtu, cs4/Seimei/src/sfx/SfxFog]
         - [0x105A00, asmtu, cs4/Seimei/fslib/lib/fsspummng]
-        - [0x106520, asmtu, cs4/Seimei/src/Event]
+        - [0x106520, c, cs4/Seimei/src/Event]
         - [0x106A90, asmtu, cs4/Seimei/src/Event0031]
         - [0x107380, asmtu, cs4/Seimei/src/Event0051]
         - [0x107E90, asmtu, cs4/Seimei/src/MiniGame/sgh/sgh]

configure.py

@@ -5,23 +5,15 @@
 from __future__ import annotations
 
 import os
-import re
 import sys
 import json
-import yaml
-import copy
 import splat
 import shutil
 import argparse
-import tempfile
 import subprocess
 import ninja_syntax
 
-from dataclasses import dataclass, asdict
+from splat.segtypes.linker_entry import LinkerEntry
-#from utils import ensure_path_and_write, normalize_object_path, to_expected_path
-
-import splat.util.options as splat_options
-from splat.segtypes.linker_entry import LinkerEntry, clean_up_path
 from spimdisasm.common.CompilerConfig import compilerOptions
 
 from typing import Any, Union, Protocol, Literal, cast
@@ -30,16 +22,6 @@ from contextlib import contextmanager
 
 from splat.scripts import split
 from splat.util.conf import load as splat_load_yaml
-from splat.segtypes.linker_entry import LinkerEntry
-
-# from tools.python.fix_gp import fix_gp
-from tools.python.fix_assets import fix_assets
-from tools.python.fix_linkerscript import fix_linkerscript
-
-from sys import stdout
-from subprocess import run
-from dataclasses import dataclass
-from struct import unpack
 
 @contextmanager
 def suppress_stdout_stderr():
@@ -110,7 +92,7 @@ def make_compiler_cmd(config_dir: Path, src_path: Path, language: str):
 
 
 def exec_shell(command: list[str]) -> str:
-    ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    ret = subprocess.run(command, capture_output=True, text=True, check=False)
     return ret.stdout
 
 
@@ -196,6 +178,21 @@ def build_stuff(
 
         object_strs = [str(obj) for obj in object_paths]
 
+        expected_strs = []
+        for obj in object_paths:
+            if obj.name.endswith(".c.o"):
+                obj = obj.with_name(obj.name.removesuffix(".c.o") + ".o")
+            elif obj.name.endswith(".s.o"):
+                obj = obj.with_name(obj.name.removesuffix(".s.o") + ".o")
+    
+            if obj.parts[:2] == ("build", "asm"):
+                p = obj.relative_to("build/asm")
+            elif obj.parts[:2] == ("build", "src"):
+                p = obj.relative_to("build/src")
+            else:
+                p = obj
+            expected_strs.append(str(Path("build/expected") / p))
+
         for object_path in object_paths:
             if object_path.name == "elf_header.s.o":
                 continue
@@ -211,6 +208,14 @@ def build_stuff(
                 implicit_outputs=implicit_outputs,
             )
 
+            ninja.build(
+                outputs=expected_strs,
+                rule="as",
+                inputs=[str((Path("asm") / s.relative_to("../src") if s.parts[:2] == ("..", "src") else s).with_suffix(".s")) for s in src_paths],
+                variables=variables,
+                implicit_outputs=implicit_outputs,
+            )
+
     ninja = ninja_syntax.Writer(open(str(ROOT / config_dir / "build.ninja"), "w"), width=9999)
 
     # Rules
@@ -295,16 +300,15 @@ def build_stuff(
             else:
                 build(entry.object_path, entry.src_paths, "cc")
 
-        elif isinstance(seg, splat.segtypes.common.databin.CommonSegDatabin):
+        elif isinstance(
-            build(entry.object_path, entry.src_paths, "as")
+                    seg,
-
+                    (
-        elif isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin):
+                        splat.segtypes.common.databin.CommonSegDatabin,
-            build(entry.object_path, entry.src_paths, "as")
+                        splat.segtypes.common.rodatabin.CommonSegRodatabin,
-
+                        splat.segtypes.common.textbin.CommonSegTextbin,
-        elif isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin):
+                        splat.segtypes.common.sbss.CommonSegSbss,
-            build(entry.object_path, entry.src_paths, "as")
+                    ),
-
+                ):
-        elif isinstance(seg, splat.segtypes.common.sbss.CommonSegSbss):
             build(entry.object_path, entry.src_paths, "as")
 
         else:
@@ -338,288 +342,6 @@ def build_stuff(
         implicit=[elf_path],
     )
 
-
-def get_line_file_path(args: AnnotationArgs):
-    if args.line_file_path is not None:
-        return args.line_file_path
-
- #   if args.elf_path.name == SH2_SERIAL and "Event/stage" in args.asm_path.as_posix():
-#        return Path(f"{TOOLS}/alessatool/dwarf") / Path(args.asm_path.name).with_suffix(".line")
-
-    return None
-
-
-def line_has_vram_addr(line: str, addr_str: str) -> bool:
-    if addr_str not in line or "*/" not in line:
-        return False
-
-    return line.index("*/") > line.index(addr_str)
-
-
-def annotate_asm(args: AnnotationArgs):
-    with open(args.asm_path, "r") as asm_file:
-        asm_contents = asm_file.read()
-
-    asm_lines = asm_contents.splitlines()
-    asm_line_index = 0
-
-    vram_start = args.vram_start
-    vram_end = args.vram_end
-
-    if vram_start is None or vram_end is None:
-        vram_start, vram_end = find_vram_bounds(asm_lines)
-
-    line_file_path = get_line_file_path(args)
-
-    if line_file_path is None or not line_file_path.exists():
-        addresses = (f"0x{v:X}" for v in range(vram_start, vram_end, 0x4))
-        proc = run([args.addr2line_path, "-e", args.elf_path, *addresses], capture_output=True, encoding=args.encoding)
-        addr2line_output_lines = proc.stdout.splitlines()
-    else:
-        # parse the binary line number file.
-
-        # the format is a list of u_shorts, one per line number.
-        # there should be one line number per vram address, and each vram
-        # address should be exactly 4 bytes apart, mirroring how the addr2line
-        # output is formatted
-
-        with open(line_file_path, "rb") as line_file:
-            line_data = line_file.read()
-            line_numbers = unpack(f"<{len(line_data) // 2}H", line_data)
-            compile_unit = args.asm_path.with_suffix(".c").name
-            addr2line_output_lines = list(map(lambda n : to_addr2line_format(compile_unit, n), line_numbers))
-
-    main_tu_name = None
-    prev_tu_name = None
-    prev_line_number = -1
-    function_count = 0
-    is_in_function_label = False
-    current_vram_addr = vram_start
-    annotated_asm_lines = []
-
-    for addr_index in range(0, len(addr2line_output_lines) - 1):
-        line = addr2line_output_lines[addr_index]
-
-        if line.startswith("?"):
-            current_vram_addr += 0x4
-            continue
-
-        separator_index = line.rfind(":")
-        current_line_number = int(line[separator_index+1:])
-
-        if current_line_number == prev_line_number:
-            current_vram_addr += 0x4
-            continue
-
-        vram_addr_str = f"{current_vram_addr:X}"
-
-        current_tu_name = line[0:separator_index]
-        if current_tu_name != main_tu_name and current_tu_name != prev_tu_name:
-            if main_tu_name:
-                print(f"[warn] alessatool/annotate: address at {vram_addr_str} belongs to {current_tu_name}")
-            else:
-                main_tu_name = current_tu_name
-        prev_tu_name = current_tu_name
-
-        while True:
-            if asm_line_index >= len(asm_lines):
-                raise AssertionError(
-                    f"address 0x{current_vram_addr:X} not found in asm"
-                )
-
-            asm_line = asm_lines[asm_line_index]
-
-            if line_has_vram_addr(asm_line, vram_addr_str):
-                break
-
-            should_append_asm_line = True
-            if args.tu:
-                asm_line_trimmed = asm_line.strip()
-
-                # track when we go in & out of function symbols
-                if asm_line_trimmed.startswith(FUNCTION_SYMBOL_LABEL):
-                    function_count += 1
-                    is_in_function_label = True
-                    annotated_asm_lines.append(f"{UNIQUE_TEXT_SECTION_DIRECTIVE}{function_count}")
-                    annotated_asm_lines.append("")
-                elif asm_line_trimmed.startswith(END_FUNCTION_SYMBOL_LABEL):
-                    is_in_function_label = False
-
-                # remove `nop`s
-                if not is_in_function_label and asm_line_trimmed.endswith("nop"):
-                    should_append_asm_line = False
-                
-                # remove `macro.inc` include directive
-                elif asm_line_trimmed == INCLUDE_MACRO_INC_DIRECTIVE:
-                    should_append_asm_line = False
-
-            if should_append_asm_line:
-                annotated_asm_lines.append(asm_line)
-
-            asm_line_index += 1
-
-        annotated_asm_lines.append(f"\t.loc 1 {current_line_number}")    
-        annotated_asm_lines.append(asm_line)
-        asm_line_index += 1
-
-        prev_line_number = current_line_number
-        current_vram_addr += 0x4
-    
-    while asm_line_index < len(asm_lines) - 1:
-        annotated_asm_lines.append(asm_lines[asm_line_index])
-        asm_line_index += 1
-
-    assert main_tu_name, "no valid compilation unit found"
-    annotated_asm_lines = [
-        ".section .debug",
-        ".previous",
-        ".text",
-        f".file 1 \"{main_tu_name}\"",
-        *annotated_asm_lines
-    ]
-
-    append_final_new_line(annotated_asm_lines)
-    annotated_asm_contents = "\n".join(annotated_asm_lines)
-
-    if not args.stdout and args.out_path:
-        with open(args.out_path, "w") as out_file:
-            out_file.write(annotated_asm_contents)
-        if args.verbose:
-            print(f"alessatool/annotate: wrote asm to {args.out_path}")
-    else:
-        stdout.write(annotated_asm_contents)
-
-
-def append_final_new_line(lines: list[str]):
-    if lines[-1] != "":
-        lines.append("")
-
-
-def make_asm(config_path: Path, config: dict[str, Any]):
-    """
-    Extracts assembly for each function into 'expected/asm/' subfolder.
-
-    The extraction is done following these steps:
-        1. create temporary directory in project folder
-        2. copy yaml to temporary directory
-        3. modify yaml to have c subsegments instead of asm subsegments
-        4. remove data, bss, etc., subsegments
-        5. run splat on the new config yaml
-        6. splat extracts assembly for each function into '<temp_dir>/asm/nonmatchings/'
-        7. copy '<temp_dir>/asm/nonmatchings/' to '<project_dir>/expected/asm/'
-    """
-    with tempfile.TemporaryDirectory(dir=config_path, prefix="tmp_") as tmp_dir:
-        tmp_path = Path(tmp_dir)
-
-        yaml_path = tmp_path / "config.yaml"
-        asm_path = tmp_path / "asm" / "nonmatchings"
-        dst_path = tmp_path / ".." / "expected" / "asm"
-        dst_path = dst_path.resolve().relative_to(ROOT)
-
-        if dst_path.exists():
-            print(f"expected asm dir '{dst_path}' already exists")
-            return
-
-        config = copy.deepcopy(config)
-        config["options"]["target_path"] = "../" + config["options"]["target_path"]
-        config["options"]["asm_path"] = "asm"
-        config["options"]["src_path"] = "src"
-        config["options"]["build_path"] = "build"
-        config["options"]["asset_path"] = "assets"
-        config["options"]["undefined_funcs_auto_path"] = "../" + config["options"]["undefined_funcs_auto_path"]
-        config["options"]["undefined_syms_auto_path"] = "../" + config["options"]["undefined_syms_auto_path"]
-        config["options"]["symbol_addrs_path"] = "../" + config["options"]["symbol_addrs_path"]
-        config["options"]["extensions_path"] = "../" + config["options"]["extensions_path"]
-
-        new_segments: list[Any] = []
-        segments: list[Any] = config["segments"]
-        for segment in segments:
-            if isinstance(segment, list):
-                new_segments.append(segment)
-            elif isinstance(segment, dict) and segment["name"] == "main":
-                new_subsegments: list[Any] = []
-                subsegments = cast(list[Any], segment["subsegments"])
-                for subsegment in subsegments:
-                    if isinstance(subsegment, list):
-                        if subsegment[1] == "asm":
-                            subsegment[1] = "c"
-                        new_subsegments.append(subsegment)
-                    elif isinstance(subsegment, dict):
-                        subsegment["type"] = subsegment["type"].strip(".")
-                        if subsegment["type"] == "rodata":
-                            # splat now requires rodata to be always prefixed with a dot (.)
-                            # for the rodata migration to work properly
-                            subsegment["type"] = ".rodata"
-                        new_subsegments.append(subsegment)
-                segment["subsegments"] = new_subsegments
-                new_segments.append(segment)
-        config["segments"] = new_segments
-
-        def rename_locals(base_path: Path):
-            for asm_file in base_path.rglob("*.s"):
-                data = asm_file.read_text()
-                data = re.sub(r"__local_\d+", "", data)
-                asm_file.write_text(data)
-
-        with yaml_path.open(mode="w") as yaml_file:
-            yaml.dump(config, yaml_file, default_flow_style=False)
-
-        with suppress_stdout_stderr():
-            split.main([yaml_path], modes=["all"], verbose=False)
-
-        # remove '__local_#' from asm
-        rename_locals(asm_path)
-
-        dst_path.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copytree(asm_path, dst_path, dirs_exist_ok=True)
-
-        print(f"expected asm extracted to '{dst_path}'")
-
-        # make expected objs
-
-        for subseg in new_segments[1]["subsegments"]:
-            if isinstance(subseg, list) and subseg[1] == "c":
-                subseg[1] = "asm"
-                subseg[2] += ".c"
-
-        config["options"]["asm_jtbl_label_macro"] = "llabel"
-
-        with yaml_path.open(mode="w") as yaml_file:
-            yaml.dump(config, yaml_file, default_flow_style=False)
-
-        shutil.rmtree(tmp_path / "asm")
-        (tmp_path / ".splache").unlink()
-
-        with suppress_stdout_stderr():
-            split.main([yaml_path], modes=["all"], verbose=False)
-
-        # remove '__local_#' from asm
-        rename_locals(asm_path)
-
-        dst_path = dst_path.parent / "obj"
-        tmp_obj_path = tmp_path / "obj"
-        tmp_asm_dir = tmp_path / "asm"
-
-        cpp = Path("..", "..", "..", (Path("tools") / "cc" / COMPILER))
-
-        for asm_file in tmp_asm_dir.rglob("*.c.s"):
-            asm_file_rel = asm_file.relative_to(tmp_path)
-            obj_file_rel = Path("obj") / asm_file.relative_to(tmp_asm_dir).with_suffix(".o")
-            obj_file = tmp_obj_path / obj_file_rel.relative_to("obj")
-            obj_file.parent.mkdir(parents=True, exist_ok=True)
-            subprocess.run(
-                f"{cpp} -I../src -I../include -Iinclude -Iinclude/sdk/ee -I../.. '{asm_file_rel}' -o - | "
-                #f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | "
-                f"mips-linux-gnu-as -no-pad-sections -EL -march=5900 -mabi=eabi -I../include -o {obj_file_rel} {asm_file_rel}",
-                shell=True,
-                cwd=tmp_path,
-            )
-
-        shutil.copytree(tmp_obj_path, dst_path, dirs_exist_ok=True)
-
-        print(f"expected obj built to '{dst_path}'")
-
-
 def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], language: str):
     """
     Generate `objdiff.json` configuration from splat YAML config.
@@ -633,7 +355,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
     """
     segments: list[Any] = config["segments"]
 
-    tu_to_diff: list[tuple[Literal["asm", "c"], str]] = []
+    tu_to_diff: list[tuple[Literal["asmtu", "c"], str]] = []
 
     for segment in segments:
         if not (isinstance(segment, dict) and segment["name"] == "main"):
@@ -653,7 +375,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
             else:
                 raise RuntimeError("invalid subsegment type")
 
-            if subs_type in ("asm", "c"):
+            if subs_type in ("asmtu", "c"):
                 if subs_name in (
                     "crt0",
                     "main/glob",
@@ -678,7 +400,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
     units: list[dict[str, Any]] = []
 
     for tu_type, tu_name in tu_to_diff:
-        target_path = Path("expected", "obj", tu_name).with_suffix(".c.o")
+        target_path = Path("build", "expected", tu_name).with_suffix(".o")
 
         # since we only compile fully decompiled TUs, the
         # "c" type implies that the TU is complete
@@ -695,7 +417,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
             "name": tu_name,
             "target_path": str(target_path),
             "base_path": str(base_path) if base_path else None,
-            "metadata": {"progress_categories": [language]},
+            "metadata": {
+                "source_path": str(Path("..", "src", tu_name).with_suffix(".c")),
+                "progress_categories": [language]
+            },
         }
 
         if not base_path:
@@ -717,10 +442,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
 
     objdiff_json: dict[str, Any] = {
         "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json",
-        "custom_make": "true",
+        "custom_make": "ninja",
         "custom_args": [],
-        "build_target": False,
+        "build_target": True,
-        "build_base": False,
+        "build_base": True,
         "watch_patterns": [],
         "units": units,
         "progress_categories": progress_categories,
@@ -858,7 +583,7 @@ def main():
     cwd = Path(os.getcwd()).resolve()
     if not ROOT.samefile(cwd):
         print("ERROR: this script must be run from it's directory!")
-        exit(1)
+        sys.exit(1)
 
     compilerOptions["MWCCPS2"].value.bigAddendWorkaroundForMigratedFunctions = False
 
@@ -876,9 +601,9 @@ def main():
 
 
     if basename not in LANGUAGES:
-        supported_elfs = f"{set(f'{elf} ({lang})' for elf, lang in LANGUAGES.items())}".replace("'", "")
+        supported_elfs = f"{ {f'{elf} ({lang})' for elf, lang in LANGUAGES.items()} }".replace("'", "")
         print(f"unsupported game ELF. Supported versions are: {supported_elfs}")
-        exit(1)
+        sys.exit(1)
 
     language = LANGUAGES[basename]
 
@@ -886,10 +611,6 @@ def main():
         clean(config_dir, config)
         return
 
-    if args.make_asm:
-        make_asm(config_dir, config)
-        return
-
     if args.clean:
         clean(config_dir, config)
 
@@ -906,36 +627,12 @@ def main():
 
     generate_lcf()
 
-
-    # fix asset .incbin path from relative to root to relative to config dir:
-    #  e.g., change:
-    #     .incbin "config/assets/name.section.bin"
-    #  to:
-    #     .incbin "assets/name.section.bin"
-    fix_assets(asm_data_path, asset_rel_path)
-
     linker_entries = split.linker_writer.entries
 
     build_stuff(config_dir, split.config, linker_entries, language)
 
     write_permuter_settings(config_dir, src_path, language)
 
-    # # replace gp_rel assembler macro with explicit offset as the gcc used
-    # # to compile the code does not support it
-    # gp_value = split.config["options"]["gp_value"]
-    # symbol_addrs_path = Path(split.config["options"]["symbol_addrs_path"])
-    # asm_rel_path = (config_dir / asm_path).resolve().relative_to(ROOT)
-    # symbol_addrs_rel_path = (config_dir / symbol_addrs_path).resolve().relative_to(ROOT)
-    # assert asm_rel_path.is_dir(), f"{asm_rel_path} not found or not a directory"
-    # assert symbol_addrs_rel_path.is_file(), f"{symbol_addrs_rel_path} not found"
-    # fix_gp(asm_rel_path, gp_value, symbol_addrs_rel_path)
-
-    # fix linkerscript by applying explicit alignments as
-    # specified in the config yaml
-    linkerscript_path = (config_dir / f"{basename}.ld").resolve().relative_to(ROOT)
-    assert linkerscript_path.is_file(), f"{linkerscript_path} not found"
-    fix_linkerscript(split.config, linkerscript_path)
-
     generate_objdiff_configuration(config_dir, split.config, language)
 
 

objdiff.patch

@@ -0,0 +1,634 @@
+diff --git a/.gitignore b/.gitignore
+index 26dc792..4b8c8a3 100644
+--- a/.gitignore
++++ b/.gitignore
+@@ -17,6 +17,7 @@ config/**/tmp_*/
+ config/**/objdiff.json
+ # ignore elf files
+ config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9]
++config/**/[A-Z][A-Z][A-Z][A-Z]_[0-9][0-9][0-9].[0-9][0-9].rom
+ .idea/
+ .vscode/
+ SLUS_210.07
+diff --git a/Makefile b/Makefile
+index 260e6df..4a4a30d 100644
+--- a/Makefile
++++ b/Makefile
+@@ -96,7 +96,7 @@ help: $(WITH_ENV) ## Show this help
+ ##
+ ##  Commands:
+ configure: ## Configure  project (needs SLUS_210.07)
+-	@python3 configure.py config/kuon.yaml -c
++	@uv run configure.py config/kuon.yaml -c
+ 
+ .build-only:
+ 	@cd config/; \
+@@ -111,9 +111,6 @@ build: ## Build project
+ 
+ extract-data:  ## Extract variables from .data in config directory
+ 	@python3 tools/python/parse_data.py
+-
+-make-asm:  ## Create expected asm folder in config directory
+-	@python3 configure.py config/kuon.yaml --make-asm
+  
+ map-mismatch:  ## Check for mismatches in mapfile
+ 	@python3 tools/python/map_mismatch.py --language 
+diff --git a/config/checksum.sha1 b/config/checksum.sha1
+index f58e64e..3278653 100644
+--- a/config/checksum.sha1
++++ b/config/checksum.sha1
+@@ -1 +1 @@
+-7124b436f61ebbe5944e4b50b7d4c04475711c11  build/SLUS_210.07
++294a1d49a96589c9caf68837b8d914b7a1ba2ce1  build/SLUS_210.07
+diff --git a/config/kuon.yaml b/config/kuon.yaml
+index 63770ca..564ff60 100644
+--- a/config/kuon.yaml
++++ b/config/kuon.yaml
+@@ -28,14 +28,6 @@ options:
+     make_full_disasm_for_code: true
+     asm_nonmatching_label_macro: ""
+     create_bss_pads: False
+-# custom field used by fix_linkerscript.py to fix linkerscript alignments
+-_section_subalign:
+-    elf_header: 8
+-    main: 2
+-    main_bss: 1
+-    reg_info: 4
+-    mips_debug: 4
+-    rest: 4
+ segments:
+     - [0, databin, elf_header]
+     - name: main
+@@ -43,12 +35,6 @@ segments:
+       start: 0x80
+       vram: 0x100000
+       bss_size: 0x80BB00 # size of .sbss + size of .bss
+-      align: 1
+-      subalign: 2 # safe subalign value. proper values are applied by fix_linkerscript.py where necessary
+-      # text has align 8 but other sections may have a different alignment ...
+-      # explicit align correction is performed by fix_linkerscript.py during
+-      # project configuration. vram address is used, when reported, in non text
+-      # sections to calculate the correct align value.
+       subsegments:
+         - { start: 0x000080, type: asmtu, name: sdk/sce/ee/crt0, ld_align_segment_start: 0x8 }
+         - [0x0002B0, asmtu, sdk/sce/ee/libgraph/graph001]
+@@ -249,10 +235,10 @@ segments:
+         - [0x032110, asmtu, cs4/Seimei/src/Camera]
+         - [0x034930, asmtu, cs4/Seimei/src/Debug]
+         - [0x034D10, asmtu, cs4/Seimei/src/Draw]
+-        - [0x038710, asmtu, cs4/Seimei/src/File]
++        - [0x038710, c, cs4/Seimei/src/File]
+         - [0x038D00, asmtu, cs4/Seimei/src/Game]
+         - [0x039A00, asmtu, cs4/Seimei/src/Load]
+-        - [0x03A380, asmtu, cs4/Seimei/src/Main]
++        - [0x03A380, c, cs4/Seimei/src/Main]
+         - [0x03AFE0, asmtu, cs4/Seimei/src/MathUtil]
+         - [0x03BCE0, asmtu, cs4/Seimei/src/MdlLink]
+         - [0x03BDC0, asmtu, cs4/Seimei/src/Mic]
+@@ -283,7 +269,7 @@ segments:
+         - [0x059C60, asmtu, cs4/Seimei/src/Dss]
+         - [0x05ACB0, asmtu, cs4/Seimei/src/Damage]
+         - [0x05B3B0, asmtu, cs4/Seimei/src/Hmd]
+-        - [0x05E360, asmtu, cs4/Seimei/src/Binder]
++        - [0x05E360, c, cs4/Seimei/src/Binder]
+         - [0x05E980, asmtu, cs4/Seimei/src/Ene]
+         - [0x060DF0, asmtu, cs4/Seimei/src/EneAct]
+         - [0x062F40, asmtu, cs4/Seimei/src/EneThk]
+@@ -421,7 +407,7 @@ segments:
+         - [0x105150, asmtu, cs4/Seimei/src/E044Act]
+         - [0x105560, asmtu, cs4/Seimei/src/sfx/SfxFog]
+         - [0x105A00, asmtu, cs4/Seimei/fslib/lib/fsspummng]
+-        - [0x106520, asmtu, cs4/Seimei/src/Event]
++        - [0x106520, c, cs4/Seimei/src/Event]
+         - [0x106A90, asmtu, cs4/Seimei/src/Event0031]
+         - [0x107380, asmtu, cs4/Seimei/src/Event0051]
+         - [0x107E90, asmtu, cs4/Seimei/src/MiniGame/sgh/sgh]
+diff --git a/configure.py b/configure.py
+index cce9d9f..9d31656 100755
+--- a/configure.py
++++ b/configure.py
+@@ -5,23 +5,15 @@
+ from __future__ import annotations
+ 
+ import os
+-import re
+ import sys
+ import json
+-import yaml
+-import copy
+ import splat
+ import shutil
+ import argparse
+-import tempfile
+ import subprocess
+ import ninja_syntax
+ 
+-from dataclasses import dataclass, asdict
+-#from utils import ensure_path_and_write, normalize_object_path, to_expected_path
+-
+-import splat.util.options as splat_options
+-from splat.segtypes.linker_entry import LinkerEntry, clean_up_path
++from splat.segtypes.linker_entry import LinkerEntry
+ from spimdisasm.common.CompilerConfig import compilerOptions
+ 
+ from typing import Any, Union, Protocol, Literal, cast
+@@ -30,16 +22,6 @@ from contextlib import contextmanager
+ 
+ from splat.scripts import split
+ from splat.util.conf import load as splat_load_yaml
+-from splat.segtypes.linker_entry import LinkerEntry
+-
+-# from tools.python.fix_gp import fix_gp
+-from tools.python.fix_assets import fix_assets
+-from tools.python.fix_linkerscript import fix_linkerscript
+-
+-from sys import stdout
+-from subprocess import run
+-from dataclasses import dataclass
+-from struct import unpack
+ 
+ @contextmanager
+ def suppress_stdout_stderr():
+@@ -110,7 +92,7 @@ def make_compiler_cmd(config_dir: Path, src_path: Path, language: str):
+ 
+ 
+ def exec_shell(command: list[str]) -> str:
+-    ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
++    ret = subprocess.run(command, capture_output=True, text=True, check=False)
+     return ret.stdout
+ 
+ 
+@@ -196,6 +178,21 @@ def build_stuff(
+ 
+         object_strs = [str(obj) for obj in object_paths]
+ 
++        expected_strs = []
++        for obj in object_paths:
++            if obj.name.endswith(".c.o"):
++                obj = obj.with_name(obj.name.removesuffix(".c.o") + ".o")
++            elif obj.name.endswith(".s.o"):
++                obj = obj.with_name(obj.name.removesuffix(".s.o") + ".o")
++    
++            if obj.parts[:2] == ("build", "asm"):
++                p = obj.relative_to("build/asm")
++            elif obj.parts[:2] == ("build", "src"):
++                p = obj.relative_to("build/src")
++            else:
++                p = obj
++            expected_strs.append(str(Path("build/expected") / p))
++
+         for object_path in object_paths:
+             if object_path.name == "elf_header.s.o":
+                 continue
+@@ -211,6 +208,14 @@ def build_stuff(
+                 implicit_outputs=implicit_outputs,
+             )
+ 
++            ninja.build(
++                outputs=expected_strs,
++                rule="as",
++                inputs=[str((Path("asm") / s.relative_to("../src") if s.parts[:2] == ("..", "src") else s).with_suffix(".s")) for s in src_paths],
++                variables=variables,
++                implicit_outputs=implicit_outputs,
++            )
++
+     ninja = ninja_syntax.Writer(open(str(ROOT / config_dir / "build.ninja"), "w"), width=9999)
+ 
+     # Rules
+@@ -295,16 +300,15 @@ def build_stuff(
+             else:
+                 build(entry.object_path, entry.src_paths, "cc")
+ 
+-        elif isinstance(seg, splat.segtypes.common.databin.CommonSegDatabin):
+-            build(entry.object_path, entry.src_paths, "as")
+-
+-        elif isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin):
+-            build(entry.object_path, entry.src_paths, "as")
+-
+-        elif isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin):
+-            build(entry.object_path, entry.src_paths, "as")
+-
+-        elif isinstance(seg, splat.segtypes.common.sbss.CommonSegSbss):
++        elif isinstance(
++                    seg,
++                    (
++                        splat.segtypes.common.databin.CommonSegDatabin,
++                        splat.segtypes.common.rodatabin.CommonSegRodatabin,
++                        splat.segtypes.common.textbin.CommonSegTextbin,
++                        splat.segtypes.common.sbss.CommonSegSbss,
++                    ),
++                ):
+             build(entry.object_path, entry.src_paths, "as")
+ 
+         else:
+@@ -338,288 +342,6 @@ def build_stuff(
+         implicit=[elf_path],
+     )
+ 
+-
+-def get_line_file_path(args: AnnotationArgs):
+-    if args.line_file_path is not None:
+-        return args.line_file_path
+-
+- #   if args.elf_path.name == SH2_SERIAL and "Event/stage" in args.asm_path.as_posix():
+-#        return Path(f"{TOOLS}/alessatool/dwarf") / Path(args.asm_path.name).with_suffix(".line")
+-
+-    return None
+-
+-
+-def line_has_vram_addr(line: str, addr_str: str) -> bool:
+-    if addr_str not in line or "*/" not in line:
+-        return False
+-
+-    return line.index("*/") > line.index(addr_str)
+-
+-
+-def annotate_asm(args: AnnotationArgs):
+-    with open(args.asm_path, "r") as asm_file:
+-        asm_contents = asm_file.read()
+-
+-    asm_lines = asm_contents.splitlines()
+-    asm_line_index = 0
+-
+-    vram_start = args.vram_start
+-    vram_end = args.vram_end
+-
+-    if vram_start is None or vram_end is None:
+-        vram_start, vram_end = find_vram_bounds(asm_lines)
+-
+-    line_file_path = get_line_file_path(args)
+-
+-    if line_file_path is None or not line_file_path.exists():
+-        addresses = (f"0x{v:X}" for v in range(vram_start, vram_end, 0x4))
+-        proc = run([args.addr2line_path, "-e", args.elf_path, *addresses], capture_output=True, encoding=args.encoding)
+-        addr2line_output_lines = proc.stdout.splitlines()
+-    else:
+-        # parse the binary line number file.
+-
+-        # the format is a list of u_shorts, one per line number.
+-        # there should be one line number per vram address, and each vram
+-        # address should be exactly 4 bytes apart, mirroring how the addr2line
+-        # output is formatted
+-
+-        with open(line_file_path, "rb") as line_file:
+-            line_data = line_file.read()
+-            line_numbers = unpack(f"<{len(line_data) // 2}H", line_data)
+-            compile_unit = args.asm_path.with_suffix(".c").name
+-            addr2line_output_lines = list(map(lambda n : to_addr2line_format(compile_unit, n), line_numbers))
+-
+-    main_tu_name = None
+-    prev_tu_name = None
+-    prev_line_number = -1
+-    function_count = 0
+-    is_in_function_label = False
+-    current_vram_addr = vram_start
+-    annotated_asm_lines = []
+-
+-    for addr_index in range(0, len(addr2line_output_lines) - 1):
+-        line = addr2line_output_lines[addr_index]
+-
+-        if line.startswith("?"):
+-            current_vram_addr += 0x4
+-            continue
+-
+-        separator_index = line.rfind(":")
+-        current_line_number = int(line[separator_index+1:])
+-
+-        if current_line_number == prev_line_number:
+-            current_vram_addr += 0x4
+-            continue
+-
+-        vram_addr_str = f"{current_vram_addr:X}"
+-
+-        current_tu_name = line[0:separator_index]
+-        if current_tu_name != main_tu_name and current_tu_name != prev_tu_name:
+-            if main_tu_name:
+-                print(f"[warn] alessatool/annotate: address at {vram_addr_str} belongs to {current_tu_name}")
+-            else:
+-                main_tu_name = current_tu_name
+-        prev_tu_name = current_tu_name
+-
+-        while True:
+-            if asm_line_index >= len(asm_lines):
+-                raise AssertionError(
+-                    f"address 0x{current_vram_addr:X} not found in asm"
+-                )
+-
+-            asm_line = asm_lines[asm_line_index]
+-
+-            if line_has_vram_addr(asm_line, vram_addr_str):
+-                break
+-
+-            should_append_asm_line = True
+-            if args.tu:
+-                asm_line_trimmed = asm_line.strip()
+-
+-                # track when we go in & out of function symbols
+-                if asm_line_trimmed.startswith(FUNCTION_SYMBOL_LABEL):
+-                    function_count += 1
+-                    is_in_function_label = True
+-                    annotated_asm_lines.append(f"{UNIQUE_TEXT_SECTION_DIRECTIVE}{function_count}")
+-                    annotated_asm_lines.append("")
+-                elif asm_line_trimmed.startswith(END_FUNCTION_SYMBOL_LABEL):
+-                    is_in_function_label = False
+-
+-                # remove `nop`s
+-                if not is_in_function_label and asm_line_trimmed.endswith("nop"):
+-                    should_append_asm_line = False
+-                
+-                # remove `macro.inc` include directive
+-                elif asm_line_trimmed == INCLUDE_MACRO_INC_DIRECTIVE:
+-                    should_append_asm_line = False
+-
+-            if should_append_asm_line:
+-                annotated_asm_lines.append(asm_line)
+-
+-            asm_line_index += 1
+-
+-        annotated_asm_lines.append(f"\t.loc 1 {current_line_number}")    
+-        annotated_asm_lines.append(asm_line)
+-        asm_line_index += 1
+-
+-        prev_line_number = current_line_number
+-        current_vram_addr += 0x4
+-    
+-    while asm_line_index < len(asm_lines) - 1:
+-        annotated_asm_lines.append(asm_lines[asm_line_index])
+-        asm_line_index += 1
+-
+-    assert main_tu_name, "no valid compilation unit found"
+-    annotated_asm_lines = [
+-        ".section .debug",
+-        ".previous",
+-        ".text",
+-        f".file 1 \"{main_tu_name}\"",
+-        *annotated_asm_lines
+-    ]
+-
+-    append_final_new_line(annotated_asm_lines)
+-    annotated_asm_contents = "\n".join(annotated_asm_lines)
+-
+-    if not args.stdout and args.out_path:
+-        with open(args.out_path, "w") as out_file:
+-            out_file.write(annotated_asm_contents)
+-        if args.verbose:
+-            print(f"alessatool/annotate: wrote asm to {args.out_path}")
+-    else:
+-        stdout.write(annotated_asm_contents)
+-
+-
+-def append_final_new_line(lines: list[str]):
+-    if lines[-1] != "":
+-        lines.append("")
+-
+-
+-def make_asm(config_path: Path, config: dict[str, Any]):
+-    """
+-    Extracts assembly for each function into 'expected/asm/' subfolder.
+-
+-    The extraction is done following these steps:
+-        1. create temporary directory in project folder
+-        2. copy yaml to temporary directory
+-        3. modify yaml to have c subsegments instead of asm subsegments
+-        4. remove data, bss, etc., subsegments
+-        5. run splat on the new config yaml
+-        6. splat extracts assembly for each function into '<temp_dir>/asm/nonmatchings/'
+-        7. copy '<temp_dir>/asm/nonmatchings/' to '<project_dir>/expected/asm/'
+-    """
+-    with tempfile.TemporaryDirectory(dir=config_path, prefix="tmp_") as tmp_dir:
+-        tmp_path = Path(tmp_dir)
+-
+-        yaml_path = tmp_path / "config.yaml"
+-        asm_path = tmp_path / "asm" / "nonmatchings"
+-        dst_path = tmp_path / ".." / "expected" / "asm"
+-        dst_path = dst_path.resolve().relative_to(ROOT)
+-
+-        if dst_path.exists():
+-            print(f"expected asm dir '{dst_path}' already exists")
+-            return
+-
+-        config = copy.deepcopy(config)
+-        config["options"]["target_path"] = "../" + config["options"]["target_path"]
+-        config["options"]["asm_path"] = "asm"
+-        config["options"]["src_path"] = "src"
+-        config["options"]["build_path"] = "build"
+-        config["options"]["asset_path"] = "assets"
+-        config["options"]["undefined_funcs_auto_path"] = "../" + config["options"]["undefined_funcs_auto_path"]
+-        config["options"]["undefined_syms_auto_path"] = "../" + config["options"]["undefined_syms_auto_path"]
+-        config["options"]["symbol_addrs_path"] = "../" + config["options"]["symbol_addrs_path"]
+-        config["options"]["extensions_path"] = "../" + config["options"]["extensions_path"]
+-
+-        new_segments: list[Any] = []
+-        segments: list[Any] = config["segments"]
+-        for segment in segments:
+-            if isinstance(segment, list):
+-                new_segments.append(segment)
+-            elif isinstance(segment, dict) and segment["name"] == "main":
+-                new_subsegments: list[Any] = []
+-                subsegments = cast(list[Any], segment["subsegments"])
+-                for subsegment in subsegments:
+-                    if isinstance(subsegment, list):
+-                        if subsegment[1] == "asm":
+-                            subsegment[1] = "c"
+-                        new_subsegments.append(subsegment)
+-                    elif isinstance(subsegment, dict):
+-                        subsegment["type"] = subsegment["type"].strip(".")
+-                        if subsegment["type"] == "rodata":
+-                            # splat now requires rodata to be always prefixed with a dot (.)
+-                            # for the rodata migration to work properly
+-                            subsegment["type"] = ".rodata"
+-                        new_subsegments.append(subsegment)
+-                segment["subsegments"] = new_subsegments
+-                new_segments.append(segment)
+-        config["segments"] = new_segments
+-
+-        def rename_locals(base_path: Path):
+-            for asm_file in base_path.rglob("*.s"):
+-                data = asm_file.read_text()
+-                data = re.sub(r"__local_\d+", "", data)
+-                asm_file.write_text(data)
+-
+-        with yaml_path.open(mode="w") as yaml_file:
+-            yaml.dump(config, yaml_file, default_flow_style=False)
+-
+-        with suppress_stdout_stderr():
+-            split.main([yaml_path], modes=["all"], verbose=False)
+-
+-        # remove '__local_#' from asm
+-        rename_locals(asm_path)
+-
+-        dst_path.parent.mkdir(parents=True, exist_ok=True)
+-        shutil.copytree(asm_path, dst_path, dirs_exist_ok=True)
+-
+-        print(f"expected asm extracted to '{dst_path}'")
+-
+-        # make expected objs
+-
+-        for subseg in new_segments[1]["subsegments"]:
+-            if isinstance(subseg, list) and subseg[1] == "c":
+-                subseg[1] = "asm"
+-                subseg[2] += ".c"
+-
+-        config["options"]["asm_jtbl_label_macro"] = "llabel"
+-
+-        with yaml_path.open(mode="w") as yaml_file:
+-            yaml.dump(config, yaml_file, default_flow_style=False)
+-
+-        shutil.rmtree(tmp_path / "asm")
+-        (tmp_path / ".splache").unlink()
+-
+-        with suppress_stdout_stderr():
+-            split.main([yaml_path], modes=["all"], verbose=False)
+-
+-        # remove '__local_#' from asm
+-        rename_locals(asm_path)
+-
+-        dst_path = dst_path.parent / "obj"
+-        tmp_obj_path = tmp_path / "obj"
+-        tmp_asm_dir = tmp_path / "asm"
+-
+-        cpp = Path("..", "..", "..", (Path("tools") / "cc" / COMPILER))
+-
+-        for asm_file in tmp_asm_dir.rglob("*.c.s"):
+-            asm_file_rel = asm_file.relative_to(tmp_path)
+-            obj_file_rel = Path("obj") / asm_file.relative_to(tmp_asm_dir).with_suffix(".o")
+-            obj_file = tmp_obj_path / obj_file_rel.relative_to("obj")
+-            obj_file.parent.mkdir(parents=True, exist_ok=True)
+-            subprocess.run(
+-                f"{cpp} -I../src -I../include -Iinclude -Iinclude/sdk/ee -I../.. '{asm_file_rel}' -o - | "
+-                #f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | "
+-                f"mips-linux-gnu-as -no-pad-sections -EL -march=5900 -mabi=eabi -I../include -o {obj_file_rel} {asm_file_rel}",
+-                shell=True,
+-                cwd=tmp_path,
+-            )
+-
+-        shutil.copytree(tmp_obj_path, dst_path, dirs_exist_ok=True)
+-
+-        print(f"expected obj built to '{dst_path}'")
+-
+-
+ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], language: str):
+     """
+     Generate `objdiff.json` configuration from splat YAML config.
+@@ -633,7 +355,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
+     """
+     segments: list[Any] = config["segments"]
+ 
+-    tu_to_diff: list[tuple[Literal["asm", "c"], str]] = []
++    tu_to_diff: list[tuple[Literal["asmtu", "c"], str]] = []
+ 
+     for segment in segments:
+         if not (isinstance(segment, dict) and segment["name"] == "main"):
+@@ -653,7 +375,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
+             else:
+                 raise RuntimeError("invalid subsegment type")
+ 
+-            if subs_type in ("asm", "c"):
++            if subs_type in ("asmtu", "c"):
+                 if subs_name in (
+                     "crt0",
+                     "main/glob",
+@@ -678,7 +400,7 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
+     units: list[dict[str, Any]] = []
+ 
+     for tu_type, tu_name in tu_to_diff:
+-        target_path = Path("expected", "obj", tu_name).with_suffix(".c.o")
++        target_path = Path("build", "expected", tu_name).with_suffix(".o")
+ 
+         # since we only compile fully decompiled TUs, the
+         # "c" type implies that the TU is complete
+@@ -695,7 +417,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
+             "name": tu_name,
+             "target_path": str(target_path),
+             "base_path": str(base_path) if base_path else None,
+-            "metadata": {"progress_categories": [language]},
++            "metadata": {
++                "source_path": str(Path("..", "src", tu_name).with_suffix(".c")),
++                "progress_categories": [language]
++            },
+         }
+ 
+         if not base_path:
+@@ -717,10 +442,10 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
+ 
+     objdiff_json: dict[str, Any] = {
+         "$schema": "https://raw.githubusercontent.com/encounter/objdiff/main/config.schema.json",
+-        "custom_make": "true",
++        "custom_make": "ninja",
+         "custom_args": [],
+-        "build_target": False,
+-        "build_base": False,
++        "build_target": True,
++        "build_base": True,
+         "watch_patterns": [],
+         "units": units,
+         "progress_categories": progress_categories,
+@@ -858,7 +583,7 @@ def main():
+     cwd = Path(os.getcwd()).resolve()
+     if not ROOT.samefile(cwd):
+         print("ERROR: this script must be run from it's directory!")
+-        exit(1)
++        sys.exit(1)
+ 
+     compilerOptions["MWCCPS2"].value.bigAddendWorkaroundForMigratedFunctions = False
+ 
+@@ -876,9 +601,9 @@ def main():
+ 
+ 
+     if basename not in LANGUAGES:
+-        supported_elfs = f"{set(f'{elf} ({lang})' for elf, lang in LANGUAGES.items())}".replace("'", "")
++        supported_elfs = f"{ {f'{elf} ({lang})' for elf, lang in LANGUAGES.items()} }".replace("'", "")
+         print(f"unsupported game ELF. Supported versions are: {supported_elfs}")
+-        exit(1)
++        sys.exit(1)
+ 
+     language = LANGUAGES[basename]
+ 
+@@ -886,10 +611,6 @@ def main():
+         clean(config_dir, config)
+         return
+ 
+-    if args.make_asm:
+-        make_asm(config_dir, config)
+-        return
+-
+     if args.clean:
+         clean(config_dir, config)
+ 
+@@ -906,36 +627,12 @@ def main():
+ 
+     generate_lcf()
+ 
+-
+-    # fix asset .incbin path from relative to root to relative to config dir:
+-    #  e.g., change:
+-    #     .incbin "config/assets/name.section.bin"
+-    #  to:
+-    #     .incbin "assets/name.section.bin"
+-    fix_assets(asm_data_path, asset_rel_path)
+-
+     linker_entries = split.linker_writer.entries
+ 
+     build_stuff(config_dir, split.config, linker_entries, language)
+ 
+     write_permuter_settings(config_dir, src_path, language)
+ 
+-    # # replace gp_rel assembler macro with explicit offset as the gcc used
+-    # # to compile the code does not support it
+-    # gp_value = split.config["options"]["gp_value"]
+-    # symbol_addrs_path = Path(split.config["options"]["symbol_addrs_path"])
+-    # asm_rel_path = (config_dir / asm_path).resolve().relative_to(ROOT)
+-    # symbol_addrs_rel_path = (config_dir / symbol_addrs_path).resolve().relative_to(ROOT)
+-    # assert asm_rel_path.is_dir(), f"{asm_rel_path} not found or not a directory"
+-    # assert symbol_addrs_rel_path.is_file(), f"{symbol_addrs_rel_path} not found"
+-    # fix_gp(asm_rel_path, gp_value, symbol_addrs_rel_path)
+-
+-    # fix linkerscript by applying explicit alignments as
+-    # specified in the config yaml
+-    linkerscript_path = (config_dir / f"{basename}.ld").resolve().relative_to(ROOT)
+-    assert linkerscript_path.is_file(), f"{linkerscript_path} not found"
+-    fix_linkerscript(split.config, linkerscript_path)
+-
+     generate_objdiff_configuration(config_dir, split.config, language)
+ 
+ 

tools/python/add_line_numbers_to_asm.py

@@ -1,265 +0,0 @@
-#!/usr/bin/env python3
-
-# SPDX-FileCopyrightText: Copyright 2025 karas84 (https://github.com/karas84)
-# SPDX-License-Identifier: MIT
-#
-# This script inserts source line debug information (.loc directives) into assembly files
-# using data extracted from a JSON "stdump" file generated by the ccc tool (version 2.1),
-# available at https://github.com/chaoticgd/ccc.
-#
-# Original concept by Mc-muffin (https://github.com/Mc-muffin).
-
-from __future__ import annotations
-
-import re
-import io
-import sys
-import json
-import argparse
-
-from typing import Callable, Protocol, cast
-from pathlib import Path
-
-
-class STDUMPJson:
-    def __init__(self, json_path: Path):
-        self._json = json.loads(json_path.read_text())
-        self._line_cache = self._build_line_cache()
-        self._validate_sub_source_files()
-
-    def _build_line_cache(self):
-        line_cache: dict[int, int] = {}
-        functions = self._json["functions"]
-
-        for fun in functions:
-            if "line_numbers" not in fun:
-                continue
-
-            for addr, num in fun["line_numbers"]:
-                # there may be more lines for the same address. Ghidra seems to
-                # keep the last one, so that's what we are also doing
-                line_cache[addr] = num
-
-        return line_cache
-
-    def _validate_sub_source_files(self):
-        functions = self._json["functions"]
-        for fun in functions:
-            if (line_numbers := fun.get("line_numbers")) is None:
-                continue
-
-            if (sub_source_files := fun.get("sub_source_files")) is None:
-                continue
-
-            line_numbers = cast(list[tuple[int, int]], line_numbers)
-            sub_source_files = cast(list[tuple[int, str]], sub_source_files)
-
-            # # some functions may have "unterminated" inlines, but they usually have just one line number
-            # if len(sub_source_files) % 2 != 0:  # and len(line_numbers) != 1:
-            #     print(fun["name"], len(line_numbers))
-
-            # for addr, source in sub_source_files:
-
-    def find_function(self, name: str):
-        functions = self._json["functions"]
-        fun = next((f for f in functions if f["name"] == name), None)
-        return fun
-
-    def get_line(self, addr: int):
-        return self._line_cache.get(addr)
-
-
-def make_range_checker(lst: list[tuple[int, str]], delimiter: str) -> Callable[[int], bool]:
-    """
-    Given a sorted list of (number, label) where number increases,
-    build a checker that returns True if x is in a valid interval.
-
-    Interpretation:
-      - Each entry (n, label) marks the interval starting at n and going
-        up to the next entry's n (exclusive). The last entry's interval
-        goes to +inf.
-      - An interval starting at n is valid iff label == delimiter.
-      - If the first entry's label != delimiter, everything before the first n is valid.
-    """
-    if not lst:
-        # no markers -> everything valid
-        return lambda x: True
-
-    # ensure sorted by number
-    lst_sorted = sorted(lst, key=lambda t: t[0])
-    assert lst == lst_sorted
-
-    nums = [t[0] for t in lst_sorted]
-    labels = [t[1] for t in lst_sorted]
-
-    # Precompute intervals as (start, end_exclusive, is_valid)
-    intervals: list[tuple[int, int | None, bool]] = []
-    n_items = len(nums)
-
-    for i in range(n_items):
-        start = nums[i]
-        end_exclusive: int | None
-
-        if i + 1 < n_items:
-            end_exclusive = nums[i + 1]
-        else:
-            end_exclusive = None  # means to +inf
-
-        is_valid = labels[i] == delimiter
-        intervals.append((start, end_exclusive, is_valid))
-
-    first_before_is_valid = labels[0] != delimiter
-
-    def is_valid_fn(x: int) -> bool:
-        # before first number
-        if x < nums[0]:
-            return first_before_is_valid
-
-        # find the interval that contains x
-        for start, end_exclusive, valid_flag in intervals:
-            if end_exclusive is None:
-                if x >= start:
-                    return valid_flag
-            else:
-                if start <= x < end_exclusive:
-                    return valid_flag
-
-        # fallback (shouldn't happen)
-        return False
-
-    return is_valid_fn
-
-
-def is_always_valid_fn(addr: int):
-    return True
-
-
-def add_lines_to_asm(
-    asm_path: Path,
-    stdump_json_path: Path,
-    fun_start_offset: int,
-    keep_original_numbers: bool,
-    asm_out: Path | None,
-):
-    stdump_json = STDUMPJson(stdump_json_path)
-
-    function_name = asm_path.stem
-
-    if (fun := stdump_json.find_function(function_name)) is None:
-        raise RuntimeError(f"Cannot find function '{function_name}' in ccc's JSON")
-
-    sub_source_files = fun.get("sub_source_files")
-    # print(len(sub_source_files) if sub_source_files is not None else None)
-
-    asm_lines = asm_path.read_text().splitlines()
-    re_instr = re.compile(r"^\s*\/\* [A-Z0-9]+ ([A-Z0-9]{8}) [A-Z0-9]{8} \*\/ .*$")
-    line_dict: dict[int, int] = {}
-
-    relative_path: str = fun["relative_path"]
-    non_func_addrs: list[int] = []
-
-    if sub_source_files:
-        checker = make_range_checker(sub_source_files, relative_path)
-    else:
-        checker = is_always_valid_fn
-
-    start_line_num: int = sys.maxsize
-
-    for line in asm_lines:
-        if m := re_instr.match(line):
-            instr_addr = int(m.group(1), 16)
-            line_num = stdump_json.get_line(instr_addr)
-
-            if line_num:
-                line_dict[instr_addr] = line_num
-
-            if not checker(instr_addr):
-                non_func_addrs.append(instr_addr)
-            elif line_num:
-                start_line_num = min(start_line_num, line_num)
-
-    if start_line_num == sys.maxsize:
-        start_line_num = 1
-
-    min_line_num = 0 if keep_original_numbers else start_line_num - 1
-
-    new_asm_lines: list[str] = []
-    asm_n: int = 0
-
-    for line in asm_lines:
-        if m := re_instr.match(line):
-            asm_n += 1
-
-            instr_addr = int(m.group(1), 16)
-            line_num = line_dict.get(instr_addr)
-
-            if line_num is not None and instr_addr in non_func_addrs:
-                new_asm_lines.append(f"  .loc 1 {line_num} # inline")
-                new_asm_lines.append(line)
-                continue
-
-            if asm_n == 1 and line_num is None:
-                # sometimes we don't have a number for the first line of assembly,
-                # so we reuse the first known line among the ones we have
-                line_num = start_line_num
-
-            if line_num is not None:
-                new_line_num = (line_num - min_line_num) + fun_start_offset
-                new_asm_lines.append(f"  .loc 1 {new_line_num} ")
-
-        new_asm_lines.append(line)
-
-    stream = io.StringIO()
-    stream.write(""".section .debug
-.previous
-.text
-.file 1 "source.c"
-
-.set noat
-.set noreorder
-
-""")
-    stream.write("\n".join(new_asm_lines))
-
-    if asm_out is not None:
-        asm_out.write_text(stream.getvalue())
-    else:
-        print(stream.getvalue())
-
-
-def main():
-    class ArgProtocol(Protocol):
-        asm_path: Path
-        stdump_json_path: Path
-        asm_out: Path | None
-        offset: int
-        keep_original: bool
-
-    parser = argparse.ArgumentParser(description="Add line debug info to assembly files")
-    parser.add_argument("--asm-path", required=True, type=Path, help="Path to the assembly file to add lines to")
-    parser.add_argument("--stdump-json-path", required=True, type=Path, help="Path to ccc's json stdump")
-    parser.add_argument("--asm-out", type=Path, required=False, help="Path to output asm (defaults to stdout)")
-    parser.add_argument(
-        "--offset", type=int, required=False, default=0, help="Offset to apply to line numbers (default: 0)"
-    )
-    parser.add_argument(
-        "--keep-original",
-        action="store_true",
-        help="Don't start line numbers from 1 (+ offset) but keep the original line numbers (+offset) instead",
-    )
-
-    args = cast(ArgProtocol, parser.parse_args())
-
-    fun_start_num = args.offset
-
-    add_lines_to_asm(
-        args.asm_path,
-        args.stdump_json_path,
-        fun_start_num,
-        args.keep_original,
-        args.asm_out,
-    )
-
-
-if __name__ == "__main__":
-    main()

tools/python/ccc_json_v7.py

@@ -1,351 +0,0 @@
-from __future__ import annotations
-
-import pydantic
-
-from typing import Optional, Literal, Any
-
-
-class AddressRange(pydantic.BaseModel):
-    low: int
-    high: int
-
-
-class ValueType(pydantic.BaseModel):
-    descriptor: Optional[str] = None
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[ValueType] = None
-    return_type: Optional[ReturnType] = None
-    modifier: Optional[str] = None
-    vtable_index: Optional[int] = None
-    is_constructor: Optional[bool] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-
-
-class DeduplicatedTypeValueType(pydantic.BaseModel):
-    descriptor: Literal["function_type", "type_name"]
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[ValueType] = None
-    return_type: Optional[ReturnType] = None
-    modifier: Optional[str] = None
-    vtable_index: Optional[int] = None
-    is_constructor: Optional[bool] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-
-
-class ParameterType(pydantic.BaseModel):
-    descriptor: str
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[ValueType] = None
-
-
-class Parameter(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    class_: str = pydantic.Field(..., alias="class")
-    storage: Storage
-    block_low: int
-    block_high: int
-    type: ParameterType
-
-
-class ReturnType(pydantic.BaseModel):
-    descriptor: str
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[ValueType] = None
-
-
-class FunctionType(pydantic.BaseModel):
-    descriptor: str
-    parameters: list[Parameter]
-    modifier: str
-    vtable_index: int
-    is_constructor: bool
-    return_type: Optional[ReturnType] = None
-
-
-class Storage(pydantic.BaseModel):
-    type: str
-    register_: Optional[str] = pydantic.Field(None, alias="register")
-    register_class: Optional[str] = None
-    dbx_register_number: Optional[int] = None
-    register_index: Optional[int] = None
-    is_by_reference: Optional[bool] = None
-    stack_offset: Optional[int] = None
-    global_location: Optional[str] = None
-    global_address: Optional[int] = None
-
-
-class Constant(pydantic.BaseModel):
-    value: int
-    name: str
-
-
-class ElementType(pydantic.BaseModel):
-    descriptor: Literal["array", "pointer", "type_name", "enum"]
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-    value_type: Optional[ValueType] = None
-    constants: Optional[list[Constant]] = None
-
-    def parsed_size(self) -> tuple[int, str]:
-        if self.descriptor == "pointer":
-            return 1, "pointer"
-
-        elif self.descriptor == "array":
-            assert self.element_type
-            assert self.element_count is not None
-            if self.element_count == 0:
-                # implicit size array
-                _, type_name = self.element_type.parsed_size()
-                return 0, type_name
-            n, type_name = self.element_type.parsed_size()
-            return self.element_count * n, type_name
-
-        elif self.descriptor == "enum":
-            return 1, "enum"
-
-        else:  # "type_name"
-            assert self.type_name
-            return 1, self.type_name
-
-
-class Local(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    class_: str = pydantic.Field(..., alias="class")
-    storage: Storage
-    block_low: int
-    block_high: int
-    type: ElementType
-    storage_class: Optional[str] = None
-
-    @property
-    def parsed_size(self) -> tuple[int, str]:
-        return self.type.parsed_size()
-
-
-class SubSourceFile(pydantic.BaseModel):
-    address: int
-    path: str
-
-
-class Function(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    address_range: AddressRange
-    type: FunctionType
-    locals: list[Local]
-    line_numbers: list[list[int]]
-    sub_source_files: list[SubSourceFile]
-    storage_class: Optional[str] = None
-    relative_path: Optional[str] = None
-
-
-class Global(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    class_: str = pydantic.Field(..., alias="class")
-    storage: Storage
-    block_low: int
-    block_high: int
-    type: ElementType
-    storage_class: Optional[str] = None
-
-    @property
-    def parsed_size(self) -> tuple[int, str]:
-        return self.type.parsed_size()
-
-
-class File(pydantic.BaseModel):
-    descriptor: str
-    path: str
-    relative_path: str
-    text_address: int
-    types: list[Any]
-    functions: list[Function]
-    globals: list[Global]
-    stabs_type_number_to_deduplicated_type_index: dict[str, int]
-
-
-class UnderlyingType(pydantic.BaseModel):
-    descriptor: str
-    source: str
-    type_name: str
-    referenced_file_index: int
-    referenced_stabs_type_number: int
-
-
-class Field(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    relative_offset_bytes: int
-    absolute_offset_bytes: int
-    size_bits: int
-    bitfield_offset_bits: Optional[int] = None
-    underlying_type: Optional[UnderlyingType] = None
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[ValueType] = None
-    base_classes: Optional[list[Any]] = None
-    fields: Optional[list[Field]] = None
-    member_functions: Optional[list[Any]] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-
-
-class FieldModel(pydantic.BaseModel):
-    descriptor: str
-    name: str
-    relative_offset_bytes: int
-    absolute_offset_bytes: int
-    size_bits: int
-    value_type: Optional[ValueType] = None
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-    bitfield_offset_bits: Optional[int] = None
-    underlying_type: Optional[UnderlyingType] = None
-    fields: Optional[list[Field]] = None
-    member_functions: Optional[list[Any]] = None
-    base_classes: Optional[list[Any]] = None
-
-
-class DeduplicatedType(pydantic.BaseModel):
-    descriptor: Literal["array", "builtin", "enum", "pointer", "struct", "type_name", "union"]
-    name: Optional[str] = None
-    storage_class: Optional[Literal["typedef"]] = None
-    stabs_type_number: int
-    files: list[int]
-    class_: Optional[str] = pydantic.Field(None, alias="class")
-    size_bits: Optional[int] = None
-    base_classes: Optional[list[Any]] = None
-    fields: Optional[list[FieldModel]] = None
-    member_functions: Optional[list[Any]] = None
-    source: Optional[str] = None
-    type_name: Optional[str] = None
-    referenced_file_index: Optional[int] = None
-    referenced_stabs_type_number: Optional[int] = None
-    value_type: Optional[DeduplicatedTypeValueType] = None
-    conflict: Optional[bool] = None
-    element_type: Optional[ElementType] = None
-    element_count: Optional[int] = None
-    constants: Optional[list[Constant]] = None
-
-
-class CCCJSONv7Model(pydantic.BaseModel):
-    version: Literal[7]
-    files: list[File]
-    deduplicated_types: list[DeduplicatedType]
-
-
-# def test(stdump_json_path: str):
-#     import json
-#
-#     with open(stdump_json_path, mode="r") as fh:
-#         json_data = fh.read()
-#
-#     model = CCCJSONv7Model.model_validate_json(json_data)
-#
-#     type_map: dict[str, int] = {}
-#
-#     for n, dt in enumerate(model.deduplicated_types):
-#         if dt.descriptor == "builtin":
-#             assert dt.name and dt.name not in type_map
-#             assert dt.class_ is not None
-#             size_bits = int(dt.class_.split("-", maxsplit=1)[0])
-#             assert size_bits % 8 == 0
-#             type_map[dt.name] = size_bits // 8
-#
-#         elif dt.descriptor == "type_name":
-#             assert dt.name is not None
-#             assert dt.type_name is not None
-#             if dt.name in type_map:
-#                 if dt.size_bits:
-#                     assert type_map[dt.name] == dt.size_bits // 8
-#             if dt.name == dt.type_name == "void":
-#                 type_map["void"] = type_map["int"]
-#                 continue
-#             assert dt.type_name in type_map, (n, dt.type_name)
-#             type_map[dt.name] = type_map[dt.type_name]
-#
-#         elif dt.descriptor == "pointer":
-#             assert dt.name is not None
-#             assert dt.value_type is not None
-#             if dt.value_type.descriptor == "function_type":
-#                 assert dt.name not in type_map
-#                 type_map[dt.name] = type_map["int"]
-#             elif dt.value_type.descriptor == "type_name":
-#                 assert dt.value_type.type_name
-#                 assert dt.value_type.type_name in type_map
-#                 type_map[dt.name] = type_map[dt.value_type.type_name]
-#
-#         elif dt.descriptor == "struct":
-#             assert dt.name is not None
-#             if not dt.conflict:
-#                 assert dt.name not in type_map, n
-#             else:
-#                 if dt.name in type_map:
-#                     continue
-#             assert dt.size_bits is not None
-#             assert dt.size_bits % 8 == 0
-#             type_map[dt.name] = dt.size_bits // 8
-#
-#         elif dt.descriptor == "array":
-#             assert dt.name is not None
-#             element_count = 1
-#             element_type = dt.element_type
-#             type_name = None
-#             while element_type:
-#                 if element_type.element_count is not None:
-#                     element_count *= element_type.element_count
-#                 if element_type.type_name is not None:
-#                     type_name = element_type.type_name
-#                     assert type_name in type_map
-#                     element_count *= type_map[type_name]
-#                 element_type = element_type.element_type
-#             assert type_name and type_name in type_map, n
-#             type_map[dt.name] = type_map[type_name] * element_count
-#
-#         elif dt.descriptor == "enum":
-#             if dt.name:
-#                 if not dt.conflict:
-#                     assert dt.name not in type_map, n
-#                 type_map[dt.name] = type_map["int"]
-#
-#         elif dt.descriptor == "union":
-#             assert dt.name
-#             assert dt.size_bits
-#             assert dt.size_bits % 8 == 0
-#             type_map[dt.name] = dt.size_bits // 8
-#
-#         else:
-#             assert False, f"unknown {n}"
-#
-#     print(json.dumps(type_map, indent=2))
-
-
-# if __name__ == "__main__":
-#     test(path-to-stdump-json)

tools/python/cstruct.py

@@ -1,322 +0,0 @@
-from __future__ import annotations
-
-import io
-import sys
-import math
-import ctypes
-import numpy as np
-import numpy.typing as npt
-from collections.abc import Sized, Sequence
-from typing import Type, TypeVar, Any, cast, TextIO, BinaryIO, Optional
-from ctypes import LittleEndianStructure, Union, Array, c_uint32
-
-_G = TypeVar("_G")
-
-CTypeType = (
-    type[ctypes.c_int8]
-    | type[ctypes.c_uint8]
-    | type[ctypes.c_int16]
-    | type[ctypes.c_uint16]
-    | type[ctypes.c_int32]
-    | type[ctypes.c_uint32]
-    | type[ctypes.c_float]
-    | type[ctypes.c_double]
-)
-
-ctypes_types: dict[str, CTypeType] = {
-    "char": ctypes.c_int8,
-    "u_char": ctypes.c_uint8,
-    "short": ctypes.c_int16,
-    "u_short": ctypes.c_uint16,
-    "int": ctypes.c_int32,
-    "u_int": ctypes.c_uint32,
-    "float": ctypes.c_float,
-    "double": ctypes.c_double,
-}
-
-
-class c_addr(c_uint32):
-    def __str__(self):
-        # if self.value == 0:
-        #     return "NULL"
-
-        # return f"0x{self.value:08x}"
-        return f"0x{self.value:x}"
-
-
-class c_str(c_uint32):
-    def to_str(self, elf: BinaryIO):
-        if self.value == 0:
-            return "NULL"
-        elf.seek(self.value)
-        buf = io.BytesIO()
-        while (c := elf.read(1)) != b"\0":
-            buf.write(c)
-        return '"' + buf.getvalue().decode("ASCII") + '"'
-
-
-class c_addr_ptr(c_uint32):
-    _addresses: dict[int, str] | None = None
-
-    @classmethod
-    def set_addresses(cls, addresses: dict[int, str] | None):
-        cls._addresses = addresses
-
-    def __str__(self):
-        if self.value == 0:
-            return "NULL"
-
-        if self._addresses and self.value in self._addresses:
-            return self._addresses[self.value]
-
-        print(f"warning: no address for pointer 0x{self.value:x}")
-
-        return f"0x{self.value:x}"
-
-
-def print_arr(arr: npt.NDArray[np.int_], lst: list[Any], file: TextIO):
-    if arr.ndim > 1:
-        for x in arr:
-            file.write("{")
-            print_arr(x, lst, file)
-            file.write("},")
-    else:
-        file.write(",".join(str(lst[i]) for i in arr))
-
-
-def print_carr(arr: Sequence[Any], file: TextIO):
-    if hasattr(arr[0], "_length_"):
-        for a in arr:
-            file.write("{")
-            print_carr(a, file)
-            file.write("},")
-    else:
-        v = str(str([x for x in arr])).replace("[", "").replace("]", "")
-        file.write(v)
-
-
-def chunks(lst: Sequence[_G], n: int):
-    """Yield successive n-sized chunks from lst."""
-    for i in range(0, len(lst), n):
-        yield lst[i : i + n]
-
-
-def format_array(lst: Sequence[_G], dims: Sequence[int], file: TextIO):
-    if len(dims) > 1:
-        for ll in chunks(lst, len(lst) // dims[0]):
-            file.write("{")
-            format_array(ll, dims[1:], file)
-            file.write("},")
-    else:
-        for n, x in enumerate(lst):
-            sep = ", " if n < len(lst) - 1 else ""
-            file.write(f"{x}{sep}")
-
-
-def resolve_annotations(namespace: dict[str, Any], annotations: dict[str, Any]):
-    module = sys.modules.get(namespace.get("__module__", ""))
-    globals_ = vars(module) if module else {}
-    resolved: dict[str, Any] = {}
-
-    for k, v in annotations.items():
-        if isinstance(v, str):
-            try:
-                resolved[k] = eval(v, globals_, namespace)
-            except Exception:
-                resolved[k] = v  # keep as string if can't resolve
-        else:
-            resolved[k] = v
-
-    return resolved
-
-
-class LittleEndianStructureFieldsFromTypeHints(type(LittleEndianStructure)):  # pyright: ignore
-    def __new__(
-        cls: Type[type],
-        name: str,
-        bases: tuple[type, ...],
-        namespace: dict[str, Any],
-        /,
-        *,
-        align: Optional[int] = None,
-        pack: Optional[int] = None,
-    ) -> LittleEndianStructureFieldsFromTypeHints:
-        annotations = namespace.get("__annotations__", {})
-        annotations = resolve_annotations(namespace, annotations)
-        if "__elf__" in annotations:
-            annotations.pop("__elf__")
-        if align is not None:
-            namespace["_align_"] = align
-        if pack is not None:
-            namespace["_pack_"] = pack
-            namespace["_layout_"] = "ms"
-        if fields := list(annotations.items()):
-            namespace["_fields_"] = fields
-        return type(LittleEndianStructure).__new__(cls, name, bases, namespace)  # pyright: ignore
-
-
-class CStructure(LittleEndianStructure, metaclass=LittleEndianStructureFieldsFromTypeHints):
-    __elf__: BinaryIO
-
-    @classmethod
-    def sizeof(cls) -> int:
-        align = getattr(cls, "_align_", 0)
-        c_size = ctypes.sizeof(cls)
-        if align > 0:
-            rem = c_size % align
-            if rem:
-                rem = align - rem
-                return c_size + rem
-
-        return c_size
-
-    @classmethod
-    def parse(cls, data: bytes):
-        cstruct_size = sizeof(cls)
-        assert len(data) % cstruct_size == 0, f"{len(data)}, {cstruct_size}"
-        cstruct_num = len(data) // cstruct_size
-        stream = io.BytesIO(data)
-        cstructs = [cls.from_buffer_copy(stream.read(cstruct_size)) for _ in range(cstruct_num)]
-        return cstructs
-
-    @classmethod
-    def dumps(
-        cls,
-        name: str,
-        data: bytes,
-        numel: int | list[int],
-        static: bool = False,
-        nosize: bool = False,
-        noarray: bool = False,
-    ):
-        cstructs = cls.parse(data)
-        stream = io.StringIO()
-        if static:
-            stream.write("static ")
-        stream.write(f"{cls.__name__} {name}")  # pyright: ignore
-        assert not (noarray and isinstance(numel, list))
-        if not noarray:
-            if isinstance(numel, int):
-                assert len(cstructs) == numel, (len(cstructs), numel)
-                numel_str = f"{len(cstructs)}" if not nosize else ""
-                stream.write(f"[{numel_str}]")
-            else:
-                numel_str = "".join([f"[{num if n > 0 or not nosize else ''}]" for n, num in enumerate(numel)])
-                stream.write(numel_str)
-        tot_numel = max(1, numel) if isinstance(numel, int) else math.prod(numel)
-        # nmdim = 1 if isinstance(numel, int) else len(numel)
-        assert len(cstructs) == tot_numel, (len(cstructs), tot_numel)
-
-        stream.write(" = ")
-        if not noarray:
-            stream.write("{\n")
-        if isinstance(numel, int):
-            for n, s in enumerate(cstructs):
-                is_last = n == len(cstructs) - 1
-                if is_last and noarray:
-                    stream.write(f"{s}\n")
-                else:
-                    stream.write(f"{s},\n")
-
-        else:
-            idxs = np.arange(tot_numel).reshape(numel)
-            print_arr(idxs, cstructs, stream)
-        if not noarray:
-            stream.write("}")
-        stream.write(";\n\n")
-        return stream.getvalue()
-
-    # def to_str(self, elf: BinaryIO):
-    def __str__(self):
-        stream = io.StringIO()
-        stream.write("    {\n")
-        for f, *_ in self._fields_:  # pyright: ignore
-            if f.startswith("_pad_"):
-                continue
-            v = getattr(self, f)  # pyright: ignore
-            if f in ("_in", "_pass"):
-                f = f[1:]  # pyright: ignore
-            if isinstance(v, c_str):
-                stream.write(f"        .{f} = {v.to_str(self.__elf__)},\n")
-            elif not isinstance(v, Array):
-                stream.write(f"        .{f} = {v},\n")
-            else:
-                arr = cast(Sized, v)
-                if len(arr) and isinstance(v[0], CStructure):
-                    arr = cast(list[CStructure], arr)
-                    stream.write(f"        .{f} = {{\n")
-                    for elem in arr:
-                        stream.write(f"        {elem},\n")
-                    stream.write("        },\n")
-                else:
-                    arr = cast(Sequence[Any], arr)
-                    if isinstance(arr[0], Array):
-                        # multidimensional ctypes array
-                        stream.write(f"        .{f} = {{")
-                        print_carr(arr, stream)
-                        stream.write("},\n")
-                    else:
-                        stream.write(f"        .{f} = {{")
-                        dims = [len(arr)]
-                        format_array(arr, dims, stream)
-
-                        stream.write("},\n")
-        stream.write("    }")
-        return stream.getvalue()
-
-
-class UnionFieldsFromTypeHints(type(Union)):  # pyright: ignore
-    def __new__(
-        cls: Type[type],
-        name: str,
-        bases: tuple[type, ...],
-        namespace: dict[str, Any],
-        /,
-        *,
-        align: Optional[int] = None,
-        pack: Optional[int] = None,
-    ) -> UnionFieldsFromTypeHints:
-        annotations = namespace.get("__annotations__", {})
-        if align is not None:
-            namespace["_align_"] = align
-        if pack is not None:
-            namespace["_pack_"] = pack
-        if fields := list(annotations.items()):
-            namespace["_fields_"] = fields
-        return type(Union).__new__(cls, name, bases, namespace)  # pyright: ignore
-
-
-class CUnion(Union, metaclass=UnionFieldsFromTypeHints):
-    pass
-
-
-_T = TypeVar("_T", bound=CStructure)
-
-
-def sizeof(cstruct_type: Type[_T]) -> int:
-    align = getattr(cstruct_type, "_align_", 0)
-    c_size = ctypes.sizeof(cstruct_type)
-    if align > 0:
-        rem = c_size % align
-        if rem:
-            rem = align - rem
-            return c_size + rem
-
-    return c_size
-
-
-def parse_cstruct(cstruct_type: Type[_T], data: bytes) -> list[_T]:
-    cstruct_size = sizeof(cstruct_type)
-    assert len(data) % cstruct_size == 0, f"{len(data)}, {cstruct_size}"
-    cstruct_num = len(data) // cstruct_size
-    stream = io.BytesIO(data)
-    cstructs = [cstruct_type.from_buffer_copy(stream.read(cstruct_size)) for _ in range(cstruct_num)]
-    return cstructs
-
-
-def print_cstruct(name: str, cstruct_type: Type[CStructure], data: bytes):
-    cstructs = parse_cstruct(cstruct_type, data)
-    print(f"{cstruct_type.__name__} {name}[{len(cstructs)}] = {{")  # pyright: ignore
-    for s in cstructs:
-        print(s)
-    print("};\n")

tools/python/fix_asm_for_m2c.py

@@ -1,56 +0,0 @@
-import argparse
-
-
-registers = {
-    "$0": "zero",
-    "$1": "at",
-    "$2": "v0",
-    "$3": "v1",
-    "$4": "a0",
-    "$5": "a1",
-    "$6": "a2",
-    "$7": "a3",
-    "$8": "t0",
-    "$9": "t1",
-    "$10": "t2",
-    "$11": "t3",
-    "$12": "t4",
-    "$13": "t5",
-    "$14": "t6",
-    "$15": "t7",
-    "$16": "s0",
-    "$17": "s1",
-    "$18": "s2",
-    "$19": "s3",
-    "$20": "s4",
-    "$21": "s5",
-    "$22": "s6",
-    "$23": "s7",
-    "$24": "t8",
-    "$25": "t9",
-    "$26": "k0",
-    "$27": "k1",
-    "$28": "gp",
-    "$29": "sp",
-    "$30": "fp",
-    "$31": "ra",
-}
-
-
-def fix_asm(asm_file: str):
-    with open(asm_file, mode="r") as fh:
-        for line in fh:
-            for reg_num, reg_mnem in reversed(registers.items()):
-                line = line.replace(reg_num, reg_mnem)
-            print(line, end="")
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("asm_file", help="source assembly file path")
-    args = parser.parse_args()
-    fix_asm(args.asm_file)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_asm_matching.py

@@ -1,125 +0,0 @@
-import os
-import re
-import argparse
-import pydantic
-
-from pathlib import Path
-from typing import Protocol, cast
-
-
-CONFIG_ROOT = Path(__file__).parent.parent.parent.resolve() / "config"
-
-# enumerate languages from config dir
-LANGUAGES = [Path(f.path).name for f in os.scandir(CONFIG_ROOT) if f.is_dir()]
-
-RE_INSTR = re.compile(r"^\t(?:[^.]|\.p2align \d+)")
-
-
-class InstructionPatch(pydantic.RootModel[tuple[int, str, str]]):
-    root: tuple[int, str, str]
-
-    @property
-    def instr_no(self):
-        return self.root[0]
-
-    @property
-    def instr_org(self):
-        return self.root[1]
-
-    @property
-    def instr_patch(self):
-        return self.root[2]
-
-
-class ASMPatch(pydantic.RootModel[dict[str, list[InstructionPatch]]]):
-    root: dict[str, list[InstructionPatch]]
-
-
-class PatchDB(pydantic.RootModel[dict[str, ASMPatch]]):
-    root: dict[str, ASMPatch]
-
-
-def fix_asm(asm_file: Path, asm_patch: ASMPatch):
-    lines = asm_file.read_text().splitlines()
-
-    def find_func(func: str):
-        for i, line in enumerate(lines):
-            if re.match(rf"^{func}:", line):
-                return i
-
-        return -1
-
-    def find_line(start: int, num: int):
-        offset = 0
-        instr_no = -1
-        for line in lines[start:]:
-            if RE_INSTR.match(line):
-                instr_no += 1
-
-            if instr_no == num:
-                return start + offset
-
-            offset += 1
-
-        raise RuntimeError("cannot find instruction!")
-
-    for func, patch_lst in asm_patch.root.items():
-        n = find_func(func)
-
-        if n == -1:
-            print(f"WARNING: cannot find function {func} in {asm_file.name}")
-            continue
-
-        for patch in patch_lst:
-            line_no = find_line(n, patch.instr_no)
-            line_org = lines[line_no]
-
-            line_org_clean = re.sub(r"\s+", " ", line_org).strip()
-            instr_org_clean = re.sub(r"\s+", " ", patch.instr_org).strip()
-            instr_patch_clean = re.sub(r"\s+", " ", patch.instr_patch).strip()
-
-            if line_org_clean != instr_org_clean:
-                print(f"WARNING: wrong line: {asm_file.name}:{func}:{line_no + 1}: {line_org} != {patch.instr_org}")
-                continue
-
-            lines[line_no] = f"\t{instr_patch_clean}"
-
-    asm_file.write_text("\n".join(lines))
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        language: str
-        asm_file: Path
-
-    parser = argparse.ArgumentParser(description="apply asm patches to assembly files")
-    parser.add_argument("language", type=str, choices=LANGUAGES, help="language of the asm that is being patched")
-    parser.add_argument("asm_file", type=Path, help="generated assembly file to patch (relative to build dir)")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    asm_file = CONFIG_ROOT / args.language / args.asm_file
-
-    if not asm_file.exists():
-        print(f"ERROR: cannot find assembly file {asm_file}")
-        exit(1)
-
-    patch_db_path = CONFIG_ROOT / args.language / "asm_patches.json"
-
-    if not patch_db_path.exists():
-        print("ERROR: patch db file doesn't exist. aborting patching asm")
-        exit(1)
-
-    patch_db = PatchDB.model_validate_json(patch_db_path.read_text())
-
-    if asm_file.name not in patch_db.root:
-        print(f"ERROR: no patches found for file {asm_file.name}")
-        exit(1)
-
-    asm_patch = patch_db.root[asm_file.name]
-
-    fix_asm(asm_file, asm_patch)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_assets.py

@@ -1,38 +0,0 @@
-import re
-import glob
-import tqdm
-import argparse
-
-from typing import Protocol, cast
-from pathlib import Path
-
-
-def fix_assets(asm_data_path: Path, asset_rel_path: Path):
-    asm_files = glob.glob(str(asm_data_path / "**/*.s"), recursive=True)
-    for asm_file in tqdm.tqdm(asm_files, desc="Fixing data asm"):
-        n: int = 0
-        with open(asm_file, mode="r") as fh:
-            data_asm: str = fh.read()
-            data_asm, n = re.subn(rf'\.incbin "{asset_rel_path}/', '.incbin "assets/', data_asm)
-
-        if n > 0:
-            with open(asm_file, mode="w") as wh:
-                wh.write(data_asm)
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        asm_data_path: Path
-        asset_rel_path: Path
-
-    parser = argparse.ArgumentParser(description="fixes data asm include path")
-    parser.add_argument("asm_path", metavar="asm-path", type=Path, help="data path in assembly root to patch")
-    parser.add_argument("asset_rel_path", metavar="asset-path", type=Path, help="relative asset path")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    fix_assets(args.asm_data_path, args.asset_rel_path)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_bin_matching.py

@@ -1,79 +0,0 @@
-from __future__ import annotations
-
-import os
-import argparse
-import pydantic
-
-from pathlib import Path
-from typing import Protocol, cast
-
-
-CONFIG_ROOT = Path(__file__).parent.parent.parent.resolve() / "config"
-
-# enumerate languages from config dir
-LANGUAGES = [Path(f.path).name for f in os.scandir(CONFIG_ROOT) if f.is_dir()]
-
-
-class BytePatch(pydantic.RootModel[tuple[int, int]]):
-    root: tuple[int, int]
-
-    @property
-    def address(self):
-        return self.root[0]
-
-    @property
-    def size(self):
-        return self.root[1]
-
-
-class BINPatch(pydantic.RootModel[dict[str, BytePatch]]):
-    root: dict[str, BytePatch]
-
-
-class PatchDB(pydantic.RootModel[dict[str, BINPatch]]):
-    root: dict[str, BINPatch]
-
-
-def fix_elf(orig_elf_path: Path, built_elf_path: Path, bin_patch: BytePatch):
-    with orig_elf_path.open("rb") as fh:
-        fh.seek(bin_patch.address)
-        data = fh.read(bin_patch.size)
-
-    with built_elf_path.open(mode="r+b") as wh:
-        wh.seek(bin_patch.address)
-        wh.write(data)
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        language: str
-        elf_file: Path
-
-    parser = argparse.ArgumentParser(description="apply asm patches to assembly files")
-    parser.add_argument("language", type=str, choices=LANGUAGES, help="language of the elf that is being patched")
-    parser.add_argument("elf_file", type=Path, help="elf file to patch (relative to build dir)")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    built_elf_file = CONFIG_ROOT / args.language / args.elf_file
-
-    if not built_elf_file.exists():
-        print(f"ERROR: cannot find elf file {built_elf_file}")
-        exit(1)
-
-    orig_elf_path = CONFIG_ROOT / args.language / args.elf_file.name
-
-    patch_db_path = CONFIG_ROOT / args.language / "bin_patches.json"
-
-    if not patch_db_path.exists():
-        exit(0)
-
-    patch_db = PatchDB.model_validate_json(patch_db_path.read_text())
-
-    for _tu_name, bin_patch in patch_db.root.items():
-        for _func_name, byte_patch in bin_patch.root.items():
-            fix_elf(orig_elf_path, built_elf_file, byte_patch)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_gp.py

@@ -1,75 +0,0 @@
-import re
-import glob
-import tqdm
-import argparse
-import functools
-
-from typing import Protocol, cast
-from pathlib import Path
-
-
-@functools.lru_cache(maxsize=1024)  # pyright: ignore[reportUntypedFunctionDecorator]
-def get_symbol_address(symbol_addrs: Path, symbol: str):
-    if match := re.match(r"^ *([^+ ]+) *\+ *(0x[0-9a-fA-F]+)$", symbol):
-        symbol = match.group(1)
-        offset = int(match.group(2), 16)
-    else:
-        offset = 0
-
-    with open(symbol_addrs, mode="r") as fh:
-        for line in fh:
-            if line.startswith(f"{symbol} "):
-                addr = int(line.split("=")[1].split(";")[0].strip(), 16)
-                return addr + offset
-
-    assert False, f"{symbol} not found"
-
-
-def fix_gp(asm_path: Path, gp_value: int, symbol_addrs: Path):
-    if gp_value <= 0:
-        return
-
-    asm_files = glob.glob(str(asm_path / "**/*.s"), recursive=True)
-    for asm_file in tqdm.tqdm(asm_files, desc="Fixing gp_rel"):
-        lines: list[str] = []
-        with open(asm_file, mode="r") as fh:
-            for line in fh:
-                if match := re.match(r"^(.*)%gp_rel\(([^)]+)\)(.*)$", line):
-                    # ol = line
-                    instr_pre = match.group(1)
-                    instr_post = match.group(3)
-                    address_str = match.group(2)
-                    if address_str.startswith("D_"):
-                        address_str = address_str.replace("D_", "0x")
-                        res = eval(address_str)
-                        address = res
-                    else:
-                        address = get_symbol_address(symbol_addrs, address_str)
-                    gp_rel = address - gp_value
-                    line = f"{instr_pre}{hex(gp_rel)}{instr_post}\n"
-                lines.append(line)
-        with open(asm_file, mode="w") as wh:
-            wh.writelines(lines)
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        asm_path: Path
-        gp_value: int
-        symbol_addrs: Path
-
-    def hex_int(x: str):
-        return int(x, 16)
-
-    parser = argparse.ArgumentParser(description="fixes asm removing gp_rel macro")
-    parser.add_argument("asm_path", metavar="asm-path", type=Path, help="assembly root path to patch")
-    parser.add_argument("gp_value", metavar="gp", type=hex_int, help="gp value in hex")
-    parser.add_argument("symbol_addrs", metavar="symbol-addrs", type=Path, help="path of symbol_addrs.txt")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    fix_gp(args.asm_path, args.gp_value, args.symbol_addrs)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_linkerscript.py

@@ -1,139 +0,0 @@
-import re
-import sys
-import yaml
-import tqdm
-
-from typing import cast, Any
-from pathlib import Path
-
-
-# e.g.: build/src/main/main.c.o(.text);
-#       build/asm/data/rodata_4.rodata.s.o(.rodata);
-#       ...
-# NOTE: also account for optional '/data/' in path as chunks of data
-#       not belonging to a c file are put into 'build/asm/data/' folder.
-re_subsegment_line = re.compile(
-    r"^(?P<indent> +)build/(?:asm|src)(?:/data)?/(?P<name>.*)\.[sc]\.o\(\.(?P<section>.+)\);$"
-)
-
-# e.g.: .main 0x100000 : AT(main_ROM_START) SUBALIGN(2)
-#       .main_bss (NOLOAD) : SUBALIGN(4)
-#       ...
-re_section_line = re.compile(r"^(?P<indent> +)\.(?P<section>[^ ]+) .* SUBALIGN\((?P<subalign>[0-9]+)\)$")
-
-
-def get_align(address: int):
-    return (
-        128 * ((address % 128) == 0)
-        or 64 * ((address % 64) == 0)
-        or 32 * ((address % 32) == 0)
-        or 16 * ((address % 16) == 0)
-        or 8 * ((address % 8) == 0)
-        or 4 * ((address % 4) == 0)
-        or 2 * ((address % 2) == 0)
-        or 1
-    )
-
-
-def make_align_map(config: dict[str, Any]):
-    segments = cast(list[dict[str, Any] | list[Any]] | None, config["segments"])
-    assert segments
-    main_segment = next(
-        (segment for segment in segments if isinstance(segment, dict) and segment.get("name") == "main"), None
-    )
-    assert main_segment, "cannot find main segment"
-
-    subsegments = cast(list[dict[str, Any] | list[Any]] | None, main_segment["subsegments"])
-    assert subsegments, "cannot find main subsegments"
-
-    align_map: dict[str, int] = {}
-
-    for subsegment in subsegments:
-        if not isinstance(subsegment, dict):
-            continue
-
-        s_type = cast(str | None, subsegment.get("type"))
-        vram = cast(int | None, subsegment.get("vram"))
-        name = cast(str | None, subsegment.get("name"))
-        if not s_type or not vram or not name:
-            continue
-
-        if name.endswith("bin"):
-            continue
-
-        align = get_align(vram)
-
-        if not s_type.startswith("."):
-            name = f"{s_type}#{name}.{s_type}"
-        else:
-            name = f"{s_type[1:]}#{name}"
-
-        align_map[name] = align
-
-    return align_map
-
-
-def fix_linkerscript(config: dict[str, Any], linkerscript_path: Path):
-    align_map = make_align_map(config)
-
-    section_subalign = cast(dict[str, int], config["_section_subalign"])
-
-    line_count = 0
-    with open(linkerscript_path, mode="r") as fh:
-        for line in fh:
-            line_count += 1
-
-    patched_lines: list[str] = []
-
-    with open(linkerscript_path, mode="r") as fh:
-        for line in tqdm.tqdm(fh, desc="Fixing linker script", total=line_count):
-            if match := re_subsegment_line.match(line):
-                indent = cast(str, match["indent"])
-                name = cast(str, match["name"])
-                section = cast(str, match["section"])
-
-                # force each subsegment in the following sections to have align 8
-                if section == "text":
-                    patched_lines.append(f"{indent}. = ALIGN(., 8);\n")
-
-                key = f"{section}#{name}"
-                if align := align_map.get(key):
-                    patched_lines.append(f"{indent}. = ALIGN(., {align});\n")
-
-            if match := re_section_line.match(line):
-                indent = cast(str, match["indent"])
-                section = cast(str, match["section"])
-                subalign = cast(str, match["subalign"])
-
-                # force each section to have the subalign specified in the yaml
-                if section in section_subalign:
-                    current_subalign = f"SUBALIGN({subalign})"
-                    fixed_subalign = f"SUBALIGN({section_subalign[section]})"
-                    line = line.replace(current_subalign, fixed_subalign)
-
-            patched_lines.append(line)
-
-    with open(linkerscript_path, mode="w") as fh:
-        fh.writelines(patched_lines)
-
-
-def main():
-    if len(sys.argv) != 3:
-        print("usage: fix_linkerscript.py CONFIG_YAML_PATH LINKERSCRIPT_PATH")
-        exit(1)
-
-    config_path = Path(sys.argv[1])
-    linkerscript_path = Path(sys.argv[2])
-
-    with open(config_path, mode="r") as fh:
-        try:
-            config = cast(dict[str, Any], yaml.safe_load(fh))
-        except yaml.YAMLError as e:
-            print(e)
-            raise e
-
-    fix_linkerscript(config, linkerscript_path)
-
-
-if __name__ == "__main__":
-    main()

tools/python/fix_report.py

@@ -1,153 +0,0 @@
-import json
-import argparse
-
-from typing import Protocol, Iterable, Any, cast
-from pathlib import Path
-
-
-def fix_unit(unit: dict[str, Any]):
-    # name: str = unit["name"]
-    fuzzy_match_percent: float | None = unit["measures"].get("fuzzy_match_percent")
-
-    assert fuzzy_match_percent and 0 < fuzzy_match_percent < 100
-
-    text_section = next(section for section in unit["sections"] if section["name"] == ".text")
-    # text_size: int = int(text_section["size"])
-    text_section_fuzzy_match_percent: float = text_section["fuzzy_match_percent"]
-
-    if 0 < text_section_fuzzy_match_percent < 100:
-        # fix text section fuzzy match percent
-        text_section["fuzzy_match_percent"] = 100.0
-
-    text_fuzzy_match_percent: float = unit["measures"]["fuzzy_match_percent"]
-
-    assert text_fuzzy_match_percent == fuzzy_match_percent
-
-    functions = unit["functions"]
-
-    total_code: int = int(unit["measures"]["total_code"])
-    matched_code: int = int(unit["measures"]["matched_code"])
-    matched_functions = unit["measures"]["matched_functions"]
-
-    computed_total_code: int = 0
-    computed_matched_code: int = 0
-    computed_matched_functions: int = 0
-
-    duplicate_functions = (
-        "Tim2CalcBufWidth__2",
-        "_ftoi0__2",
-        "ItemGetMain__2",
-        "setD3_CHCR__2",
-        "setD4_CHCR__2",
-        "setD4_CHCR__3",
-        "_fpadd_parts__2",
-    )
-
-    for function in functions:
-        function_size: int = int(function["size"])
-        try:
-            function_fuzzy_match_percent: float = function["fuzzy_match_percent"]
-        except KeyError:
-            # fix known function duplicates
-            if function["name"] in duplicate_functions:
-                function["fuzzy_match_percent"] = 100.0
-                function_fuzzy_match_percent = 100.0
-                matched_code += function_size
-                matched_functions += 1
-            else:
-                raise
-
-        if function_fuzzy_match_percent == 100.0:
-            computed_matched_code += function_size
-            computed_matched_functions += 1
-
-        # fix function fuzzy match percent
-        function["fuzzy_match_percent"] = 100.0
-
-        computed_total_code += function_size
-
-    assert total_code == computed_total_code
-    assert matched_code == computed_matched_code
-    assert matched_functions == computed_matched_functions
-
-    # fix unit measures
-    unit["measures"]["fuzzy_match_percent"] = 100.0
-    unit["measures"]["matched_code"] = unit["measures"]["total_code"]
-    unit["measures"]["matched_code_percent"] = 100.0
-    unit["measures"]["matched_functions"] = unit["measures"]["total_functions"]
-    unit["measures"]["matched_functions_percent"] = 100.0
-
-
-def fix_report(report_path: Path):
-    report = json.loads(report_path.read_text())
-
-    units: Iterable[Any] = report["units"]
-
-    computed_total_code: int = 0
-    computed_matched_code: int = 0
-
-    computed_total_functions: int = 0
-    computed_matched_functions: int = 0
-
-    total_code: int = int(report["measures"]["total_code"])
-    total_functions: int = report["measures"]["total_functions"]
-
-    for unit in units:
-        # name: str = unit["name"]
-        unit_total_code: int = int(unit["measures"]["total_code"])
-        unit_total_functions: int = unit["measures"]["total_functions"]
-        fuzzy_match_percent: float | None = unit["measures"].get("fuzzy_match_percent")
-
-        if fuzzy_match_percent and 0 < fuzzy_match_percent < 100:
-            fix_unit(unit)
-
-        if fuzzy_match_percent and fuzzy_match_percent > 0:
-            computed_matched_code += unit_total_code
-            computed_matched_functions += unit_total_functions
-
-        computed_total_functions += unit_total_functions
-        computed_total_code += unit_total_code
-
-    assert total_code == computed_total_code
-    assert total_functions == computed_total_functions
-
-    # fix report measures
-    report["measures"]["fuzzy_match_percent"] = 100.0 * computed_matched_code / computed_total_code
-    report["measures"]["matched_code"] = str(computed_matched_code)
-    report["measures"]["matched_code_percent"] = report["measures"]["fuzzy_match_percent"]
-    report["measures"]["matched_functions"] = computed_matched_functions
-    report["measures"]["matched_functions_percent"] = 100.0 * computed_matched_functions / computed_total_functions
-
-    categories = report["categories"]
-    assert len(categories) == 1
-    assert categories[0]["measures"]["total_code"] == report["measures"]["total_code"]
-    assert categories[0]["measures"]["total_units"] == report["measures"]["total_units"]
-
-    categories[0]["measures"]["fuzzy_match_percent"] = report["measures"]["fuzzy_match_percent"]
-    categories[0]["measures"]["matched_code"] = report["measures"]["matched_code"]
-    categories[0]["measures"]["matched_code_percent"] = report["measures"]["matched_code_percent"]
-    categories[0]["measures"]["matched_functions"] = report["measures"]["matched_functions"]
-    categories[0]["measures"]["matched_functions_percent"] = report["measures"]["matched_functions_percent"]
-
-    # /path/to/report.json -> /path/to/report_fixed.json
-    # fixed_report_path = report_path.with_name(f"{report_path.stem}_fixed{report_path.suffix}")
-    # fixed_report_path.write_text(json.dumps(report))
-    report_path.write_text(json.dumps(report))
-
-    print(f"Wrote fixed report to {report_path}")
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        report_path: Path
-
-    parser = argparse.ArgumentParser(description="fixes objdiff report")
-    parser.add_argument("report_path", metavar="report-path", type=Path, help="path to the report generated by objdiff")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    fix_report(args.report_path)
-
-
-if __name__ == "__main__":
-    main()

tools/python/map_mismatch.py

@@ -1,32 +0,0 @@
-import re
-import argparse
-
-from pathlib import Path
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--language", required=True, choices=["us", "eu"])
-
-    args = parser.parse_args()
-
-    if args.language == "us":
-        map_path = Path("config/us/build/SLUS_203.88.map")
-    else:
-        map_path = Path("config/eu/build/SLES_508.21.map")
-
-    with open(map_path, mode="r") as fh:
-        for n, line in enumerate(fh):
-            line = line.rstrip("\n")
-            if match := re.match(r"^\s*0x([0-9a-fA-F]+)\s+[^ ]+?([0-9a-fA-F]{6,8})\s*$", line):
-                addr = match.group(1)
-                label = match.group(2)
-                if addr.upper() != label.upper():
-                    print(f"{map_path}:{n+1} {line}")
-                    return
-
-    print("no mismatches found")
-
-
-if __name__ == "__main__":
-    main()

tools/python/parse_data.py

@@ -1,438 +0,0 @@
-# pyright: reportInvalidTypeForm=false
-from __future__ import annotations
-
-import os
-import re
-import io
-import enum
-import math
-import tqdm
-import argparse
-import pydantic
-import subprocess
-
-from ctypes import c_uint16, c_int16, c_uint64, c_uint32, c_int32, c_uint8, c_float, sizeof as c_sizeof
-from typing import Protocol, BinaryIO, Type, Iterable, cast
-from pathlib import Path
-
-from cstruct import CStructure, ctypes_types, CTypeType, c_addr_ptr, c_str, print_carr
-
-
-def vram2offset(vram: int):
-    return vram - 0xFF000
-
-
-sceVu0FVECTOR = c_float * 4
-sceVu0IVECTOR = c_int32 * 4
-sceVu0FMATRIX = sceVu0FVECTOR * 4
-
-qword = c_int32 * 4
-
-
-class ANI_CODE(c_uint16):
-    def __str__(self):
-        return f"{self.value}"
-
-
-class GS_REG(c_uint64):
-    def __str__(self):
-        return f"0x{self.value:x}"
-
-
-class HEX_ADDR(c_uint32):
-    def __str__(self):
-        return f"0x{self.value:x}"
-
-
-# CTypeTypeEX = CTypeType | type[sceVu0FVECTOR] | type[sceVu0IVECTOR]
-
-
-###########################################################################
-
-# struct _MNG_STATE_ST {
-#     // Members
-#     int state; // offset 0x0, size 0x4
-#     u_int flag; // offset 0x4, size 0x4
-#     u_int calc_total_time; // offset 0x8, size 0x4
-#     u_int total_time; // offset 0xC, size 0x4
-#     u_int get_time; // offset 0x10, size 0x4
-#     u_int dmy[3]; // offset 0x14, size 0xC
-# };
-class MNG_STATE_ST(CStructure):
-    state: c_int32
-    flag: c_uint32
-    calc_total_time: c_uint32
-    total_time: c_uint32
-    get_time: c_uint32
-    flag: c_uint32
-    dmy: c_uint32 * 3
-
-
-# struct _TMP_MENU_MNG_ST {
-#     // Members
-#     struct _MNG_STATE_ST mng_state; // offset 0x0, size 0x20
-#     int DrawState; // offset 0x20, size 0x4
-#     u_char** pTmpMenuFileBuf; // offset 0x24, size 0x4
-#     float draw_up_rate; // offset 0x28, size 0x4
-#     u_char dmy[12]; // offset 0x2C, size 0xC
-# };
-class TMP_MENU_MNG_ST(CStructure):
-    mng_state: MNG_STATE_ST
-    DrawState: c_int32
-    pTmpMenuFileBuf: c_addr_ptr
-    draw_up_rate: c_float
-    dmy: c_uint8 * 12
-
-###########################################################################
-
-elf_names: dict[str, str] = {
-    "us": "SLUS_203.88",
-    "eu": "SLES_508.21",
-}
-
-
-class VRamElf(BinaryIO):
-    def __init__(self, elf: BinaryIO):
-        self.elf = elf
-
-    def read(self, n: int = -1):
-        return self.elf.read(n)
-
-    def seek(self, offset: int, whence: int = 0):
-        return self.elf.seek(vram2offset(offset), whence)
-
-
-# matches "var_name = 0x12345678; // attr1:val1 attr2:val2 ... attrN:valN"
-line_pattern = re.compile(r"^([^\s=]+)\s*=\s*(0x[0-9a-fA-F]+);\s*\/\/((?:\s*[0-9a-zA-Z_]+(:?:[0-9a-zA-Z_\*]+)?)+)\s*$")
-# matches multiple attributes in the form of attr:val or attr
-attr_pattern = re.compile(r"([0-9a-zA-Z_]+)(?::([0-9a-zA-Z_\*]+))?")
-# match address lines: "0xaabbccdd = &var"
-addr_pattern = re.compile(r"^\s*(0x[0-9a-fA-F]+)\s*=\s*(.*)\s*;\s*(\/\/.*)?$")
-
-
-class DataVar(pydantic.BaseModel):
-    model_config = pydantic.ConfigDict(extra="forbid")
-
-    _elf: VRamElf = pydantic.PrivateAttr()
-
-    address: int
-    name: str
-    type: Type[CStructure] | CTypeType | type[sceVu0FVECTOR] | type[sceVu0IVECTOR] | type[qword] | type[c_str]
-    numel: int | list[int] = 0
-    nosize: bool = False
-    static: bool = False
-
-    num_ptr: int = 0
-
-    @pydantic.model_validator(mode="after")
-    def store_num_ptr(self):
-        # transfer class attribute to instance
-        if DataVar.num_ptr > 0:
-            self.num_ptr = DataVar.num_ptr
-        return self
-
-    @pydantic.field_validator("type", mode="before")
-    @classmethod
-    def type_from_str(  # pyright: ignore
-        cls, v: str | Type[CStructure] | CTypeType
-    ) -> Type[CStructure] | CTypeType | sceVu0FVECTOR | sceVu0IVECTOR | qword | ANI_CODE | GS_REG | HEX_ADDR:
-        if not isinstance(v, str):
-            return v
-        cls.num_ptr = v.count("*")  # temporary store num_ptr in class attribute
-        v = v.rstrip("*")  # remove pointer(s)
-        if v in ctypes_types:
-            return ctypes_types[v]
-        if v == "sceVu0FVECTOR":
-            return sceVu0FVECTOR
-        if v == "sceVu0IVECTOR":
-            return sceVu0IVECTOR
-        if v == "qword":
-            return qword
-        if v == "ANI_CODE":
-            return ANI_CODE
-        if v == "GS_REG":
-            return GS_REG
-        if v == "HEX_ADDR":
-            return HEX_ADDR
-        if v == "c_str":
-            return c_str
-        class_type = globals()[v]
-        if issubclass(class_type, CStructure):
-            return class_type
-        raise ValueError(f"{v} is unknown/not a valid type")
-
-    def data_var_dumps(self, addresses: dict[int, str]):
-        self._elf.seek(self.address, os.SEEK_SET)
-
-        if isinstance(self.numel, list):
-            numel = math.prod(self.numel)
-            ndims = len(self.numel)
-        else:
-            numel = max(1, self.numel)
-            ndims = 1
-        if self.num_ptr > 0:
-            var_data = (c_addr_ptr * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(c_addr_ptr)))
-            if self.type == ANI_CODE:  # pyright: ignore
-                type_str = "ANI_CODE" + ("*" * self.num_ptr)
-            if self.type == GS_REG:  # pyright: ignore
-                type_str = "u_long" + ("*" * self.num_ptr)
-            if self.type == HEX_ADDR:  # pyright: ignore
-                type_str = "u_int" + ("*" * self.num_ptr)
-            elif issubclass(self.type, CStructure):  # pyright: ignore
-                type_str = self.type.__name__ + ("*" * self.num_ptr)
-            elif self.type == sceVu0FVECTOR:  # pyright: ignore
-                type_str = "sceVu0FVECTOR" + ("*" * self.num_ptr)
-            elif self.type == sceVu0IVECTOR:  # pyright: ignore
-                type_str = "sceVu0IVECTOR" + ("*" * self.num_ptr)
-            else:
-                type_str = next(
-                    k
-                    for k, v in ctypes_types.items()
-                    if getattr(v, "_type_") == getattr(self.type, "_type_")  # pyright: ignore
-                ) + ("*" * self.num_ptr)
-            stream = io.StringIO()
-            if self.static:
-                stream.write("static ")
-            stream.write(f"{type_str} {self.name}")
-            if numel > 1:
-                numel = f"{self.numel}" if not self.nosize else ""
-                var_str = f"{{ {', '.join(str(var) for var in cast(Iterable[c_addr_ptr], var_data))}, }}"  # pyright: ignore
-                stream.write(f"[{numel}]")
-            else:
-                # should have been parsed as c_float_Array_4_Array_1
-                assert len(var_data) == 1 and var_data[0].__class__ is c_addr_ptr
-                var_str = str(var_data[0])
-            stream.write(f" = {var_str};")
-        elif issubclass(self.type, CStructure):  # pyright: ignore
-            data = self._elf.read(numel * self.type.sizeof())
-
-            return self.type.dumps(
-                self.name,
-                data,
-                numel=self.numel,
-                static=self.static,
-                nosize=self.nosize,
-                noarray=self.numel == 0,
-            )
-        elif self.type == sceVu0FVECTOR:  # pyright: ignore[reportUnknownMemberType]
-            var_data = (sceVu0FVECTOR * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(sceVu0FVECTOR)))
-            type_str = "sceVu0FVECTOR"
-            stream = io.StringIO()
-            if self.static:
-                stream.write("static ")
-            stream.write(f"{type_str} {self.name}")
-
-            def sceVu0FVECTOR_to_str(_v: sceVu0FVECTOR):  # pyright: ignore
-                return f"{{ {', '.join(f'{flt}f' for flt in cast(Iterable[c_float], _v))} }}"
-
-            if numel > 1:
-                numel = f"{self.numel}" if not self.nosize else ""
-                var_str = (
-                    f"{{ {', '.join(sceVu0FVECTOR_to_str(var) for var in cast(Iterable[sceVu0FVECTOR], var_data))}, }}"  # pyright: ignore
-                )
-                stream.write(f"[{numel}]")
-            else:
-                # should have been parsed as c_float_Array_4_Array_1
-                assert len(var_data) == 1 and var_data[0].__class__ is sceVu0FVECTOR
-                var_str = sceVu0FVECTOR_to_str(var_data[0])
-            stream.write(f" = {var_str};")
-        elif self.type == sceVu0IVECTOR:  # pyright: ignore[reportUnknownMemberType]
-            var_data = (sceVu0IVECTOR * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(sceVu0IVECTOR)))
-            type_str = "sceVu0IVECTOR"
-            stream = io.StringIO()
-            if self.static:
-                stream.write("static ")
-            stream.write(f"{type_str} {self.name}")
-
-            def sceVu0IVECTOR_to_str(_v: sceVu0IVECTOR):  # pyright: ignore
-                return f"{{ {', '.join(f'{int32}' for int32 in cast(Iterable[c_int32], _v))} }}"
-
-            if numel > 1:
-                numel = f"{self.numel}" if not self.nosize else ""
-                var_str = (
-                    f"{{ {', '.join(sceVu0IVECTOR_to_str(var) for var in cast(Iterable[sceVu0IVECTOR], var_data))}, }}"  # pyright: ignore
-                )
-                stream.write(f"[{numel}]")
-            else:
-                # should have been parsed as c_int_Array_4_Array_1
-                assert len(var_data) == 1 and var_data[0].__class__ is sceVu0IVECTOR
-                var_str = sceVu0IVECTOR_to_str(var_data[0])
-            stream.write(f" = {var_str};")
-        elif self.type == c_str:  # pyright: ignore
-            assert numel > 1, "we only handle string arrays here, simple strings should be embedded in the source"
-            var_data = (self.type * numel).from_buffer_copy(self._elf.read(numel * c_sizeof(self.type)))  # pyright: ignore
-            stream = io.StringIO()
-            if self.static:
-                stream.write("static ")
-            numel = f"{self.numel}" if not self.nosize else ""
-            stream.write(f"char *{self.name}[{numel}] = {{")
-            for var in cast(Iterable[c_str], var_data):
-                stream.write(f"{var.to_str(self._elf)},")
-            stream.write("};")
-        else:
-            typ = self.type  # pyright: ignore
-            if ndims == 1:
-                typ *= numel  # pyright: ignore
-            elif isinstance(self.numel, list):
-                for n in reversed(self.numel):
-                    typ *= n  # pyright: ignore
-            var_data = (typ).from_buffer_copy(self._elf.read(numel * c_sizeof(self.type)))  # pyright: ignore
-            if self.type.__name__ == "ANI_CODE":  # pyright: ignore
-                type_str = "ANI_CODE"
-            elif self.type.__name__ == "GS_REG":  # pyright: ignore
-                type_str = "u_long"
-            elif self.type.__name__ == "HEX_ADDR":  # pyright: ignore
-                type_str = "u_int"
-            else:
-                type_str = next(
-                    k
-                    for k, v in ctypes_types.items()
-                    if getattr(v, "_type_") == getattr(self.type, "_type_")  # pyright: ignore
-                )
-            stream = io.StringIO()
-            if self.static:
-                stream.write("static ")
-            stream.write(f"{type_str} {self.name}")
-            if ndims == 1:
-                numel = f"{self.numel}" if not self.nosize else ""
-                var_str = f"{{ {', '.join(f'{var}' for var in cast(Iterable[CTypeType], var_data))} }}"
-                stream.write(f"[{numel}]")
-            elif ndims > 1:
-                assert isinstance(self.numel, list)
-                numel = ""
-                for n, num in enumerate(self.numel):
-                    numel += "[]" if n == 0 and self.nosize else f"[{num}]"
-                stream.write(numel)
-                tmpbuf = io.StringIO()
-                tmpbuf.write("{")
-                print_carr(var_data, tmpbuf)  # pyright: ignore
-                tmpbuf.write("}")
-                var_str = tmpbuf.getvalue()
-            else:
-                var_str = f"{var_data.value}"  # pyright: ignore
-            stream.write(f" = {var_str};")
-        return stream.getvalue()
-
-
-def parse_data_vars(data_vars_txt: Path, strict: bool = True):
-    data_vars: list[DataVar] = []
-    addr_vals: dict[int, str] = {}
-
-    with open(data_vars_txt, mode="r") as fh:
-        for n, line in enumerate(fh):
-            line = line.strip()
-
-            # skip empty lines and comments
-            if not line or line.startswith("//"):
-                continue
-
-            valid = False
-
-            if addr_match := addr_pattern.match(line):
-                address = int(addr_match.group(1), 16)
-                value = addr_match.group(2)
-                addr_vals[address] = value
-                valid = True
-
-            elif line_match := line_pattern.match(line):
-                address = int(line_match.group(2), 16)
-                options = line_match.group(3)
-                name = line_match.group(1)
-                if attr_match := attr_pattern.findall(options):
-                    attrs: dict[str, str | list[str] | bool] = {}
-                    for attr_name, attr_val in cast(list[tuple[str, str]], attr_match):
-                        if attr_name not in attrs:
-                            attrs[attr_name] = attr_val
-                        elif not isinstance(attrs[attr_name], list):
-                            attrs[attr_name] = cast(list[str], [attrs[attr_name], attr_val])
-                        else:
-                            cast(list[str], attrs[attr_name]).append(attr_val)
-                    for key, value in attrs.items():
-                        if not isinstance(value, bool) and len(value) == 0:
-                            attrs[key] = True
-                    try:
-                        data_var = DataVar(address=address, name=name, **attrs)  # pyright: ignore[reportArgumentType]
-                        data_vars.append(data_var)
-                        if address not in addr_vals:
-                            if (
-                                isinstance(data_var.numel, list) or data_var.numel > 0 or data_var.type == sceVu0FVECTOR  # type: ignore
-                            ):
-                                addr_vals[address] = name
-                            else:
-                                addr_vals[address] = f"&{name}"
-                        valid = True
-                    except pydantic.ValidationError as ve:
-                        print(f"syntax error at line {n}")
-                        raise ve
-
-            if strict and not valid:
-                raise ValueError(f"syntax error at line {n}: {line}")
-
-    return data_vars, addr_vals
-
-
-def parse_data(lang: str):
-    data_vars_txt = Path("config") / lang / "data_vars.txt"
-
-    data_vars, addr_vals = parse_data_vars(data_vars_txt)
-
-    if not data_vars:
-        return
-
-    if len(set(data_var.name for data_var in data_vars)) != len(data_vars):
-        raise RuntimeError("duplicate names in data_vars.txt")
-
-    elf_path = Path("config") / lang / elf_names[lang]
-
-    include_path = Path("config") / lang / "include" / "data"
-    include_path.mkdir(parents=True, exist_ok=True)
-
-    c_addr_ptr.set_addresses(addr_vals)
-
-    extract_filter: list[str] = []
-    extract_filter_str = os.environ.get("EXTRACT_DATA_FILTER", "").strip()
-    if extract_filter_str:
-        extract_filter: list[str] = extract_filter_str.split(",")
-
-    with open(elf_path, mode="rb") as elf:
-        vram_elf = VRamElf(elf)
-        CStructure.__elf__ = vram_elf
-
-        for data_var in tqdm.tqdm(data_vars, desc="Extracting data"):
-            header_path = include_path / f"{data_var.name}.h"
-
-            if extract_filter and data_var.name not in extract_filter:
-                continue
-
-            with header_path.open(mode="w") as fw:
-                data_var._elf = vram_elf  # pyright: ignore[reportPrivateUsage]
-
-                fw.write(data_var.data_var_dumps(addr_vals))
-
-            try:
-                subprocess.run(["clang-format", "-i", "--style=Microsoft", header_path])
-            except Exception:
-                # formatting is best effort for now
-                pass
-
-
-def main():
-    class Language(str, enum.Enum):
-        US = "us"
-        EU = "eu"
-
-    languages = [lang.value for lang in Language]
-
-    class ArgsProtocol(Protocol):
-        language: Language
-
-    parser = argparse.ArgumentParser(description="fixes asm removing gp_rel macro")
-    parser.add_argument("language", type=str, choices=languages, help="language to extract data for")
-
-    args = cast(ArgsProtocol, parser.parse_args())
-
-    parse_data(args.language)
-
-
-if __name__ == "__main__":
-    main()

tools/python/parse_globals.py

@@ -1,288 +0,0 @@
-# pyright: reportUnknownMemberType=false
-
-import re
-import argparse
-
-from typing import cast
-from pathlib import Path
-from elftools.elf.elffile import ELFFile
-from elftools.elf.sections import SymbolTableSection
-
-
-"""
-matches variable declaration:
-/* SECTION ADDRESS */ VAR_TYPE NAME[NUMEL];
-"""
-re_glob = re.compile(
-    r"^/\* (?P<section>.*?) (?P<address>.*?) \*/ (?P<var_type>[^(]*) (?P<name>.*?)(?:\[(?P<numel>.*?)\])?;"
-)
-
-"""
-matches structs (or unions) with no typedef:
-struct NAME { // SIZE
-    ...
-};
-"""
-re_struct = re.compile(
-    r"^(?:struct|union) (?P<name>.*?) \{ // (?P<size>0x[0-9a-f]+)\n.*?^\};", flags=re.MULTILINE | re.DOTALL
-)
-
-"""
-matches structs (or unions) with typedef:
-typedef struct { // SIZE
-    ...
-} NAME;
-"""
-re_typedef_struct = re.compile(
-    r"^typedef (?:struct|union) \{ // (?P<size>0x[0-9a-f]+)\n.*?^\} (?P<name>.*?);", flags=re.MULTILINE | re.DOTALL
-)
-
-sizes: dict[str, int] = {
-    "char": 1,
-    "u_char": 1,
-    "short": 2,
-    "short int": 2,
-    "u_short": 2,
-    "int": 4,
-    "u_int": 4,
-    "float": 4,
-    "u_long128": 8,
-    "sceVu0FMATRIX": 4 * 4 * 4,
-    "sceVu0FVECTOR": 4 * 4,
-    "sceSifClientData": 0x2C,
-}
-
-command_script_keywords = (
-    "VERSION",
-    "SECTIONS",
-    "ABSOLUTE",
-    "LOADADDR",
-    "ALIGN",
-    "DEFINED",
-    "NEXT",
-    "SIZEOF",
-    "SIZEOF_HEADERS",
-    "MAX",
-    "MIN",
-    "PHDRS",
-    "CREATE_OBJECT_SYMBOLS",
-    "BYTE",
-    "SHORT",
-    "LONG",
-    "SQUAD",
-    "FILL",
-    "BLOCK",
-    "NOLOAD",
-    "AT",
-    "OVERLAY",
-    "NOCROSSREFS",
-    "PT_NULL",
-    "PT_LOAD",
-    "PT_DYNAMIC",
-    "PT_INTERP",
-    "PT_NOTE",
-    "PT_SHLIB",
-    "PT_PHDR",
-    "ENTRY",
-    "FLOAT",
-    "NOFLOAT",
-    "FORCE_COMMON_ALLOCATION",
-    "INCLUDE",
-    "INPUT",
-    "GROUP",
-    "OUTPUT",
-    "OUTPUT_ARCH",
-    "OUTPUT_FORMAT",
-    "SEARCH_DIR",
-    "STARTUP",
-    "TARGET",
-    "NOCROSSREFS",
-)
-
-
-class GlobalVarLineMatch:
-    max_section_len: int = 0
-    max_address_len: int = 0
-    max_var_type_len: int = 0
-    max_name_len: int = 0
-    max_numel_len: int = 0
-
-    _name_cache: list[str] = []
-
-    @classmethod
-    def _get_unique_name(cls, name: str):
-        i = 1
-        unique_name = f"{name}__local_{i}"
-        while unique_name in cls._name_cache:
-            i += 1
-            unique_name = f"{name}__local_{i}"
-        cls._name_cache.append(unique_name)
-        return unique_name
-
-    def __init__(
-        self,
-        symtab: SymbolTableSection,
-        /,
-        *,
-        section: str,
-        address: str | int,
-        var_type: str,
-        name: str,
-        numel: str | int | None = None,
-    ):
-        self.symtab: SymbolTableSection = symtab
-
-        self.section: str = section
-        self.address: int = address if isinstance(address, int) else int(address, 16)
-        self.var_type: str = var_type
-        self.name: str = name
-        self.numel: int
-
-        while self.name.startswith("*"):
-            self.name = self.name[1:]
-            self.var_type += "*"
-
-        # resolve function pointers (e.g., 'void (*SpecialEventInitTbl[0])...')
-        if match := re.match(r"\(\*(.*?)\[.*?\]\).*", self.name):
-            self.name = match.group(1)
-
-        self.name = self.name.replace("*", "")
-
-        if numel is not None:
-            if isinstance(numel, int):
-                self.numel = numel
-            else:
-                prod_numel = 1
-                numels = numel.split("][")
-                for n in numels:
-                    prod_numel *= int(n)
-                self.numel = prod_numel
-
-        if numel is None:
-            self.numel = 1
-
-        # get symbol
-        symbol = self.symtab.get_symbol_by_name(self.name)
-
-        assert isinstance(symbol, list)
-
-        symbol = next(
-            (sym for sym in symbol if sym.name == self.name and sym.entry["st_value"] == self.address),
-            None,
-        )
-
-        assert symbol
-
-        self.symbol = symbol
-
-        if not self.is_global:
-            self.name = self._get_unique_name(self.name)
-
-        escaped_name_len = len(str(self.name))
-        if self.name in command_script_keywords:
-            escaped_name_len += 2
-
-        GlobalVarLineMatch.max_section_len = max(GlobalVarLineMatch.max_section_len, len(str(self.section)))
-        GlobalVarLineMatch.max_address_len = max(GlobalVarLineMatch.max_address_len, len(str(self.address)))
-        GlobalVarLineMatch.max_var_type_len = max(GlobalVarLineMatch.max_var_type_len, len(str(self.var_type)))
-        GlobalVarLineMatch.max_name_len = max(GlobalVarLineMatch.max_name_len, escaped_name_len)
-        GlobalVarLineMatch.max_numel_len = max(GlobalVarLineMatch.max_numel_len, len(str(self.numel)))
-
-    @property
-    def is_global(self):
-        return cast(str, self.symbol["st_info"]["bind"]) == "STB_GLOBAL"
-
-    @property
-    def is_hidden(self):
-        return cast(str, self.symbol["st_other"]["visibility"]) == "STV_HIDDEN"
-
-    @property
-    def size(self):
-        size = cast(int, self.symbol["st_size"])
-        if size == 0:
-            if "*" in self.var_type:
-                size = 4
-            elif self.var_type in sizes:
-                size = sizes[self.var_type] * self.numel
-        return size
-
-    def to_string(self, as_linker_command_file: bool):
-        name = self.name if self.name not in command_script_keywords else f'"{self.name}"'
-        cls_str = f"{name:{GlobalVarLineMatch.max_name_len}s} = 0x{self.address:08x};"
-
-        if not as_linker_command_file:
-            if self.size > 0:
-                cls_str = f"{cls_str} // size:0x{self.size:x}"
-            else:
-                cls_str = f"{cls_str} //0 {self.var_type} * {self.numel}"
-
-            # cls_str += f" bind:{'global' if self.is_global else 'local'}"
-            # cls_str += f" visibility:{'hidden' if self.is_hidden else 'visible'}"
-
-        return cls_str
-
-    def __str__(self):
-        return self.to_string(as_linker_command_file=False)
-
-
-def parse_globals(elf_path: Path, globals_path: Path, types_path: Path, as_linker_command_file: bool):
-    with open(elf_path, mode="rb") as fh:
-        elf = ELFFile(fh)
-
-        # Find the symbol table.
-        symtab = elf.get_section_by_name(".symtab")
-        assert isinstance(symtab, SymbolTableSection)
-
-        with open(types_path, mode="r") as f:
-            types_data = f.read()
-
-            for struct_type, size_hex_str in re_struct.findall(types_data):
-                struct_type = cast(str, struct_type)
-                struct_size = int(cast(str, size_hex_str), 16)
-                # assert not (struct_type in sizes and sizes[struct_type] != struct_size)
-                sizes[struct_type] = struct_size
-
-            for size_hex_str, struct_type in re_typedef_struct.findall(types_data):
-                struct_type = cast(str, struct_type)
-                struct_size = int(cast(str, size_hex_str), 16)
-                # assert not (struct_type in sizes and sizes[struct_type] != struct_size), (
-                #     struct_type,
-                #     sizes[struct_type],
-                #     struct_size,
-                # )
-                sizes[struct_type] = struct_size
-
-            if "tagSE_WRK" in sizes:
-                sizes["SE_WRK"] = sizes["tagSE_WRK"]
-
-        with open(globals_path, mode="r") as f:
-            lines = [line.strip() for line in f.readlines() if line.startswith("/*") and line.strip().endswith(";")]
-
-        matches = [GlobalVarLineMatch(symtab, **match.groupdict()) for line in lines if (match := re_glob.match(line))]
-        matches.sort(key=lambda match: match.address)
-
-        for match in matches:
-            print(match.to_string(as_linker_command_file=as_linker_command_file))
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--language", required=True, choices=["us", "eu"])
-    parser.add_argument("--as-linker-command-file", action="store_true")
-
-    args = parser.parse_args()
-
-    if args.language == "us":
-        elf_path = Path("config/us/SLUS_203.88")
-        globals_path = Path("ccc/SLUS_203.88/globals.h")
-        types_path = Path("ccc/SLUS_203.88/types.h")
-    else:
-        elf_path = Path("config/eu/SLES_508.21")
-        globals_path = Path("ccc/SLES_508.21/globals.h")
-        types_path = Path("ccc/SLES_508.21/types.h")
-
-    parse_globals(elf_path, globals_path, types_path, args.as_linker_command_file)
-
-
-if __name__ == "__main__":
-    main()

tools/python/parse_symbols.py

@@ -1,378 +0,0 @@
-# pyright: reportUnknownMemberType=false
-
-from __future__ import annotations
-
-import re
-import json
-import argparse
-import itertools
-
-from typing import Protocol, TextIO, cast
-from pathlib import Path
-from elftools.elf.elffile import ELFFile
-from elftools.elf.sections import SymbolTableSection, Symbol
-
-import ccc_json_v7
-
-Range = tuple[int, int]
-
-command_script_keywords = (
-    "VERSION",
-    "SECTIONS",
-    "ABSOLUTE",
-    "LOADADDR",
-    "ALIGN",
-    "DEFINED",
-    "NEXT",
-    "SIZEOF",
-    "SIZEOF_HEADERS",
-    "MAX",
-    "MIN",
-    "PHDRS",
-    "CREATE_OBJECT_SYMBOLS",
-    "BYTE",
-    "SHORT",
-    "LONG",
-    "SQUAD",
-    "FILL",
-    "BLOCK",
-    "NOLOAD",
-    "AT",
-    "OVERLAY",
-    "NOCROSSREFS",
-    "PT_NULL",
-    "PT_LOAD",
-    "PT_DYNAMIC",
-    "PT_INTERP",
-    "PT_NOTE",
-    "PT_SHLIB",
-    "PT_PHDR",
-    "ENTRY",
-    "FLOAT",
-    "NOFLOAT",
-    "FORCE_COMMON_ALLOCATION",
-    "INCLUDE",
-    "INPUT",
-    "GROUP",
-    "OUTPUT",
-    "OUTPUT_ARCH",
-    "OUTPUT_FORMAT",
-    "SEARCH_DIR",
-    "STARTUP",
-    "TARGET",
-    "NOCROSSREFS",
-)
-
-skip_symbols = (
-    "_fbss",
-    "_gp",
-)
-
-size_exceptions = {
-    "dorcon": 0x1C,
-}
-
-
-def parse_types(ccc_model_v7: ccc_json_v7.CCCJSONv7Model):
-    type_map: dict[str, int] = {}
-
-    for n, dt in enumerate(ccc_model_v7.deduplicated_types):
-        if dt.descriptor == "builtin":
-            assert dt.name and dt.name not in type_map
-            assert dt.class_ is not None
-            size_bits = int(dt.class_.split("-", maxsplit=1)[0])
-            assert size_bits % 8 == 0
-            type_map[dt.name] = size_bits // 8
-
-        elif dt.descriptor == "type_name":
-            assert dt.name is not None
-            assert dt.type_name is not None
-            if dt.name in type_map:
-                if dt.size_bits:
-                    assert type_map[dt.name] == dt.size_bits // 8
-            if dt.name == dt.type_name == "void":
-                type_map["void"] = type_map["int"]
-                continue
-            assert dt.type_name in type_map, (n, dt.type_name)
-            type_map[dt.name] = type_map[dt.type_name]
-
-        elif dt.descriptor == "pointer":
-            assert dt.name is not None
-            assert dt.value_type is not None
-            if dt.value_type.descriptor == "function_type":
-                assert dt.name not in type_map
-                type_map[dt.name] = type_map["int"]
-            elif dt.value_type.descriptor == "type_name":
-                assert dt.value_type.type_name
-                assert dt.value_type.type_name in type_map
-                type_map[dt.name] = type_map[dt.value_type.type_name]
-
-        elif dt.descriptor == "struct":
-            assert dt.name is not None
-            if not dt.conflict:
-                assert dt.name not in type_map, n
-            else:
-                if dt.name in type_map:
-                    continue
-            assert dt.size_bits is not None
-            assert dt.size_bits % 8 == 0
-            type_map[dt.name] = dt.size_bits // 8
-
-        elif dt.descriptor == "array":
-            assert dt.name is not None
-            element_count = 1
-            element_type = dt.element_type
-            type_name = None
-            while element_type:
-                if element_type.element_count is not None:
-                    element_count *= element_type.element_count
-                if element_type.type_name is not None:
-                    type_name = element_type.type_name
-                    assert type_name in type_map
-                    element_count *= type_map[type_name]
-                element_type = element_type.element_type
-            assert type_name and type_name in type_map, n
-            type_map[dt.name] = type_map[type_name] * element_count
-
-        elif dt.descriptor == "enum":
-            if dt.name:
-                if not dt.conflict:
-                    assert dt.name not in type_map, n
-                type_map[dt.name] = type_map["int"]
-
-        elif dt.descriptor == "union":
-            assert dt.name
-            assert dt.size_bits
-            assert dt.size_bits % 8 == 0
-            type_map[dt.name] = dt.size_bits // 8
-
-        else:
-            assert False, f"unknown {n}"
-
-    return type_map
-
-
-def in_range(ranges: list[Range], address: int):
-    if not ranges:
-        return True
-
-    in_range = any((ra[0] <= address <= ra[1]) for ra in ranges)
-
-    return in_range
-
-
-def parse_ccc_model_v7(ccc_model_v7: ccc_json_v7.CCCJSONv7Model, ranges: list[Range]):
-    type_sizes = parse_types(ccc_model_v7)
-
-    if "pointer" not in type_sizes:
-        type_sizes["pointer"] = type_sizes["int"]
-
-    static_locals: list[ccc_json_v7.Local] = []
-    global_vars: list[ccc_json_v7.Global] = []
-
-    for file in ccc_model_v7.files:
-        for global_var in file.globals:
-            assert global_var.storage.global_address
-            if in_range(ranges, global_var.storage.global_address):
-                global_vars.append(global_var)
-
-        for function in file.functions:
-            for local in function.locals:
-                if local.storage_class == "static":
-                    assert local.storage.global_address
-                    if in_range(ranges, local.storage.global_address):
-                        static_locals.append(local)
-
-    return type_sizes, global_vars, static_locals
-
-
-class SymbolWithNoNameException(Exception): ...
-
-
-class ParsedSymbol:
-    _name_map: dict[str, list[ParsedSymbol]] = {}
-    _max_name_len: int = 0
-
-    def __init__(self, symtab: SymbolTableSection, symbol: Symbol) -> None:
-        self.symtab = symtab
-        self.symbol = symbol
-
-        if not self.name:
-            raise SymbolWithNoNameException
-
-        if self.name not in ParsedSymbol._name_map:
-            ParsedSymbol._name_map[self.name] = []
-
-        ParsedSymbol._name_map[self.name].append(self)
-        ParsedSymbol._max_name_len = max(ParsedSymbol._max_name_len, len(self.name))
-
-    @property
-    def name(self) -> str:
-        return re.sub(r"^(.*?)(\.\d+)?$", r"\1", self.symbol.name)
-
-    @property
-    def size(self) -> int:
-        return cast(int, self.symbol["st_size"])
-
-    @property
-    def address(self) -> int:
-        return cast(int, self.symbol.entry["st_value"])
-
-    def in_range(self, range: Range):
-        return range[0] <= self.address <= range[1]
-
-    def to_undefined_syms(self) -> str:
-        homonyms = ParsedSymbol._name_map[self.name]
-        assert len(homonyms) > 0
-        if len(homonyms) == 1:
-            assert homonyms[0] == self
-            name = self.name
-        else:
-            idx = homonyms.index(self)
-            name = f"{self.name}__local_{idx + 1}"
-
-        name = name if name not in command_script_keywords else f'"{name}"'
-
-        # name_len_fmt = ParsedSymbol._max_name_len + len("__local_9999")
-        name_len_fmt = 40
-
-        return f"{name:{name_len_fmt}s} = 0x{self.address:08x};"
-
-    def to_symbol_addrs(
-        self,
-        type_sizes: dict[str, int],
-        global_vars: list[ccc_json_v7.Global],
-        static_locals: list[ccc_json_v7.Local],
-    ) -> str:
-        homonyms = ParsedSymbol._name_map[self.name]
-        assert len(homonyms) > 0
-        if len(homonyms) == 1:
-            assert homonyms[0] == self
-            name = self.name
-        else:
-            idx = homonyms.index(self)
-            name = f"{self.name}__local_{idx + 1}"
-
-        # name_len_fmt = ParsedSymbol._max_name_len + len("__local_9999")
-        name_len_fmt = 40
-
-        size = size_exceptions.get(self.name, self.size)
-        if size == 0:
-            var = next(
-                (global_ for global_ in global_vars if global_.storage.global_address == self.address), None
-            ) or next((local for local in static_locals if local.storage.global_address == self.address), None)
-
-            if not var:
-                print(f"cannot find {self.name} with size 0 in json")
-            else:
-                element_count, type_name = var.parsed_size
-                type_size = type_sizes.get(type_name)
-                assert type_size is not None, type_name
-                size = element_count * type_size
-
-                if not size:
-                    print(f"size of {self.name} is also 0 using json")
-
-        size_str = f"size:0x{size:x}" if size else ""
-
-        return f"{name:{name_len_fmt}s} = 0x{self.address:08x}; // {size_str}"
-
-
-def parse_symbols_safe(elf_path: Path, dest_path: Path, json_path: Path, ranges: list[Range]):
-    if not dest_path.is_dir():
-        raise RuntimeError(f"{dest_path} is not a directory")
-
-    symbol_addrs_path = dest_path / "symbols_addrs.txt"
-    undefined_syms_path = dest_path / "undefined_syms.txt"
-
-    if symbol_addrs_path.exists() or undefined_syms_path.exists():
-        raise RuntimeError("symbols_addrs.txt or undefined_syms.txt already exist in dest folder")
-
-    with open(elf_path, mode="rb") as elf_fh, open(json_path, mode="r") as json_fh:
-        elf = ELFFile(elf_fh)
-
-        json_data = json.load(json_fh)
-        ccc_model_v7 = ccc_json_v7.CCCJSONv7Model.model_validate(json_data)
-        type_sizes, global_vars, static_locals = parse_ccc_model_v7(ccc_model_v7, ranges)
-
-        with open(symbol_addrs_path, mode="w") as symbol_addrs, open(undefined_syms_path, mode="w") as undefined_syms:
-            parse_symbols(elf, symbol_addrs, undefined_syms, ranges, type_sizes, global_vars, static_locals)
-
-
-def parse_symbols(
-    elf: ELFFile,
-    symbol_addrs: TextIO,
-    undefined_syms: TextIO,
-    ranges: list[Range],
-    type_sizes: dict[str, int],
-    global_vars: list[ccc_json_v7.Global],
-    static_locals: list[ccc_json_v7.Local],
-):
-    # find symbol table
-    symtab = elf.get_section_by_name(".symtab")
-    assert isinstance(symtab, SymbolTableSection)
-
-    parsed_symbols: list[ParsedSymbol] = []
-
-    for symbol in symtab.iter_symbols():
-        try:
-            parsed_symbol = ParsedSymbol(symtab, symbol)
-            if parsed_symbol.name in skip_symbols:
-                continue
-            parsed_symbols.append(parsed_symbol)
-        except SymbolWithNoNameException:
-            pass
-
-    parsed_symbols.sort(key=lambda x: x.address)
-
-    for parsed_symbol in parsed_symbols:
-        if ranges:
-            in_range = any(parsed_symbol.in_range(ra) for ra in ranges)
-            if not in_range:
-                continue
-
-        parsed_symbol.address
-        symbol_addrs.write(parsed_symbol.to_symbol_addrs(type_sizes, global_vars, static_locals))
-        symbol_addrs.write("\n")
-
-        undefined_syms.write(parsed_symbol.to_undefined_syms())
-        undefined_syms.write("\n")
-
-
-def main():
-    class ArgsProtocol(Protocol):
-        elf_path: Path
-        dest_path: Path
-        json_path: Path
-        ranges: list[Range]
-
-    def range_type(arg: str) -> Range:
-        if m := re.match(r"^([0-9a-f]+)-([0-9a-f]+)$", arg):
-            range_start, range_end = int(m.group(1), 16), int(m.group(2), 16)
-            if not (range_end > range_start):
-                raise argparse.ArgumentTypeError("non monotonic range")
-            return range_start, range_end
-        raise argparse.ArgumentTypeError("invalid range")
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("elf_path", metavar="ELF", type=Path, help="path to ELF file")
-    parser.add_argument("--json", dest="json_path", type=Path, help="path to CCC json (v7)")
-    parser.add_argument(
-        "--dest",
-        dest="dest_path",
-        type=Path,
-        required=True,
-        help="folder where symbol_addrs.txt and undefined_syms.txt will be created "
-        "(existing files will not be overwritten)",
-    )
-    parser.add_argument("--range", action="append", dest="ranges", nargs="+", type=range_type)
-
-    args = parser.parse_args()
-    args.ranges = list(itertools.chain(*args.ranges))
-    args = cast(ArgsProtocol, args)
-
-    parse_symbols_safe(args.elf_path, args.dest_path, args.json_path, args.ranges)
-
-
-if __name__ == "__main__":
-    main()