weirdbeardgame/Mulberry
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added MWCC, Added WIBO:

Browse Source 
- Implemented WIBO command for MWCC
- Implemented WIBO command for MWLD

- Need to get matching build
- Wip work on generate_lcf and inc.lcf
This commit is contained in:
weirdbeardgame
2026-07-25 14:13:31 -05:00
committed by Timothy O'Barr
parent 7620a6c671
commit a12dc1c6b2
14 changed files with 508 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configure.py
+295 -72
View File
@@ -17,6 +17,13 @@ import tempfile
import subprocess
import ninja_syntax
from dataclasses import dataclass, asdict
#from utils import ensure_path_and_write, normalize_object_path, to_expected_path
import splat.util.options as splat_options
from splat.segtypes.linker_entry import LinkerEntry, clean_up_path
from spimdisasm.common.CompilerConfig import compilerOptions
from typing import Any, Union, Protocol, Literal, cast
from pathlib import Path
from contextlib import contextmanager
@@ -29,6 +36,10 @@ from splat.segtypes.linker_entry import LinkerEntry
from tools.python.fix_assets import fix_assets
from tools.python.fix_linkerscript import fix_linkerscript
from sys import stdout
from subprocess import run
from dataclasses import dataclass
from struct import unpack
@contextmanager
def suppress_stdout_stderr():
@@ -50,30 +61,20 @@ TOOLS_DIR = ROOT / "tools"
CROSS = "mips-ps2-decompals-"
COMPILER = "ee-gcc2.9-991111-01"
COMPILER_FLAGS = "-DMATCHING_DECOMP -O2 -g2 -gstabs" # -malign-functions=3
COMPILER = "mwcps2-3.0b52-030722"
COMPILER_FLAGS = "-O3,p -sym on -str readonly"
WIBO = f"{TOOLS_DIR}/wibo-x86_64"
LANGUAGES: dict[str, str] = {
"SLUS_210.07": "",
}
# list of object file names (e.g., "tim2_new.c.o"), that have asm that needs
# to be patched. the respective c files (e.g. "tim2_new.c") will be compiled
# in two steps: c -> asm, then asm -> obj. in between, the intermediate
# asm will be passed to `tools/python/fix_asm_matching.py` for patching.
ASM_PATCH_LIST: list[str] = [
"effect_oth.c.o",
"photo.c.o",
"ig_camra.c.o",
]
BIN_PATH_LIST: list[str] = [
"photo.c.o",
]
EUC_JP_LIST = [
"title.c.o",
]
FUNCTION_SYMBOL_LABEL = "glabel"
END_FUNCTION_SYMBOL_LABEL = "endlabel"
UNIQUE_TEXT_SECTION_DIRECTIVE = '.section .text,"ax",@progbits,unique,'
INCLUDE_MACRO_INC_DIRECTIVE = '.include "macro.inc"'
def get_compiler_command(command: str):
@@ -82,33 +83,28 @@ def get_compiler_command(command: str):
ee_compiler_dirname = next(os.walk(ee_dir))[1][0]
commands = {
"ee-gcc": compiler_dir / "bin" / "ee-gcc",
"cc1": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cc1",
"cc1plus": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cc1plus",
"cpp": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cpp",
"as": compiler_dir / "ee" / "bin" / "as",
#"ee-gcc": compiler_dir / "bin" / "ee-gcc",
#"cc1": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cc1",
#"cc1plus": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cc1plus",
#"cpp": compiler_dir / "lib" / "gcc-lib" / "ee" / ee_compiler_dirname / "cpp",
#"as": compiler_dir / "ee" / "bin" / "as",
}
return commands[command]
#return commands[command]
def make_compiler_cmd(config_dir: Path, src_path: Path, language: str):
rel_root = Path(os.path.relpath(ROOT, config_dir))
tools_dir = rel_root / "tools"
game_cc_dir = f"{tools_dir}/cc/{COMPILER}/bin"
lib_cc_dir = f"{tools_dir}/cc/{COMPILER}/bin"
game_cc_dir = f"{tools_dir}/cc/{COMPILER}"
compiler_includes = f"-I{src_path.parent / 'src'} -I{src_path.parent / 'include'} -Iinclude -isystem include/sdk/ee -isystem include/gcc"
assembler_includes = f"-Wa,-I{src_path.parent / 'include'} -Wa,-I{src_path.parent}"
compiler_includes = f"-I{src_path.parent / 'src'} -I{src_path.parent / 'include'} -Iinclude/sdk/ee"
assembler_includes = f"-I{src_path.parent / 'include'} -I{src_path.parent}"
common_includes = f"{compiler_includes} {assembler_includes}"
game_compile_cmd = f"{game_cc_dir}/ee-gcc -c {common_includes} {COMPILER_FLAGS}"
lib_compile_cmd = f"{lib_cc_dir}/ee-gcc -c -isystem include/{COMPILER} {common_includes} {COMPILER_FLAGS}"
game_compile_cmd = f"{WIBO} {game_cc_dir}/mwccps2.exe -c {common_includes} {COMPILER_FLAGS}"
return game_compile_cmd, lib_compile_cmd, common_includes
WIBO_VER = "0.6.4"
return game_compile_cmd, common_includes
def exec_shell(command: list[str]) -> str:
@@ -149,17 +145,17 @@ def clean(config_dir: Path, config: dict[str, Any]):
def write_permuter_settings(config_dir: Path, src_path: Path, language: str):
game_compile_cmd, _, _ = make_compiler_cmd(config_dir, src_path, language)
game_compile_cmd, _ = make_compiler_cmd(config_dir, src_path, language)
with open(config_dir / "permuter_settings.toml", "w") as f:
f.write(
f"""compiler_command = "{game_compile_cmd} -D__GNUC__"
assembler_command = "{CROSS}as -march=r5900 -mabi=eabi -Iinclude"
compiler_type = "gcc"
compiler_type = "mwcc"
[preserve_macros]
[decompme.compilers]
"tools/build/cc/gcc/gcc" = "{COMPILER}"
"tools/build/cc/mwcc/mwcc" = "{COMPILER}"
"""
)
@@ -177,7 +173,7 @@ def build_stuff(
pre_elf_path = f"build/{basename}.elf"
src_path = Path(config["options"]["src_path"])
game_compile_cmd, lib_compile_cmd, common_includes = make_compiler_cmd(config_dir, src_path, language)
game_compile_cmd, common_includes = make_compiler_cmd(config_dir, src_path, language)
built_objects: set[Path] = set()
@@ -213,9 +209,10 @@ def build_stuff(
ninja = ninja_syntax.Writer(open(str(ROOT / config_dir / "build.ninja"), "w"), width=9999)
# Rules
ld_args = "--no-warn-rwx-segments -EL -T undefined_syms.txt -T undefined_syms_auto.txt -T undefined_funcs_auto.txt -Map $mapfile -T $in -o $out"
ld_args = "-map -lcf $in -o $out"
cpp = Path("..", "..", get_compiler_command("cpp"))
ld = Path("..", (Path("tools") / "cc" / COMPILER / "mwldps2.exe"))
cpp = Path("..", (Path("tools") / "cc" / COMPILER / "mwccps2.exe"))
ninja.rule(
"as",
@@ -223,7 +220,7 @@ def build_stuff(
# NOTE: Japanese strings are EUC-JP encoded!!
# We need to convert them from (-f) UTF-8 to (-t) EUC-JP while compiling,
# otherwise Japanese strings will be compiled wrong!
command=f"{cpp} {common_includes} $in -o - | iconv -f=UTF-8 -t=EUC-JP | {CROSS}as -no-pad-sections -EL -march=5900 -mabi=eabi -I{src_path.parent / 'include'} -o $out",
command=f"{WIBO} {cpp} {common_includes} $in -o - | iconv -f=UTF-8 -t=EUC-JP | {CROSS}as -no-pad-sections -EL -march=5900 -mabi=eabi -I{src_path.parent / 'include'} -o $out",
)
ninja.rule(
@@ -235,14 +232,14 @@ def build_stuff(
f"{game_compile_cmd.replace(' -c ', ' -S ')} $in -o $$s_in && " # .......... 3) replace -c with -S in gcc command to generate intermediate assembly file instead of object
f"python3 {ROOT}/tools/python/fix_asm_matching.py {language} $$s_in && " # . 4) fix assembly file using known patterns with tools/python/fix_asm_matching.py
f"{game_compile_cmd} $$s_in -o $out && " # ................................. 5) compile assembly file into object
f"{CROSS}strip $out -N dummy-symbol-name" # ................................ 6) strip 'dummy-symbol-name' from object
f"{ld}strip $out -N dummy-symbol-name" # ................................ 6) strip 'dummy-symbol-name' from object
),
)
ninja.rule(
"cc",
description="cc $in",
command=f"{game_compile_cmd} $in -o $out && {CROSS}strip $out -N dummy-symbol-name",
command=f"{game_compile_cmd} $in -o $out && {ld}strip $out -N dummy-symbol-name",
)
ninja.rule(
@@ -251,22 +248,22 @@ def build_stuff(
command=(
f"eucjp_in=$$(echo $in.eucjp.c | sed 's,^[^/]*/[^/]*/,cc-src/,') && " # . 1) remove ../../ from path + prefix with cc-src/ + suffix with .eucjp.c and store it into eucjp_in var: ../src/file.c -> eucjp_in=cc-src/src/file.c.eucjp.c
f'mkdir -p $$(dirname "$$eucjp_in") && ' # .............................. 2) create directory from eucjp_in var: s_in=cc-src/src/path/to/file.c.eucjp.c -> mkdir -p cc-src/src/path/to/
f"iconv -o $$eucjp_in -f=UTF-8 -t=EUC-JP $in && " # ..................... 3) convert source file to EUC-JP (save converted source to cc-src/src/file.c.eucjp.c)
#f"iconv -o $$eucjp_in -f=UTF-8 -t=EUC-JP $in && " # ..................... 3) convert source file to EUC-JP (save converted source to cc-src/src/file.c.eucjp.c)
f'{game_compile_cmd} -I$$(dirname "$in") $$eucjp_in -o $out && ' # ...... 4) compile converted source file into object (also add original source directory as include path to allow relative imports)
f"{CROSS}strip $out -N dummy-symbol-name" # ............................. 5) strip 'dummy-symbol-name' from object
),
)
ninja.rule(
"libcc",
description="cc $in",
command=f"{lib_compile_cmd} $in -o $out && {CROSS}strip $out -N dummy-symbol-name",
)
#ninja.rule(
# "libcc",
# description="cc $in",
# command=f"{lib_compile_cmd} $in -o $out && {ld}strip $out -N dummy-symbol-name",
# )
ninja.rule(
"ld",
description="link $out",
command=f"{CROSS}ld {ld_args}",
command=f"{WIBO} {ld} {ld_args}",
)
ninja.rule(
@@ -278,7 +275,7 @@ def build_stuff(
ninja.rule(
"elf",
description="elf $out",
command=f"{CROSS}objcopy $in $out -O binary && python3 {ROOT}/tools/python/fix_bin_matching.py {language} $out",
command=f"{ld}objcopy $in $out -O binary && python3 {ROOT}/tools/python/fix_bin_matching.py {language} $out",
)
for entry in linker_entries:
@@ -304,12 +301,6 @@ def build_stuff(
if any(str(src_path).startswith("src/lib/") for src_path in entry.src_paths):
build(entry.object_path, entry.src_paths, "libcc")
elif entry.object_path.name in ASM_PATCH_LIST:
build(entry.object_path, entry.src_paths, "cc-s")
elif entry.object_path.name in EUC_JP_LIST:
build(entry.object_path, entry.src_paths, "cc-eucjp")
else:
build(entry.object_path, entry.src_paths, "cc")
@@ -351,6 +342,162 @@ def build_stuff(
)
def get_line_file_path(args: AnnotationArgs):
if args.line_file_path is not None:
return args.line_file_path
# if args.elf_path.name == SH2_SERIAL and "Event/stage" in args.asm_path.as_posix():
# return Path(f"{TOOLS}/alessatool/dwarf") / Path(args.asm_path.name).with_suffix(".line")
return None
def line_has_vram_addr(line: str, addr_str: str) -> bool:
if addr_str not in line or "*/" not in line:
return False
return line.index("*/") > line.index(addr_str)
def annotate_asm(args: AnnotationArgs):
with open(args.asm_path, "r") as asm_file:
asm_contents = asm_file.read()
asm_lines = asm_contents.splitlines()
asm_line_index = 0
vram_start = args.vram_start
vram_end = args.vram_end
if vram_start is None or vram_end is None:
vram_start, vram_end = find_vram_bounds(asm_lines)
line_file_path = get_line_file_path(args)
if line_file_path is None or not line_file_path.exists():
addresses = (f"0x{v:X}" for v in range(vram_start, vram_end, 0x4))
proc = run([args.addr2line_path, "-e", args.elf_path, *addresses], capture_output=True, encoding=args.encoding)
addr2line_output_lines = proc.stdout.splitlines()
else:
# parse the binary line number file.
# the format is a list of u_shorts, one per line number.
# there should be one line number per vram address, and each vram
# address should be exactly 4 bytes apart, mirroring how the addr2line
# output is formatted
with open(line_file_path, "rb") as line_file:
line_data = line_file.read()
line_numbers = unpack(f"<{len(line_data) // 2}H", line_data)
compile_unit = args.asm_path.with_suffix(".c").name
addr2line_output_lines = list(map(lambda n : to_addr2line_format(compile_unit, n), line_numbers))
main_tu_name = None
prev_tu_name = None
prev_line_number = -1
function_count = 0
is_in_function_label = False
current_vram_addr = vram_start
annotated_asm_lines = []
for addr_index in range(0, len(addr2line_output_lines) - 1):
line = addr2line_output_lines[addr_index]
if line.startswith("?"):
current_vram_addr += 0x4
continue
separator_index = line.rfind(":")
current_line_number = int(line[separator_index+1:])
if current_line_number == prev_line_number:
current_vram_addr += 0x4
continue
vram_addr_str = f"{current_vram_addr:X}"
current_tu_name = line[0:separator_index]
if current_tu_name != main_tu_name and current_tu_name != prev_tu_name:
if main_tu_name:
print(f"[warn] alessatool/annotate: address at {vram_addr_str} belongs to {current_tu_name}")
else:
main_tu_name = current_tu_name
prev_tu_name = current_tu_name
while True:
if asm_line_index >= len(asm_lines):
raise AssertionError(
f"address 0x{current_vram_addr:X} not found in asm"
)
asm_line = asm_lines[asm_line_index]
if line_has_vram_addr(asm_line, vram_addr_str):
break
should_append_asm_line = True
if args.tu:
asm_line_trimmed = asm_line.strip()
# track when we go in & out of function symbols
if asm_line_trimmed.startswith(FUNCTION_SYMBOL_LABEL):
function_count += 1
is_in_function_label = True
annotated_asm_lines.append(f"{UNIQUE_TEXT_SECTION_DIRECTIVE}{function_count}")
annotated_asm_lines.append("")
elif asm_line_trimmed.startswith(END_FUNCTION_SYMBOL_LABEL):
is_in_function_label = False
# remove `nop`s
if not is_in_function_label and asm_line_trimmed.endswith("nop"):
should_append_asm_line = False
# remove `macro.inc` include directive
elif asm_line_trimmed == INCLUDE_MACRO_INC_DIRECTIVE:
should_append_asm_line = False
if should_append_asm_line:
annotated_asm_lines.append(asm_line)
asm_line_index += 1
annotated_asm_lines.append(f"\t.loc 1 {current_line_number}")
annotated_asm_lines.append(asm_line)
asm_line_index += 1
prev_line_number = current_line_number
current_vram_addr += 0x4
while asm_line_index < len(asm_lines) - 1:
annotated_asm_lines.append(asm_lines[asm_line_index])
asm_line_index += 1
assert main_tu_name, "no valid compilation unit found"
annotated_asm_lines = [
".section .debug",
".previous",
".text",
f".file 1 \"{main_tu_name}\"",
*annotated_asm_lines
]
append_final_new_line(annotated_asm_lines)
annotated_asm_contents = "\n".join(annotated_asm_lines)
if not args.stdout and args.out_path:
with open(args.out_path, "w") as out_file:
out_file.write(annotated_asm_contents)
if args.verbose:
print(f"alessatool/annotate: wrote asm to {args.out_path}")
else:
stdout.write(annotated_asm_contents)
def append_final_new_line(lines: list[str]):
if lines[-1] != "":
lines.append("")
def make_asm(config_path: Path, config: dict[str, Any]):
"""
Extracts assembly for each function into 'expected/asm/' subfolder.
@@ -456,7 +603,7 @@ def make_asm(config_path: Path, config: dict[str, Any]):
tmp_obj_path = tmp_path / "obj"
tmp_asm_dir = tmp_path / "asm"
cpp = Path("..", "..", "..", get_compiler_command("cpp"))
cpp = Path("..", "..", "..", (Path("tools") / "cc" / COMPILER))
for asm_file in tmp_asm_dir.rglob("*.c.s"):
asm_file_rel = asm_file.relative_to(tmp_path)
@@ -464,8 +611,8 @@ def make_asm(config_path: Path, config: dict[str, Any]):
obj_file = tmp_obj_path / obj_file_rel.relative_to("obj")
obj_file.parent.mkdir(parents=True, exist_ok=True)
subprocess.run(
f"{cpp} -I../src -I../include -Iinclude -isystem include/sdk/ee -isystem include/gcc -Wa,-I../include -Wa,-I../.. '{asm_file_rel}' -o - | "
f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | "
f"{cpp} -I../src -I../include -Iinclude -Iinclude/sdk/ee -I../.. '{asm_file_rel}' -o - | "
#f"iconv -f=UTF-8 -t=EUC-JP '{asm_file_rel}' | "
f"mips-linux-gnu-as -no-pad-sections -EL -march=5900 -mabi=eabi -I../include -o {obj_file_rel} {asm_file_rel}",
shell=True,
cwd=tmp_path,
@@ -590,6 +737,85 @@ def generate_objdiff_configuration(config_path: Path, config: dict[str, Any], la
print(f"Wrote objdiff configuration ({len(units)} units) to {objdiff_path}")
SECTION_ALIGNMENT_PAIRS: list[tuple[str, int]] = [
(".text", 0x10),
(".data", 0x8),
(".rodata", 0x8),
(".lit4", 0x4),
(".sdata", 0x4),
(".sbss", 0x4),
(".bss", 0x10),
]
TRACKED_SEGMENT_TYPES = ["asm", "c", "data", "rodata"]
def generate_lcf():
'''
Generate a linker command file. It uses an oversimplified method of looping
over the linker entries emitted by Splat and looking for splits that are
parented to a `code` segment. Then it writes out the list of object files
for each section.
Expects a template file with the extension `.inc.lcf`.
'''
entries_by_section_type: dict[str, list[LinkerEntry]] = dict()
for section_type, _ in SECTION_ALIGNMENT_PAIRS:
entries_by_section_type[section_type] = []
for entry in split.linker_writer.entries:
segment = entry.segment
parent = segment.parent
if not parent or parent.type != "code":
continue
section_type = entry.section_link_type
if section_type not in entries_by_section_type:
continue
entries_by_section_type[section_type].append(entry)
lcf_blocks: list[str] = []
for section_type, alignment in SECTION_ALIGNMENT_PAIRS:
objects = entries_by_section_type[section_type]
if not objects:
continue
if section_type == ".bss":
alignment = args.bss_alignment
block = [
f"\t\t# {section_type}",
f"\t\t__{section_type[1:]}_start = .;",
f"\t\tALIGNALL(0x{alignment:X});",
]
for entry in objects:
alignment = entry.segment.ld_align_segment_start
if alignment is not None:
block.append(f"\t\tALIGNALL(0x{alignment:X});")
object_name = entry.object_path.name
block.append(f"\t\t{object_name} ({section_type})")
lcf_blocks.append("\n".join(block))
generated_lcf = "\n\n".join(lcf_blocks)
template_path = args.template_path
output_path = args.lcf_output_path
template = template_path.read_text()
output = template.replace(
"### 🔴 code 🔴 ###",
generated_lcf.strip(),
)
output_path.write_text(output)
if args.verbose:
print(f"✅ Mullberry/generate: wrote LCF to {output_path}")
def main():
class ArgsProtocol(Protocol):
YAML_FILE: Path
@@ -622,22 +848,13 @@ def main():
)
args = cast(ArgsProtocol, parser.parse_args())
# try:
# exec_shell(["wibo"])
# except FileNotFoundError:
# print("ERROR: wibo does not appear to be accessible")
# print("To install it, please download it and put it in your PATH:")
# print(
# f" wget https://github.com/decompals/wibo/releases/download/{WIBO_VER}/wibo && "
# f"chmod +x wibo && sudo mv wibo /usr/bin/"
# )
# sys.exit(1)
cwd = Path(os.getcwd()).resolve()
if not ROOT.samefile(cwd):
print("ERROR: this script must be run from it's directory!")
exit(1)
compilerOptions["MWCCPS2"].value.bigAddendWorkaroundForMigratedFunctions = False
config = splat_load_yaml(
[args.YAML_FILE],
modes=["all"],
@@ -648,6 +865,9 @@ def main():
basename = config["options"]["basename"]
config_dir = Path(args.YAML_FILE).parent
compiler_dir = Path("tools") / "cc" / COMPILER
if basename not in LANGUAGES:
supported_elfs = f"{set(f'{elf} ({lang})' for elf, lang in LANGUAGES.items())}".replace("'", "")
print(f"unsupported game ELF. Supported versions are: {supported_elfs}")
@@ -677,6 +897,9 @@ def main():
assert asm_data_path.is_dir(), f"{asm_data_path} not found or not a directory"
assert asset_rel_path.is_dir(), f"{asset_rel_path} not found or not a directory"
generate_lcf()
# fix asset .incbin path from relative to root to relative to config dir:
# e.g., change:
# .incbin "config/assets/name.section.bin"