diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b62d235..a3bac5ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # splat Release Notes +### 0.50.0 + +* We have skipped to version 0.50.0, as this version may introduce several breaking changes and require some extra care when migrating +* BREAKING CHANGES: + * Declaring an overlay symbol but not specifying its `segment:` or `rom:` in symbol_addrs is no loger accepted. A warning will be emitted and the symbol won't be used. + * Associating a symbol with a segment that is not within the Vram or Rom address space for that segment is no longer accepted. + * Use the new prioritized segment system to tell splat to use a symbol from a specific segment. + * Declaring a symbol without an associated segment and outside the Vram space of any global segments is no longer accepted. + * Use the new absoulte symbols system to declare symbols that aren't part of any real segment. + * `splat.utils.symbols.all_symbols_dict` and `splat.utils.symbols.all_symbols_ranges` have been removed. + * The algorithm for checking vram overlaps between global segments and overlays have been changed and it is more strict now. This means it is more likely to report conflicts, which may result in errors. + * A common source of errors is giving a vram address (usually `vram: 0xA4000040`) for the ipl3 segment on n64 projects. A recommened workaround is to either remove the vram for this segment or tag it with its own `exclusive_ram_id` (i.e. `exclusive_ram_id: ipl3`). + * Autogenerated symbol names may have changed + * Previously, splat failed to properly set some attributes required for autogenerated symbol naming and thus symbolized them incorrectly. For example, it only used the vram address (like `D_{VRAM}`) when it should have been using both the vram and rom addresses (`D_{VRAM}_{ROM}`) for naming overlay symbols. These attributes are now correctly set, resulting in improved symbol names. This may break existing symbols in C code, so please keep this in mind when migrating. +* Rewrite symbol management system. + * Completely change how the internal symbol tracking system works, aiming to fix issues with overlapping addresses in overlays. + * Ideally this should fix most issues related to using a symbol from overlay A in overlay B even when both are marked with the same `exclusive_ram_id`. + * Because of this, splat is a lot more strict about user-declared symbols. From now on, splat will warn and discard symbols that it can't figure out the corresponding segment on its own, instead of just trying to use them whenever they fit. This is mostly an issue for declaring symbols for overalys with overlapping addresses (segments declared with `exclusive_ram_id`). +* Add prioritized segment system. + * Allows to tell which segments should be prioritized when looking up for symbol references. + * This way a user can choose how to disambiguate references to multiple segments with overlapping addresses, for example, referencing a an address shared by multiple overlays (say A, B and C) by telling the segment to lookup symbols from segment B first instead of looking into any other arbitrarily. + * Also allows global segments to reference symbols from specific overlays. + * Each segment can have their own list of prioritized segments. This list can be declared with the `prioritized_segments` yaml attribute on the segment level. + * This attribute is only accepted for top-level segments, subsegments silently ignore this attribute. +* Add absolute symbols system. + * Allows declaring symbols that are not part of any segment of the ROM. + * Declared with attribute symbol `absolute:True`. + * Symbols declared this way always take priority over everything else when looking up for symbol references. + * This should not be used to declare symbols that are part of the actual address space of the rom. +* `alabel` is recognized as a symbol type. + * An alternative entrypoint for functions. + * Works similarly to branch labels, in the way that it is used to declare symbols in the middle of functions. +* `spimdisasm` 1.42.1 or above is now required. + ### 0.41.1 * Mark `__romPos` as `HIDDEN`. @@ -28,8 +62,8 @@ ### 0.40.0 -* Append an `-include` directive to the generated elf dependency file. - * Includes the dependency files for each object needed to build the target, simplifying dependency management on the build system. +* Append an `-include` directive to the generated elf dependency file. + * Includes the dependency files for each object needed to build the target, simplifying dependency management on the build system. * This behavior is enabled by default, and can be disabled with the new option `ld_dependencies_include`. ### 0.39.1 diff --git a/README.md b/README.md index c50e1871..9248cf12 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin If you use a `requirements.txt` file in your repository, then you can add this library with the following line: ```txt -splat64[mips]>=0.41.1,<1.0.0 +splat64[mips]>=0.50.0,<1.0.0 ``` ### Optional dependencies diff --git a/docs/Adding-Symbols.md b/docs/Adding-Symbols.md index 38e77ed1..64d886da 100644 --- a/docs/Adding-Symbols.md +++ b/docs/Adding-Symbols.md @@ -1,3 +1,5 @@ +# Declaring symbols + Symbols (i.e. labelling a function or variable) are controlled by the `symbols_addrs.txt` file. The format for defining symbols is: @@ -5,6 +7,7 @@ The format for defining symbols is: ```ini symbol = address; // option1:value1 option2:value2 ``` + e.g. ```ini osInitialize = 0x801378C0; // type:func @@ -30,10 +33,12 @@ An optional `key:pair` list of settings, note that each option should be separat ### `type` Override splat's automatic type detection, possible values are: + - `func`: Functions - `jtbl`: Jumptables - `jtbl_label`: Jumptables labels (inside functions) - `label`: Branch labels (inside functions) +- `alabel`: Alternative function entrypoints (inside functions). - `s8`, `u8`: To specify data/rodata to be disassembled as `.byte`s - `s16`, `u16`: To specify data/rodata to be disassembled as `.short`s - `s32`, `u32`: To specify data/rodata to be disassembled as `.word`s (the default) @@ -61,7 +66,9 @@ RawHuffmanTable = 0x8022E0E0; // type:symbol size:0x100 ### `rom` -The ROM offset for the symbol, useful (potentially mandatory) for symbols in overlays where multiple symbols could share the same VRAM address. +The ROM offset for the symbol. + +It is mandatory to either specify at least this or [segment](#segment) for symbols in overlays where multiple symbols could share the same VRAM address. If both are missing, the symbol is assumed to be part of any of the non-overlay segments. **Example:** ```ini @@ -70,13 +77,32 @@ create_particle_effect = 0x802D5F4C; // type:func rom:0x6E75FC ### `segment` -Allows specifying to which specific segment this symbol belongs to, useful to disambiguate symbols from segments that share the same VRAM address. This name must be the same as the name of a segment listed in the yaml. +Allows specifying to which specific segment this symbol belongs to, useful to disambiguate symbols from segments that share the same VRAM address. + +This name must be the same as the name of a segment listed in the yaml. +The segment's Vram and Rom address must include the addresses of the given symbol. + +It is mandatory to either specify at least this or [rom](#rom) for symbols in overlays where multiple symbols could share the same VRAM address. If both are missing, the symbol is assumed to be part of any of the non-overlay segments. **Example:** ```ini sMenuTexture = 0x06004040; // segment:menu_assets ``` +### `absolute` + +Declares this symbol as being associated to no segment. + +Some symbols are special in the way they exist outside the address space of the ROM. Instead they are provided by the OS, hardware, etc. `absolute` can be used to name these kind of symbols. + +`absolute` symbols also have the property of being prioritized over everything else when looking up for address references. + +**Example:** + +```ini +osTvType = 0x80000300; // absolute:True +``` + ### `name_end` Emits a symbol after the end of the data of the current symbol. Useful to reference the end of an assembly symbol, like RSP data. diff --git a/docs/Segments.md b/docs/Segments.md index fb59132d..9194a361 100644 --- a/docs/Segments.md +++ b/docs/Segments.md @@ -642,3 +642,35 @@ This value expects the name of the other segment that should be paired to the cu - [0x2C6B0, .rodata, libultra/audio/init_15550] # -- snip -- ``` + +### `prioritized_segments` + +A list of top-level segments. + +This segment is allowed to "see" the symbols from the segments on that list, even if it shouldn't given their `exclusive_ram_id`. + +This also allows control over how to disambiguate references over multiple overlay segments with overlaping addresses. + +Note the visibility is unidirectional; if you want two segments to see each other, you need to add a `prioritized_segments` on both, listing each other's names. + +```yaml + - type: code + name: volcano_assets + dir: volcano + start: 0x326C10 + vram: 0x800FFF90 + bss_size: 0x20 + exclusive_ram_id: unk800FFF90 + + # -- SNIP -- + + - name: volcano_code + dir: volcano + type: code + start: 0x7272E0 + vram: 0x802D60E0 + bss_size: 0x170 + exclusive_ram_id: level + prioritized_segments: + - volcano_assets +``` diff --git a/pyproject.toml b/pyproject.toml index ff928e1a..c51deacf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "splat64" # Should be synced with src/splat/__init__.py -version = "0.41.1" +version = "0.50.0" description = "A binary splitting tool to assist with decompilation and modding projects" readme = "README.md" license = {file = "LICENSE"} @@ -20,7 +20,7 @@ dependencies = [ [project.optional-dependencies] mips = [ - "spimdisasm>=1.41.0,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py + "spimdisasm>=1.42.1,<2.0.0", # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py "rabbitizer>=1.12.0,<2.0.0", "pygfxd>=1.0.5", "n64img>=0.3.3", diff --git a/requirements.txt b/requirements.txt index ccee9aba..b6084600 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ tqdm==4.67.1 intervaltree==3.1.0 colorama==0.4.6 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py and pyproject.toml -spimdisasm>=1.41.0 +spimdisasm>=1.42.1 rabbitizer>=1.10.0 pygfxd>=1.0.5 n64img>=0.1.4 diff --git a/src/splat/__init__.py b/src/splat/__init__.py index 56992ae1..d6b12807 100644 --- a/src/splat/__init__.py +++ b/src/splat/__init__.py @@ -1,7 +1,7 @@ __package_name__ = __name__ # Should be synced with pyproject.toml -__version__ = "0.41.1" +__version__ = "0.50.0" __author__ = "ethteck" from . import util as util diff --git a/src/splat/disassembler/spimdisasm_disassembler.py b/src/splat/disassembler/spimdisasm_disassembler.py index c7e1b177..a28ecdd3 100644 --- a/src/splat/disassembler/spimdisasm_disassembler.py +++ b/src/splat/disassembler/spimdisasm_disassembler.py @@ -7,7 +7,7 @@ class SpimdisasmDisassembler(disassembler.Disassembler): # This value should be kept in sync with the version listed on requirements.txt and pyproject.toml - SPIMDISASM_MIN = (1, 41, 0) + SPIMDISASM_MIN = (1, 42, 1) def configure(self): # Configure spimdisasm diff --git a/src/splat/scripts/split.py b/src/splat/scripts/split.py index 2751cb32..afc63085 100755 --- a/src/splat/scripts/split.py +++ b/src/splat/scripts/split.py @@ -3,17 +3,24 @@ import argparse import hashlib import importlib -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union from pathlib import Path from collections import defaultdict, deque from .. import __package_name__, __version__ from ..disassembler import disassembler_instance -from ..util import cache_handler, progress_bar, vram_classes, statistics, file_presets +from ..util import ( + cache_handler, + progress_bar, + vram_classes, + statistics, + file_presets, + metadata, +) from colorama import Style -from intervaltree import Interval, IntervalTree +from intervaltree import IntervalTree import sys from ..segtypes.linker_entry import ( @@ -168,29 +175,6 @@ def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: return ret -def assign_symbols_to_segments(): - for symbol in symbols.all_symbols: - if symbol.segment: - continue - - if symbol.rom: - cands: Set[Interval] = segment_roms[symbol.rom] - if len(cands) > 1: - log.error("multiple segments rom overlap symbol", symbol) - elif len(cands) == 0: - log.error("no segment rom overlaps symbol", symbol) - else: - cand: Interval = cands.pop() - seg: Segment = cand.data - seg.add_symbol(symbol) - else: - cands = segment_rams[symbol.vram_start] - segs: List[Segment] = [cand.data for cand in cands] - for seg in segs: - if not seg.get_exclusive_ram_id(): - seg.add_symbol(symbol) - - def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: s = seg.name.strip() if len(s) > limit: @@ -294,11 +278,10 @@ def initialize_all_symbols(all_segments: List[Segment]): symbols.initialize(all_segments) relocs.initialize() - # Assign symbols to segments - assign_symbols_to_segments() + metadata.segment_manager.initialize(all_segments, symbols.all_symbols) if options.opts.is_mode_active("code"): - symbols.initialize_spim_context(all_segments) + symbols.initialize_spim_context(metadata.segment_manager.manager) relocs.initialize_spim_context() @@ -482,7 +465,7 @@ def write_undefined_funcs_auto(): to_write = [ s for s in symbols.all_symbols - if s.referenced and not s.defined and s.type == "func" + if (s.referenced or s.user_declared) and not s.defined and s.type == "func" ] to_write.sort(key=lambda x: x.vram_start) @@ -494,9 +477,9 @@ def write_undefined_syms_auto(): to_write = [ s for s in symbols.all_symbols - if s.referenced + if (s.referenced or s.user_declared) and not s.defined - and s.type not in {"func", "label", "jtbl_label"} + and s.type not in {"func", "label", "jtbl_label", "alabel"} ] to_write.sort(key=lambda x: x.vram_start) diff --git a/src/splat/segtypes/common/bss.py b/src/splat/segtypes/common/bss.py index 0dce9658..fd651141 100644 --- a/src/splat/segtypes/common/bss.py +++ b/src/splat/segtypes/common/bss.py @@ -118,7 +118,10 @@ def disassemble_data(self, rom_bytes: bytes): for spim_sym in self.spim_section.get_section().symbolList: symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), spim_sym.contextSym, force_in_segment=True + self.get_most_parent(), + self, + spim_sym.contextSym, + force_in_segment=True, ) def should_scan(self) -> bool: diff --git a/src/splat/segtypes/common/c.py b/src/splat/segtypes/common/c.py index fef195b3..d81731bb 100644 --- a/src/splat/segtypes/common/c.py +++ b/src/splat/segtypes/common/c.py @@ -269,7 +269,9 @@ def split(self, rom_bytes: bytes): or options.opts.disassemble_all ): rodata_sym = self.get_symbol( - spim_rodata_sym.vram, in_segment=True, local_only=True + spim_rodata_sym.vram, + in_segment=True, + local_only=True, ) assert rodata_sym is not None @@ -470,7 +472,9 @@ def create_c_file( else: for spim_rodata_sym in entry.rodataSyms: rodata_sym = self.get_symbol( - spim_rodata_sym.vram, in_segment=True, local_only=True + spim_rodata_sym.vram, + in_segment=True, + local_only=True, ) assert rodata_sym is not None diff --git a/src/splat/segtypes/common/codesubsegment.py b/src/splat/segtypes/common/codesubsegment.py index 9582a9e7..37e89c89 100644 --- a/src/splat/segtypes/common/codesubsegment.py +++ b/src/splat/segtypes/common/codesubsegment.py @@ -119,22 +119,24 @@ def process_insns( self.parent: CommonSegCode = self.parent symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), func_spim.contextSym, force_in_segment=False + self.get_most_parent(), self, func_spim.contextSym, force_in_segment=True ) # Gather symbols found by spimdisasm and create those symbols in splat's side - for referenced_vram in func_spim.referencedVrams: - context_sym = self.spim_section.get_section().getSymbol( - referenced_vram, tryPlusOffset=False - ) - if context_sym is not None: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym, force_in_segment=False - ) + for i in range(0, func_spim.sizew * 4, 4): + reloc_info = func_spim.getReloc(i) + if reloc_info is not None: + if isinstance(reloc_info.symbol, spimdisasm.common.ContextSymbol): + symbols.create_symbol_from_spim_symbol( + self.get_most_parent(), + self, + reloc_info.symbol, + force_in_segment=False, + ) # Main loop - for i, insn in enumerate(func_spim.instructions): - if options.opts.platform == "ps2": + if options.opts.platform == "ps2": + for insn in func_spim.instructions: from .c import CommonSegC from rabbitizer import TrinaryValue @@ -144,34 +146,21 @@ def process_insns( insn.flag_r5900UseDollar = TrinaryValue.TRUE insn.flag_r5900DisasmAsData = TrinaryValue.TRUE - instr_offset = i * 4 - - # update pointer accesses from this function - if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: - sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] - - context_sym = self.spim_section.get_section().getSymbol(sym_address) - if context_sym is not None: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym, force_in_segment=False - ) - def print_file_boundaries(self): if not self.show_file_boundaries or not self.spim_section: return assert isinstance(self.rom_start, int) - for in_file_offset in self.spim_section.get_section().fileBoundaries: + spim_section = self.spim_section.get_section() + for in_file_offset in spim_section.fileBoundaries: if not self.parent.reported_file_split: self.parent.reported_file_split = True # Look up for the last symbol in this boundary sym_addr = 0 - for sym in self.spim_section.get_section().symbolList: - symOffset = ( - sym.inFileOffset - self.spim_section.get_section().inFileOffset - ) + for sym in spim_section.symbolList: + symOffset = sym.inFileOffset - spim_section.inFileOffset if in_file_offset == symOffset: break sym_addr = sym.vram diff --git a/src/splat/segtypes/common/data.py b/src/splat/segtypes/common/data.py index bedff760..cc78b8cd 100644 --- a/src/splat/segtypes/common/data.py +++ b/src/splat/segtypes/common/data.py @@ -127,19 +127,25 @@ def disassemble_data(self, rom_bytes): rodata_encountered = False - for symbol in self.spim_section.get_section().symbolList: + spim_section = self.spim_section.get_section() + for symbol in spim_section.symbolList: symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym, force_in_segment=True + self.get_most_parent(), self, symbol.contextSym, force_in_segment=True ) # Gather symbols found by spimdisasm and create those symbols in splat's side for referenced_vram in symbol.referencedVrams: - context_sym = self.spim_section.get_section().getSymbol( - referenced_vram, tryPlusOffset=False + context_sym = spim_section.getSymbol( + referenced_vram, + tryPlusOffset=False, + allowOutsideIfVramIsInside=False, ) if context_sym is not None: symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym, force_in_segment=False + self.get_most_parent(), + self, + context_sym, + force_in_segment=False, ) # Hint to the user that we are now in the .rodata section and no longer in the .data section (assuming rodata follows data) diff --git a/src/splat/segtypes/common/rodata.py b/src/splat/segtypes/common/rodata.py index 51df5717..857e7e0d 100644 --- a/src/splat/segtypes/common/rodata.py +++ b/src/splat/segtypes/common/rodata.py @@ -106,20 +106,29 @@ def disassemble_data(self, rom_bytes): last_jumptable_addr_remainder = 0 misaligned_jumptable_offsets: List[int] = [] - for symbol in self.spim_section.get_section().symbolList: + spim_section = self.spim_section.get_section() + for symbol in spim_section.symbolList: generated_symbol = symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym, force_in_segment=True + self.get_most_parent(), + self, + symbol.contextSym, + force_in_segment=True, ) generated_symbol.linker_section = self.get_linker_section_linksection() # Gather symbols found by spimdisasm and create those symbols in splat's side for referenced_vram in symbol.referencedVrams: - context_sym = self.spim_section.get_section().getSymbol( - referenced_vram, tryPlusOffset=False + context_sym = spim_section.getSymbol( + referenced_vram, + tryPlusOffset=False, + allowOutsideIfVramIsInside=False, ) if context_sym is not None: symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym, force_in_segment=False + self.get_most_parent(), + self, + context_sym, + force_in_segment=False, ) possible_text = self.get_possible_text_subsegment_for_symbol(symbol) diff --git a/src/splat/segtypes/n64/gfx.py b/src/splat/segtypes/n64/gfx.py index f7a8b7a5..cf35ce67 100644 --- a/src/splat/segtypes/n64/gfx.py +++ b/src/splat/segtypes/n64/gfx.py @@ -42,8 +42,6 @@ from ..common.codesubsegment import CommonSegCodeSubsegment -from ...util import symbols - LIGHTS_RE = re.compile(r"\*\(Lightsn \*\)0x[0-9A-F]{8}") @@ -102,68 +100,159 @@ def get_gfxd_target(self): log.error(f"Unknown target {opt}") def tlut_handler(self, addr, idx, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def timg_handler(self, addr, fmt, size, width, height, pal): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def cimg_handler(self, addr, fmt, size, width): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def zimg_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def dl_handler(self, addr): # Look for 'Gfx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Gfx") + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "Gfx", + ) if not sym: sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + addr=addr, + in_segment=self.in_segment, + type="Gfx", + reference=True, ) gfxd_printf(self.format_sym_name(sym)) return 1 def mtx_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "Mtx", ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + type="Mtx", + reference=True, + ) gfxd_printf(f"&{self.format_sym_name(sym)}") return 1 def lookat_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "LookAt", ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + type="LookAt", + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def light_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "Light", ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + type="Light", + reference=True, + ) gfxd_printf(self.format_sym_name(sym)) return 1 def vtx_handler(self, addr, count): # Look for 'Vtx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Vtx") + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "Vtx", + ) if not sym: sym = self.create_symbol( @@ -179,9 +268,21 @@ def vtx_handler(self, addr, count): return 1 def vp_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + sym = self.get_symbol( + addr, + in_segment=False, + reference=True, + local_only=False, + validation=lambda sym: sym.type == "Vp", ) + + if not sym: + sym = self.create_symbol( + addr=addr, + in_segment=self.in_segment, + type="Vp", + reference=True, + ) gfxd_printf(f"&{self.format_sym_name(sym)}") return 1 @@ -206,7 +307,10 @@ def disassemble_data(self, rom_bytes): out_str = "" if self.data_only else options.opts.generated_c_preamble + "\n\n" sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True + addr=self.vram_start, + in_segment=True, + type="Gfx", + define=True, ) gfxd_input_buffer(gfx_data) @@ -252,7 +356,9 @@ def light_sub_func(match): light = match.group(0) addr = int(light[12:], 0) sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True + addr=addr, + in_segment=self.in_segment, + reference=True, ) return self.format_sym_name(sym) diff --git a/src/splat/segtypes/n64/vtx.py b/src/splat/segtypes/n64/vtx.py index 1a172d08..bda1478d 100644 --- a/src/splat/segtypes/n64/vtx.py +++ b/src/splat/segtypes/n64/vtx.py @@ -70,7 +70,10 @@ def disassemble_data(self, rom_bytes) -> str: vertex_count = segment_length // 16 sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True + addr=self.vram_start, + in_segment=True, + type="Vtx", + define=True, ) if not self.data_only: diff --git a/src/splat/segtypes/segment.py b/src/splat/segtypes/segment.py index 79c5d444..f086a3ea 100644 --- a/src/splat/segtypes/segment.py +++ b/src/splat/segtypes/segment.py @@ -4,14 +4,16 @@ import importlib.util from pathlib import Path -from typing import Dict, List, Optional, Set, Type, TYPE_CHECKING, Union, Tuple +from typing import Callable, Dict, List, Optional, Type, TYPE_CHECKING, Union, Tuple -from intervaltree import Interval, IntervalTree from ..util import vram_classes from ..util.vram_classes import VramClass -from ..util import log, options, symbols +from ..util import log, options from ..util.symbols import Symbol, to_cname +from ..util.metadata.parent_segment_info import ParentSegmentInfo +from ..util.metadata.segment_metadata import SegmentMetadata +from ..util.metadata.segment_manager import manager from .. import __package_name__ @@ -69,6 +71,10 @@ def parse_segment_section_order(segment: Union[dict, list]) -> List[str]: return default +def default_sym_validation(_sym: Symbol) -> bool: + return True + + SegmentType = str @@ -306,17 +312,15 @@ def __init__( self.align: Optional[int] = None self.given_subalign: Optional[int] = options.opts.subalign self.exclusive_ram_id: Optional[str] = None + self.prioritized_segments: list[str] = [] self.given_dir: Path = Path() # Default to global options. self.given_find_file_boundaries: Optional[bool] = None - # Symbols known to be in this segment - self.given_seg_symbols: Dict[int, List[Symbol]] = {} - - # Ranges for faster symbol lookup - self.symbol_ranges_ram: IntervalTree = IntervalTree() - self.symbol_ranges_rom: IntervalTree = IntervalTree() + # Metadata, symbols and symbol lookup information, + # only the most_parent has this information. + self.owned_metadata: Optional[SegmentMetadata] = None self.given_section_order: List[str] = options.opts.section_order @@ -465,6 +469,7 @@ def from_yaml( if isinstance(yaml, dict): ret.extract = bool(yaml.get("extract", ret.extract)) ret.exclusive_ram_id = yaml.get("exclusive_ram_id") + ret.prioritized_segments = yaml.get("prioritized_segments", []) ret.given_dir = Path(yaml.get("dir", "")) ret.has_linker_entry = bool(yaml.get("linker_entry", True)) ret.given_find_file_boundaries = yaml.get("find_file_boundaries", None) @@ -573,24 +578,6 @@ def get_exclusive_ram_id(self) -> Optional[str]: return self.parent.get_exclusive_ram_id() return self.exclusive_ram_id - def add_symbol(self, symbol: Symbol): - if symbol.vram_start not in self.given_seg_symbols: - self.given_seg_symbols[symbol.vram_start] = [] - self.given_seg_symbols[symbol.vram_start].append(symbol) - - # For larger symbols, add their ranges to interval trees for faster lookup - if symbol.size > 4: - self.symbol_ranges_ram.addi(symbol.vram_start, symbol.vram_end, symbol) - if symbol.rom is not None: - self.symbol_ranges_rom.addi(symbol.rom, symbol.rom_end, symbol) - - @property - def seg_symbols(self) -> Dict[int, List[Symbol]]: - if self.parent: - return self.parent.seg_symbols - else: - return self.given_seg_symbols - @property def size(self) -> Optional[int]: if self.rom_start is not None and self.rom_end is not None: @@ -787,120 +774,63 @@ def unique_id(self): return s + self.type + "_" + self.name - @staticmethod - def visible_ram(seg1: "Segment", seg2: "Segment") -> bool: - if seg1.get_most_parent() == seg2.get_most_parent(): - return True - if seg1.get_exclusive_ram_id() is None or seg2.get_exclusive_ram_id() is None: - return True - return seg1.get_exclusive_ram_id() != seg2.get_exclusive_ram_id() - - def retrieve_symbol( - self, syms: Dict[int, List[Symbol]], addr: int - ) -> Optional[Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - # Filter out symbols that are in different top-level segments with the same unique_ram_id - items = [ - i - for i in items - if i.segment is None or Segment.visible_ram(self, i.segment) - ] - - if len(items) > 1: - # print(f"Trying to retrieve {addr:X} from symbol dict but there are {len(items)} entries to pick from - picking the first") - pass - if len(items) == 0: - return None - return items[0] - - def retrieve_sym_type( - self, syms: Dict[int, List[Symbol]], addr: int, type: str - ) -> Optional[symbols.Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - items = [ - i - for i in items - if (i.segment is None or Segment.visible_ram(self, i.segment)) - and (type == i.type) - ] - - if len(items) == 0: - return None - - return items[0] + def get_parent_segment_info( + self, + ) -> tuple["Segment", "Optional[ParentSegmentInfo]"]: + most_parent = self.get_most_parent() + parent_segment_info = None + if most_parent.rom_start is not None and most_parent.vram_start is not None: + parent_segment_info = ParentSegmentInfo( + most_parent.name, + most_parent.rom_start, + most_parent.vram_start, + most_parent.exclusive_ram_id, + ) + return most_parent, parent_segment_info def get_symbol( self, addr: int, in_segment: bool = False, type: Optional[str] = None, - create: bool = False, define: bool = False, reference: bool = False, search_ranges: bool = False, local_only: bool = False, + validation: Callable[[Symbol], bool] = default_sym_validation, ) -> Optional[Symbol]: ret: Optional[Symbol] = None - rom: Optional[int] = None - - most_parent = self.get_most_parent() - if in_segment: - # If the vram address is within this segment, we can calculate the symbol's rom address - rom = most_parent.ram_to_rom(addr) - ret = most_parent.retrieve_symbol(most_parent.seg_symbols, addr) - - if not ret and search_ranges: - # Search ranges first, starting with rom - if rom is not None: - cands: Set[Interval] = most_parent.symbol_ranges_rom[rom] - if cands: - ret = cands.pop().data - # and then vram if we can't find a rom match - if not ret: - cands = most_parent.symbol_ranges_ram[addr] - if cands: - ret = cands.pop().data + most_parent, parent_segment_info = self.get_parent_segment_info() + + if parent_segment_info is None: + # Somehow we don't have segment info, + # fallback to the unknown segment. + seg_meta = manager.unknown_segment + ret = seg_meta.find_symbol(addr, search_ranges) + elif in_segment: + if most_parent.owned_metadata is not None: + # Check if this address actually is inside the segment or not. + if most_parent.owned_metadata.in_vram_range(addr): + seg_meta = most_parent.owned_metadata + else: + # Avoid creating a symbol inside this segment if it doesn't belong to. + seg_meta = manager.unknown_segment + else: + # We don't have a reference to our own metadata? + # Try to retrieve it. + seg_meta = manager.find_owned_segment(parent_segment_info) + ret = seg_meta.find_symbol(addr, search_ranges) elif not local_only: - ret = most_parent.retrieve_symbol(symbols.all_symbols_dict, addr) - - if not ret and search_ranges: - cands = symbols.all_symbols_ranges[addr] - if cands: - ret = cands.pop().data - - # Create the symbol if it doesn't exist - if not ret and create: - ret = Symbol(addr, rom=rom, type=type) - symbols.add_symbol(ret) - - if in_segment: - ret.segment = most_parent - if addr not in most_parent.seg_symbols: - most_parent.seg_symbols[addr] = [] - most_parent.seg_symbols[addr].append(ret) - - if ret: - if define: - ret.defined = True - if reference: - ret.referenced = True - if ret.type is None: - ret.type = type - if ret.rom is None: - ret.rom = rom - if in_segment: - if ret.segment is None: - ret.segment = most_parent + # Try to find the symbol anywhere. + ret = manager.find_symbol_from_any_segment( + addr, + parent_segment_info, + search_ranges, + validation, + ) + _update_symbol(ret, reference, define, type) return ret def create_symbol( @@ -911,22 +841,53 @@ def create_symbol( define: bool = False, reference: bool = False, search_ranges: bool = False, - local_only: bool = False, ) -> Symbol: - ret = self.get_symbol( - addr, - in_segment=in_segment, - type=type, - create=True, - define=define, - reference=reference, - search_ranges=search_ranges, - local_only=local_only, - ) - assert ret is not None + most_parent, parent_segment_info = self.get_parent_segment_info() + + if parent_segment_info is None: + # Somehow we don't have segment info, + # fallback to the unknown segment. + seg_meta = manager.unknown_segment + elif in_segment: + # Check if we know our own segment metadata, + # if not, then default to look it up. + if most_parent.owned_metadata is not None: + if most_parent.owned_metadata.in_vram_range(addr): + seg_meta = most_parent.owned_metadata + else: + # Avoid creating a symbol inside this segment if it doesn't belong to. + seg_meta = manager.unknown_segment + else: + seg_meta = manager.find_owned_segment(parent_segment_info) + else: + # Try to figure out a feasible segment for this. + seg_meta = manager.find_referenced_segment_for_creation( + addr, + parent_segment_info, + ) + + ret = seg_meta.create_symbol(addr, search_ranges) + _update_symbol(ret, reference, define, type) return ret def __repr__(self) -> str: # Shows a nicer string on the debugging screen return f"{self.name} ({self.type})" + + +def _update_symbol( + ret: Optional[Symbol], + reference: bool, + define: bool, + type: Optional[str], +) -> None: + if ret is None: + return + + if define: + ret.defined = True + if reference: + ret.referenced = True + if ret.type is None: + ret.type = type diff --git a/src/splat/util/__init__.py b/src/splat/util/__init__.py index d08c7839..a569403d 100644 --- a/src/splat/util/__init__.py +++ b/src/splat/util/__init__.py @@ -14,3 +14,4 @@ from . import symbols as symbols from . import utils as utils from . import vram_classes as vram_classes +from . import metadata as metadata diff --git a/src/splat/util/metadata/__init__.py b/src/splat/util/metadata/__init__.py new file mode 100644 index 00000000..816721d5 --- /dev/null +++ b/src/splat/util/metadata/__init__.py @@ -0,0 +1,4 @@ +from . import overlay_metadata as overlay_metadata +from . import parent_segment_info as parent_segment_info +from . import segment_manager as segment_manager +from . import segment_metadata as segment_metadata diff --git a/src/splat/util/metadata/overlay_metadata.py b/src/splat/util/metadata/overlay_metadata.py new file mode 100644 index 00000000..fb219682 --- /dev/null +++ b/src/splat/util/metadata/overlay_metadata.py @@ -0,0 +1,79 @@ +import dataclasses +from typing import Optional, TYPE_CHECKING + +from .segment_metadata import SegmentMetadata, SegmentKind + +from .. import log + + +if TYPE_CHECKING: + from ...segtypes.common.segment import Segment + + +@dataclasses.dataclass +class OverlayMetadata: + """ + A group of overlays that share the same `exclusive_ram_id`. + """ + + exclusive_ram_id: str + + # Ranges covering all the overlays for this `exclusive_ram_id`. + rom_start: int + rom_end: int + vram_start: int + vram_end: int + + segments: dict[int, SegmentMetadata] + """key: rom address""" + + def in_rom_range(self, rom: int) -> bool: + if rom < self.rom_start: + return False + if rom >= self.rom_end: + return False + return True + + def in_vram_range(self, vram: int) -> bool: + if vram < self.vram_start: + return False + if vram >= self.vram_end: + return False + return True + + def add_segment( + self, + name: str, + rom_start: int, + rom_end: int, + vram_start: int, + vram_end: int, + prioritized_segments: list[str], + segment: Optional["Segment"], + ) -> SegmentMetadata: + old_segment = self.segments.get(rom_start) + if old_segment is not None: + log.error( + f"Tried to create an overlay at a duplicated rom address {rom_start} for exclusive_ram_id={self.exclusive_ram_id}.\n" + f" Old segment '{old_segment.name}'. Rom 0x{old_segment.rom_start:08X}~0x{old_segment.rom_end:08X}. Vram 0x{old_segment.vram_start:08X}~0x{old_segment.vram_start:08X}\n" + f" New segment '{name}'. Rom 0x{rom_start:08X}~0x{rom_end:08X}. Vram 0x{vram_start:08X}~0x{vram_start:08X}\n" + ) + + self.rom_start = min(self.rom_start, rom_start) + self.rom_end = max(self.rom_end, rom_end) + self.vram_start = min(self.vram_start, vram_start) + self.vram_end = max(self.vram_end, vram_end) + + seg = SegmentMetadata( + SegmentKind.Overlay, + name, + rom_start, + rom_end, + vram_start, + vram_end, + prioritized_segments, + self.exclusive_ram_id, + segment, + ) + self.segments[rom_start] = seg + return seg diff --git a/src/splat/util/metadata/parent_segment_info.py b/src/splat/util/metadata/parent_segment_info.py new file mode 100644 index 00000000..5a4d16f5 --- /dev/null +++ b/src/splat/util/metadata/parent_segment_info.py @@ -0,0 +1,25 @@ +import dataclasses +from typing import Optional + + +@dataclasses.dataclass +class ParentSegmentInfo: + """ + Information from the parent segment, used to locate the segment's metadata + or to locate segments that can be referenced by this parent segment. + """ + + name: str + segment_rom: int + segment_vram: int + exclusive_ram_id: Optional[str] + + def __repr__(self) -> str: + if self.exclusive_ram_id is None: + exclusive_ram_id = "None" + else: + exclusive_ram_id = f"{self.exclusive_ram_id!r}" + return f"ParentSegmentInfo(name={self.name}, segment_rom=0x{self.segment_rom:08X}, segment_vram=0x{self.segment_vram:08X}, exclusive_ram_id={exclusive_ram_id})" + + def __str__(self) -> str: + return self.__repr__() diff --git a/src/splat/util/metadata/segment_manager.py b/src/splat/util/metadata/segment_manager.py new file mode 100644 index 00000000..868e240e --- /dev/null +++ b/src/splat/util/metadata/segment_manager.py @@ -0,0 +1,687 @@ +from typing import Callable, Dict, Optional, Set, Tuple, TYPE_CHECKING + +from .segment_metadata import SegmentMetadata, SegmentKind +from .parent_segment_info import ParentSegmentInfo +from .overlay_metadata import OverlayMetadata + +from ..symbols import Symbol +from .. import log, options + +# circular import +if TYPE_CHECKING: + from ...segtypes.segment import Segment + + +class SegmentManager: + """ + Grouping for all segment metadatas. + """ + + def __init__(self) -> None: + # User-declared symbols that do not belong to any other segment. + self.absolute_segment: SegmentMetadata = SegmentMetadata( + SegmentKind.Absolute, + "$absolute", + 0x0, + 0x0, + 0x00000000, + 0xFFFFFFFF, + prioritized_segments=[], + exclusive_ram_id=None, + segment=None, + ) + + # Globally visible segments. + # They have no address overlapping issues with other segments. + self.global_segments: list[SegmentMetadata] = [] + + # Overlays. + # They have address overlapping between them. + self.overlay_segments: dict[str, OverlayMetadata] = {} + """key: exclusive_ram_id""" + + # Dumpster for failed segment lookups. + self.unknown_segment: SegmentMetadata = SegmentMetadata( + SegmentKind.Unknown, + "$unknown", + 0x0, + 0x0, + 0x00000000, + 0xFFFFFFFF, + prioritized_segments=[], + exclusive_ram_id=None, + segment=None, + ) + + # ? + self.all_symbols: list[Symbol] = [] + + self.global_rom_start: Optional[int] = None + self.global_rom_end: Optional[int] = None + self.global_vram_start: Optional[int] = None + self.global_vram_end: Optional[int] = None + + def find_owned_segment(self, info: ParentSegmentInfo) -> SegmentMetadata: + """ + Find the segment metadata corresponding to the given segment parent info. + """ + + if info.exclusive_ram_id is not None: + segments_per_rom = self.overlay_segments.get(info.exclusive_ram_id) + if segments_per_rom is not None: + owned_segment = segments_per_rom.segments.get(info.segment_rom) + if owned_segment is not None: + return owned_segment + else: + for owned_segment in self.global_segments: + if owned_segment.in_rom_range(info.segment_rom): + return owned_segment + elif owned_segment.in_vram_range(info.segment_vram): + # Global segment doesn't have overlapping issues, so it should + # be fine to check for vram address. + # This can be required by segments that only have bss sections. + return owned_segment + + # log.write(f"Error: Unable to find an owned segment for {info=}.", status="warn") + return self.unknown_segment + + def find_referenced_segment_for_creation( + self, + vram: int, + info: ParentSegmentInfo, + ) -> SegmentMetadata: + """ + Find a segment where a symbol with the given address could be created. + + This may return the corresponding owned segment if the vram address + corresponds to the segment, or a prioritized segment for the owned + segment that fits the given vram address. Global segments are always + checked, even if they aren't prioritized or owned. + """ + + # First, check the global segments. + # Overlays shouldn't overlap with the global segments, so this should be fine. + for seg in self.global_segments: + if seg.in_vram_range(vram): + return seg + if seg.rom_start == info.segment_rom: + segment = self._find_prioritized_segment(vram, seg) + if segment is not None: + return segment + + # Look up in overlays + if len(self.overlay_segments) > 0: + overlay_segment = self._find_referenced_overlay_segment_for_creation( + vram, + info, + ) + if overlay_segment is not None: + return overlay_segment + + # Fallback to the unknown segment + return self.unknown_segment + + def _find_referenced_overlay_segment_for_creation( + self, + vram: int, + info: ParentSegmentInfo, + ) -> Optional[SegmentMetadata]: + # If the parent info has no exclusive_ram_id, then it is a global segment, + # meaning it shouldn't be referencing an overlay symbol by default. + if info.exclusive_ram_id is None: + return None + + # Check the segment corresponding to this specific overlay. + segments_per_rom = self.overlay_segments.get(info.exclusive_ram_id) + if segments_per_rom is not None: + owned_segment = segments_per_rom.segments.get(info.segment_rom) + if owned_segment is not None: + if owned_segment.in_vram_range(vram): + return owned_segment + + # Check for any prioiritised overlay, if any. + segment = self._find_prioritized_segment(vram, owned_segment) + if segment is not None: + return segment + + # Don't check other overlay segments here! + # We don't have a way to know what segment this overlay is referencing, + # picking an arbitrary one for symbol creation will lead to nasty bugs. + + return None + + def _find_prioritized_segment( + self, + vram: int, + owned_segment: SegmentMetadata, + ) -> Optional[SegmentMetadata]: + """ + Find a prioritized segment that fits the given vram. + """ + for prioritized_segment in owned_segment.get_prioritized_segments(): + for segments_per_rom in self.overlay_segments.values(): + if not segments_per_rom.in_vram_range(vram): + continue + for segment in segments_per_rom.segments.values(): + if segment.name == prioritized_segment and segment.in_vram_range( + vram + ): + return segment + return None + + def find_symbol_from_any_segment( + self, + vram: int, + info: ParentSegmentInfo, + allow_addend: bool, + validate: Callable[[Symbol], bool], + ) -> Optional[Symbol]: + """ + Check all segments looking for a symbol matching the given address. + + Applies visibility rules based on the segment for `info`. + + The symbol will be checked against the `validate` callback, + if the callback returns `False` then the next segment will be checked + and so on. + """ + + # Absolute symbols first. + sym = self.absolute_segment.find_symbol(vram, allow_addend) + if sym is not None: + return sym + + # Global segments second. + for seg in self.global_segments: + if seg.in_vram_range(vram): + # If we find this vram is within a global segment then we can stop + # searching, because we know this should be the only segment that + # should overlap this segment. + sym = seg.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + return None + if seg.rom_start == info.segment_rom: + # Check for prioritized segments, if any. + sym = self._find_symbol_from_prioritized_segments( + vram, + allow_addend, + seg, + validate, + ) + if sym is not None: + return sym + + # Overlays third + if len(self.overlay_segments) > 0: + sym = self._find_symbol_from_overlay_segments( + vram, + info, + allow_addend, + validate, + ) + if sym is not None: + return sym + + # Lastly the dumpster + sym = self.unknown_segment.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + return None + + def _find_symbol_from_overlay_segments( + self, + vram: int, + info: ParentSegmentInfo, + allow_addend: bool, + validate: Callable[[Symbol], bool], + ) -> Optional[Symbol]: + exclusive_ram_id = info.exclusive_ram_id + + # First, look up for the segment associated to this exclusive_ram_id + # which matches the rom address of the parent segment so we can + # prioritize it. + if exclusive_ram_id is not None: + segments_per_rom = self.overlay_segments.get(exclusive_ram_id) + if segments_per_rom is not None: + owned_segment = segments_per_rom.segments.get(info.segment_rom) + if owned_segment is not None: + if owned_segment.in_vram_range(vram): + sym = owned_segment.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + return None + + # Check for prioritized segments, if any. + sym = self._find_symbol_from_prioritized_segments( + vram, + allow_addend, + owned_segment, + validate, + ) + if sym is not None: + return sym + + # If not found, then we should check every exclusive_ram_id except the + # one associated with the parent segment. + + # First, we look for exclusive_ram_id that contain a single segment. + # This way is less likely we grab the wrong symbol. + for ovl_id, segments_per_rom in self.overlay_segments.items(): + if exclusive_ram_id == ovl_id: + continue + if not segments_per_rom.in_vram_range(vram): + continue + + if len(segments_per_rom.segments) != 1: + continue + + for segment in segments_per_rom.segments.values(): + if segment.in_vram_range(vram): + sym = segment.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + + # if we haven't found the symbol yet, then just look up everywhere else. + for ovl_id, segments_per_rom in self.overlay_segments.items(): + if exclusive_ram_id == ovl_id: + continue + if not segments_per_rom.in_vram_range(vram): + continue + + if len(segments_per_rom.segments) == 1: + continue + + for segment in segments_per_rom.segments.values(): + if segment.in_vram_range(vram): + sym = segment.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + + return None + + def _find_symbol_from_prioritized_segments( + self, + vram: int, + allow_addend: bool, + owned_segment: SegmentMetadata, + validate: Callable[[Symbol], bool], + ) -> Optional[Symbol]: + for prioritized_segment in owned_segment.get_prioritized_segments(): + for segments_per_rom in self.overlay_segments.values(): + if not segments_per_rom.in_vram_range(vram): + continue + for segment in segments_per_rom.segments.values(): + if segment.name == prioritized_segment and segment.in_vram_range( + vram + ): + sym = segment.find_symbol(vram, allow_addend) + if sym is not None and validate(sym): + return sym + + return None + + def _add_global_segment( + self, + name: str, + rom_start: int, + rom_end: int, + vram_start: int, + vram_end: int, + prioritized_segments: list[str], + segment, + ) -> SegmentMetadata: + if self.global_rom_start is None or rom_start < self.global_rom_start: + self.global_rom_start = rom_start + if self.global_rom_end is None or self.global_rom_end < rom_end: + self.global_rom_end = rom_end + if self.global_vram_start is None or vram_start < self.global_vram_start: + self.global_vram_start = vram_start + if self.global_vram_end is None or self.global_vram_end < vram_end: + self.global_vram_end = vram_end + + seg_meta = SegmentMetadata( + SegmentKind.Global, + name, + rom_start, + rom_end, + vram_start, + vram_end, + prioritized_segments, + None, + segment, + ) + self.global_segments.append(seg_meta) + return seg_meta + + def _add_overlay_segment( + self, + exclusive_ram_id: str, + name: str, + rom_start: int, + rom_end: int, + vram_start: int, + vram_end: int, + prioritized_segments: list[str], + segment: Optional["Segment"], + ) -> SegmentMetadata: + ovl_meta = self.overlay_segments.setdefault( + exclusive_ram_id, + OverlayMetadata( + exclusive_ram_id, + rom_start, + rom_end, + vram_start, + vram_end, + {}, + ), + ) + return ovl_meta.add_segment( + name, + rom_start, + rom_end, + vram_start, + vram_end, + prioritized_segments, + segment, + ) + + def _initialize_segments( + self, + all_segments: "list[Segment]", + ) -> Tuple[Dict[str, SegmentMetadata], Set[str]]: + global_rom_start: Optional[int] = None + global_rom_end: Optional[int] = None + global_vram_start: Optional[int] = options.opts.global_vram_start + global_vram_end: Optional[int] = options.opts.global_vram_end + global_segment_largest_vram: Optional[Segment] = None + + seen_global_rom_start: Optional[int] = None + seen_global_rom_end: Optional[int] = None + seen_global_vram_start: Optional[int] = None + seen_global_vram_end: Optional[int] = None + overlay_segments: list[SegmentMetadata] = [] + + segments_by_name: Dict[str, SegmentMetadata] = {} + skipped_segments: Set[str] = set() + + global_segments: list[Segment] = [] + global_segments_after_overlays: list[Segment] = [] + + # Create all segments in the grouping + for segment in all_segments: + if ( + not isinstance(segment.vram_start, int) + or not isinstance(segment.vram_end, int) + or not isinstance(segment.rom_start, int) + or not isinstance(segment.rom_end, int) + ): + skipped_segments.add(segment.name) + continue + + ram_id = segment.get_exclusive_ram_id() + if ram_id is None and segment.special_vram_segment: + # Special segments which should not be accounted in the global VRAM calculation, like N64's IPL3 + ram_id = "$special_vram_segment" + + if ram_id is not None: + # Overlay + + if segment.vram_start == segment.vram_end: + # Skip zero-sized segments. + continue + + seg_meta = self._add_overlay_segment( + ram_id, + segment.name, + segment.rom_start, + segment.rom_end, + segment.vram_start, + segment.vram_end, + segment.prioritized_segments, + segment, + ) + segment.owned_metadata = seg_meta + overlay_segments.append(seg_meta) + segments_by_name[seg_meta.name] = seg_meta + else: + # Global segment + + seg_meta = self._add_global_segment( + segment.name, + segment.rom_start, + segment.rom_end, + segment.vram_start, + segment.vram_end, + segment.prioritized_segments, + segment, + ) + segment.owned_metadata = seg_meta + segments_by_name[seg_meta.name] = seg_meta + global_segments.append(segment) + + if global_rom_start is None or segment.rom_start < global_rom_start: + global_rom_start = segment.rom_start + + if global_rom_end is None or global_rom_end < segment.rom_end: + global_rom_end = segment.rom_end + + if global_vram_start is None or segment.vram_start < global_vram_start: + global_vram_start = segment.vram_start + + if global_vram_end is None: + global_vram_end = segment.vram_end + global_segment_largest_vram = segment + elif global_vram_end < segment.vram_end: + global_vram_end = segment.vram_end + global_segment_largest_vram = segment + + if len(overlay_segments) > 0: + # Global segment *after* overlay segments? + global_segments_after_overlays.append(segment) + + if ( + seen_global_rom_start is None + or segment.rom_start < seen_global_rom_start + ): + seen_global_rom_start = segment.rom_start + if seen_global_rom_end is None or seen_global_rom_end < segment.rom_end: + seen_global_rom_end = segment.rom_end + + if ( + seen_global_vram_start is None + or segment.vram_start < seen_global_vram_start + ): + seen_global_vram_start = segment.vram_start + if ( + seen_global_vram_end is None + or seen_global_vram_end < segment.vram_end + ): + seen_global_vram_end = segment.vram_end + + if ( + global_vram_start is not None + and global_vram_end is not None + and global_rom_start is not None + and global_rom_end is not None + ): + # Create extra global segments in case they are needed + if ( + seen_global_vram_start is not None + and seen_global_vram_end is not None + and seen_global_rom_start is not None + and seen_global_rom_end is not None + ): + # Account for options.opts.global_vram_start and options.opts.global_vram_end for PSX and PSP + if global_vram_start < seen_global_vram_start: + rom_start = ( + seen_global_rom_start + + global_vram_start + - seen_global_vram_start + ) + seg_meta = self._add_global_segment( + "$global_left", + rom_start, + seen_global_rom_start, + global_vram_start, + seen_global_vram_start, + [], + None, + ) + segments_by_name[seg_meta.name] = seg_meta + if global_vram_end > seen_global_vram_end: + rom_end = ( + seen_global_rom_end + global_vram_end - seen_global_vram_end + ) + seg_meta = self._add_global_segment( + "$global_right", + seen_global_rom_end, + rom_end, + seen_global_vram_end, + global_vram_end, + [], + None, + ) + segments_by_name[seg_meta.name] = seg_meta + + # Validation + + overlaps_found = False + # Check the vram range of the global segment does not overlap with any overlay segment + for ovl_segment in overlay_segments: + assert ovl_segment.vram_start <= ovl_segment.vram_end, ( + f"{ovl_segment.vram_start:08X} {ovl_segment.vram_end:08X}" + ) + if ( + ovl_segment.vram_end > global_vram_start + and global_vram_end > ovl_segment.vram_start + ): + log.write( + f"Error: Overlay segment {ovl_segment.name} with vram range ([0x{ovl_segment.vram_start:08X}, 0x{ovl_segment.vram_end:08X}]) of the non-global segment at rom address 0x{ovl_segment.rom_start:X} overlaps with the global vram range ([0x{global_vram_start:08X}, 0x{global_vram_end:08X}])", + status="warn", + ) + overlaps_found = True + if overlaps_found: + log.write( + "Overlaps between non-global and global segments were found.\n" + "This is usually caused by missing `exclusive_ram_id` tags on segments that have a higher vram address than other `exclusive_ram_id`-tagged segments" + ) + if len(global_segments) > 0: + log.write( + "These are all the global segments:", + status="warn", + ) + for seg in global_segments_after_overlays: + log.write( + f" '{seg.name}', rom: 0x{seg.rom_start:06X}, vram: 0x{seg.vram_start:08X}" + ) + + if global_segment_largest_vram is not None: + log.write( + f"The global segment with the largest vram seen is {global_segment_largest_vram}. Rom: 0x{global_segment_largest_vram.rom_start:X}, Vram: 0x{global_segment_largest_vram.vram_start:08X}" + ) + + if len(global_segments_after_overlays) > 0: + log.write( + "These segments are the main suspects for missing a `exclusive_ram_id` tag:", + status="warn", + ) + for seg in global_segments_after_overlays: + log.write( + f" '{seg.name}', rom: 0x{seg.rom_start:06X}, vram: 0x{seg.vram_start:08X}" + ) + else: + log.write("No suspected segments??", status="warn") + log.error("Stopping due to the above errors") + + return segments_by_name, skipped_segments + + def _initialize_symbols( + self, + all_symbols: "list[Symbol]", + segments_by_name: Dict[str, SegmentMetadata], + skipped_segments: Set[str], + ) -> None: + # Pass every symbol to its corresponding segment. + lost_symbols = [] + for sym in all_symbols: + # User segment takes priority over everything + if sym.absolute: + self.absolute_segment.add_user_symbol(sym) + continue + + # Then look up for explicit associated segments first. + if sym.segment is not None: + meta = segments_by_name.get(sym.segment.name) + if meta is not None: + meta.add_user_symbol(sym) + continue + elif sym.segment.name in skipped_segments: + log.write( + f"Error: Unable to associated '{sym}' to segment '{sym.segment}' because that segment is missing a vram/rom address.", + status="warn", + ) + else: + log.write( + f"Warning (Maybe bug): User-declared symbol '{sym}' is unexpectely associated to segment '{sym.segment}'.\n" + " This is an issue because unexpected segments should have been filtered on a previous step.\n" + " Please report.", + status="warn", + ) + + # Then try to look up for global segments. + found_global = False + for meta_seg in self.global_segments: + if meta_seg.in_vram_range(sym.vram_start): + meta_seg.add_user_symbol(sym) + found_global = True + break + if found_global: + continue + + # We run out of places to put this symbol into. + # We need the user to give us more info on what to do with this. + possible_segments = [ + f"{seg_meta.name} (Vram: 0x{seg_meta.vram_start:08X}, Rom: 0x{seg_meta.rom_start:X})" + for seg_meta in segments_by_name.values() + if seg_meta.in_vram_range(sym.vram_start) + ] + possible_segments_str = ( + f"[{', '.join(possible_segments)}]" + if len(possible_segments) > 0 + else "None" + ) + lost_symbols.append( + f"{sym.name} (Vram: 0x{sym.vram_start:08X}). Suspected segments: {possible_segments_str}" + ) + self.unknown_segment.add_user_symbol(sym) + + if len(lost_symbols) > 0: + log.write( + "\nError: Unable to determine a segment for the following user-declared symbols.\n" + " Try specifying the segment they belong to with 'segment:segment_name' in your symbol_addrs file.\n" + " If the address of this symbol is not part of any segment, or if you believe this symbol should be\n" + " globally visible and take priority over other symbol references then use the `absolute:True`\n" + " user attribute instead.", + status="warn", + ) + log.write(" " + "\n ".join(lost_symbols)) + log.write("\n") + # TODO: uncomment on a future version + # log.error("Stopping due to the above issues.") + + self.all_symbols = all_symbols + + +manager = SegmentManager() + + +def initialize(all_segments: "list[Segment]", all_symbols: "list[Symbol]") -> None: + global manager + segments_by_name, skipped_segments = manager._initialize_segments(all_segments) + manager._initialize_symbols(all_symbols, segments_by_name, skipped_segments) + + +def reset() -> None: + global manager + manager = SegmentManager() diff --git a/src/splat/util/metadata/segment_metadata.py b/src/splat/util/metadata/segment_metadata.py new file mode 100644 index 00000000..973fbd3c --- /dev/null +++ b/src/splat/util/metadata/segment_metadata.py @@ -0,0 +1,262 @@ +import dataclasses +import enum +from typing import Optional, TYPE_CHECKING + +from spimdisasm.common import SortedDict + +from ..symbols import Symbol, label_types, get_all_symbols +from .. import log + +from .parent_segment_info import ParentSegmentInfo + +if TYPE_CHECKING: + from ...segtypes.common.segment import Segment + + +class SegmentKind(enum.Enum): + """ + The type of segment metadata. + """ + + Global = 0 + """ + Globally vissible segment, usually the first segments. + + It has no overlapping addresses with any other segment. + """ + + Overlay = 1 + """ + Segments that have overlapping Vram addresses with other segments. + + This kind of segment have an `exclusive_ram_id` to group them. + """ + + Unknown = 2 + """ + Junk. + + Referenced addresses that we couldn't find to which segment they belong to. + """ + + Absolute = 3 + """ + User-declared symbols that are not part of the address space of any segment. + """ + + +@dataclasses.dataclass +class SegmentMetadata: + """ + Metadata for a top-level segment. + """ + + kind: SegmentKind + name: str + + rom_start: int + rom_end: int + vram_start: int + vram_end: int + + prioritized_segments: list[str] + """ + Other segments this segment is allowed to see and prioritize. + """ + + exclusive_ram_id: Optional[str] + + segment: Optional["Segment"] + """ + Reference to the corresponding top-level segment. + + This is `None` for `Unknown` and `Absolute` segments. + """ + + symbols: SortedDict[Symbol] + """ + All symbols defined in this segment. + + This only contains symbols that are part of this segment. + Sorted by Vram address. + """ + + def __init__( + self, + kind: SegmentKind, + name: str, + rom_start: int, + rom_end: int, + vram_start: int, + vram_end: int, + prioritized_segments: list[str], + exclusive_ram_id: Optional[str], + segment: Optional["Segment"], + ) -> None: + if rom_start > rom_end: + log.error( + f"Error creating segment '{name}': rom_start (0x{rom_start:X}) is larger than rom_end (0x{rom_end:X})" + ) + if vram_start > vram_end: + log.error( + f"Error creating segment '{name}': rom_start (0x{vram_start:X}) is larger than rom_end (0x{vram_end:X})" + ) + + self.kind = kind + self.name = name + self.rom_start = rom_start + self.rom_end = rom_end + self.vram_start = vram_start + self.vram_end = vram_end + self.prioritized_segments = prioritized_segments + self.exclusive_ram_id = exclusive_ram_id + self.segment = segment + + self.symbols = SortedDict() + + def in_rom_range(self, rom: int) -> bool: + if rom < self.rom_start: + return False + if rom >= self.rom_end: + return False + return True + + def in_vram_range(self, vram: int) -> bool: + if vram < self.vram_start: + return False + if vram >= self.vram_end: + return False + return True + + def rom_from_vram(self, vram: int) -> Optional[int]: + """ + Converts a Vram address into a Rom address. + + Returns `None` if either: + - This segment is either the Absolute or Unknown special segments. + - The given Vram is outside the Vram space of this segment. + """ + + if self.kind == SegmentKind.Unknown or self.kind == SegmentKind.Absolute: + return None + + if not self.in_vram_range(vram): + return None + rom = vram - self.vram_start + self.rom_start + return rom + + def is_owned_segment(self, info: ParentSegmentInfo) -> bool: + """ + Checks if the passed info corresponds to this segment metadata. + """ + if self.kind == SegmentKind.Absolute or self.kind == SegmentKind.Unknown: + return False + + if self.exclusive_ram_id is None and info.exclusive_ram_id is None: + # Global segment + if not self.in_vram_range(info.segment_vram): + return False + return True + + else: + # Overlay + if self.exclusive_ram_id != info.exclusive_ram_id: + return False + if self.rom_start != info.segment_rom: + return False + return True + + def create_symbol(self, vram: int, allow_addend: bool) -> Symbol: + """ + Creates an autogenerated symbol (not user-declared). + """ + + if not self.in_vram_range(vram): + log.write( + f"\nWARNING: Bug! Adding symbol 0x{vram:08X} to segment '{self.name}' ({self.kind}),\n" + f" but the address of the symbol is outside the segment vram range (0x{self.vram_start:08X} ~ 0x{self.vram_end:08X})\n", + status="warn", + ) + + symbol = self.find_symbol(vram, allow_addend) + if symbol is None: + # Pass as much info as we can to this symbol when creating it. + symbol = Symbol( + vram, + rom=self.rom_from_vram(vram), + segment=self.segment, + seg_meta=self, + ) + self.symbols[vram] = symbol + get_all_symbols().append(symbol) + if self.kind == SegmentKind.Unknown: + symbol.unknown_segment = True + + return symbol + + def add_user_symbol(self, sym: Symbol) -> None: + """ + Add an user declared symbol. + """ + + if not self.in_vram_range(sym.vram_start): + log.write( + f"\nWARNING: Adding user-declared symbol '{sym.name}' (0x{sym.vram_start:08X}) to segment '{self.name}' ({self.kind}),\n" + f"but the address of the symbol is outside the segment vram range (0x{self.vram_start:08X} ~ 0x{self.vram_end:08X})\n" + f"This is accepted for now, but may become a hard error in the future. Expect buggy symbol references.\n", + status="warn", + ) + + existing_sym = self.find_symbol(sym.vram_start, True) + if existing_sym is not None: + if existing_sym is sym: + # there's a bug somewhere... + pass + elif sym.type in {"alabel", *label_types} or existing_sym in { + "alabel", + *label_types, + }: + # It is expected for labels to overlap with functions + pass + else: + if existing_sym.given_size is None: + existing_size = "None" + else: + existing_size = f"0x{existing_sym.given_size:X}" + size = f"0x{sym.given_size:X}" if sym.given_size is not None else "None" + msg = ( + f"The user declared symbol '{sym.name}' (Vram 0x{sym.vram_start:08X}, size {size}, segment '{sym.segment}')\n" + f" overlaps with the previously defined '{existing_sym.name}' (Vram 0x{existing_sym.vram_start:08X}, size {existing_size}, segment '{existing_sym.segment}')" + ) + # TODO: Change this into a hard error. + if False: + log.error(f"\nERROR: {msg}") + else: + log.write(f"\nWARNING: {msg}", status="warn") + + self.symbols[sym.vram_start] = sym + if sym.rom is None: + sym.rom = self.rom_from_vram(sym.vram_start) + sym.seg_meta = self + + def find_symbol(self, vram: int, allow_addend: bool) -> Optional[Symbol]: + """ + Lookup for a symbol given its address, + optionally checking its size as a ranged search. + """ + + if allow_addend: + pair = self.symbols.getKeyRight(vram, True) + if pair is None: + return None + + symbol_vram, sym = pair + size = sym.given_size or 1 + if vram >= symbol_vram + size: + return None + return sym + + return self.symbols.get(vram) + + def get_prioritized_segments(self) -> list[str]: + return self.prioritized_segments diff --git a/src/splat/util/symbols.py b/src/splat/util/symbols.py index 7b47bd07..d612afae 100644 --- a/src/splat/util/symbols.py +++ b/src/splat/util/symbols.py @@ -4,19 +4,18 @@ import spimdisasm -from intervaltree import IntervalTree from ..disassembler import disassembler_instance from pathlib import Path # circular import if TYPE_CHECKING: from ..segtypes.segment import Segment + from .metadata.segment_manager import SegmentManager + from ..util.metadata.segment_metadata import SegmentMetadata from . import log, options, progress_bar all_symbols: List["Symbol"] = [] -all_symbols_dict: Dict[int, List["Symbol"]] = {} -all_symbols_ranges = IntervalTree() ignored_addresses: Set[int] = set() to_mark_as_defined: Set[str] = set() @@ -26,7 +25,8 @@ TRUEY_VALS = ["true", "on", "yes", "y"] FALSEY_VALS = ["false", "off", "no", "n"] -splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} +splat_sym_types = {"func", "jtbl", "jtbl_label", "label", "alabel"} +label_types = {"jtbl_label", "label"} ILLEGAL_FILENAME_CHARS = ["<", ">", ":", '"', "/", "\\", "|", "?", "*"] @@ -52,16 +52,13 @@ def is_falsey(str: str) -> bool: return str.lower() in FALSEY_VALS -def add_symbol(sym: "Symbol"): +def add_symbol(sym: "Symbol", all_symbols_dict: Dict[int, List["Symbol"]]): all_symbols.append(sym) - if sym.vram_start is not None: - if sym.vram_start not in all_symbols_dict: - all_symbols_dict[sym.vram_start] = [] - all_symbols_dict[sym.vram_start].append(sym) - - # For larger symbols, add their ranges to interval trees for faster lookup - if sym.size > 4: - all_symbols_ranges.addi(sym.vram_start, sym.vram_end, sym) + items = all_symbols_dict.get(sym.vram_start) + if items is None: + all_symbols_dict[sym.vram_start] = [sym] + else: + items.append(sym) def to_cname(symbol_name: str) -> str: @@ -74,7 +71,9 @@ def to_cname(symbol_name: str) -> str: def handle_sym_addrs( - path: Path, sym_addrs_lines: List[str], all_segments: "List[Segment]" + path: Path, + sym_addrs_lines: List[str], + all_segments: "List[Segment]", ): def get_seg_for_name(name: str) -> Optional["Segment"]: for segment in all_segments: @@ -88,6 +87,8 @@ def get_seg_for_rom(rom: int) -> Optional["Segment"]: return segment return None + all_symbols_dict: Dict[int, List["Symbol"]] = {} + seen_symbols: Dict[str, "Symbol"] = {} prog_bar = progress_bar.get_progress_bar(sym_addrs_lines) prog_bar.set_description(f"Loading symbols ({path.stem})") @@ -270,6 +271,8 @@ def get_seg_for_rom(rom: int) -> Optional["Segment"]: continue if attr_name == "use_non_matching_label": sym.use_non_matching_label = tf_val + if attr_name == "absolute": + sym.absolute = tf_val if ignore_sym: if sym.given_size is None or sym.given_size == 0: @@ -278,68 +281,103 @@ def get_seg_for_rom(rom: int) -> Optional["Segment"]: spim_context.addBannedSymbolRangeBySize( sym.vram_start, sym.given_size ) - continue - if sym.segment is None and sym.rom is not None: - sym.segment = get_seg_for_rom(sym.rom) - - if sym.segment: - sym.segment.add_symbol(sym) - - sym.user_declared = True + if sym.absolute: + sym.defined = True - if sym.name in seen_symbols: - item = seen_symbols[sym.name] - if not sym.allow_duplicated or not item.allow_duplicated: + # Most attributes make no sense for a absolute symbol. + # For now just warn about these two. + if sym.rom is not None: log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicate symbol detected! {sym.name} has already been defined at vram 0x{item.vram_start:08X}" + log.write( + f"Warning: The symbol '{sym.name}' (0x{sym.vram_start:08X}) has both a rom address (0x{sym.rom:08X}) and absolute:True.", + status="warn", ) - - if addr in all_symbols_dict: - items = all_symbols_dict[addr] - for item in items: - have_same_rom_addresses = sym.rom == item.rom - same_segment = sym.segment == item.segment - - if have_same_rom_addresses and same_segment: - if not sym.allow_duplicated or not item.allow_duplicated: + if sym.segment is not None: + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Warning: The symbol '{sym.name}' (0x{sym.vram_start:08X}) has both an associated segment ({sym.segment}) and absolute:True.", + status="warn", + ) + else: + if sym.segment is None and sym.rom is not None: + sym.segment = get_seg_for_rom(sym.rom) + + if sym.segment: + if ( + sym.segment.vram_start is not None + and sym.segment.vram_end is not None + and not sym.segment.contains_vram(sym.vram_start) + ): + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Warning: User-declared symbol '{sym.name}' was associated to segment '{sym.segment.name}', " + "but its address is outside the segment's vram range.\n" + f" The symbol's Vram 0x{sym.vram_start:08X} is outside from the segment's Vram range 0x{sym.segment.vram_start:08X} ~ 0x{sym.segment.vram_end:08X}", + status="warn", + ) + if sym.rom is not None: + expected_rom = sym.segment.ram_to_rom(sym.vram_start) + if expected_rom is not None and expected_rom != sym.rom: log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicate symbol detected! {sym.name} clashes with {item.name} defined at vram 0x{addr:08X}.\n If this is intended, specify either a segment or a rom address for this symbol" + log.write( + f"Warning: User-declared symbol '{sym.name}' has a wrong user-declared rom address.\n" + f" This symbol has been mapped to segment '{sym.segment.name}', but the expected Rom\n" + f" address for a symbol with Vram address 0x{sym.vram_start:08X} in that segment is\n" + f" 0x{expected_rom:X}, but the given Rom address is 0x{sym.rom:X}.", + status="warn", ) - if len(sym.filename) > 253 or any( - c in ILLEGAL_FILENAME_CHARS for c in sym.filename - ): - log.parsing_error_preamble(path, line_num, line) - log.error( - # sym.name is written on its own line so reading the error message is nicer because the sym name will be very long. - # Other lines have two spaces to make identation nicer and consistent - f"Ilegal symbol filename detected!\n" - f" The symbol\n" - f" {sym.name}\n" - f" exceeds the 255 bytes filename limit that most OS imposes or uses illegal characters,\n" - f" which will be a problem when writing the symbol to its own file.\n" - f" To fix this specify a `filename` for this symbol, like `filename:func_{sym.vram_start:08X}`.\n" - f" Make sure the filename does not exceed 253 bytes nor it contains any of the following characters:\n" - f" {ILLEGAL_FILENAME_CHARS}" - ) + sym.user_declared = True + + if sym.name in seen_symbols: + item = seen_symbols[sym.name] + if not sym.allow_duplicated or not item.allow_duplicated: + log.parsing_error_preamble(path, line_num, line) + log.error( + f"Duplicate symbol detected! {sym.name} has already been defined at vram 0x{item.vram_start:08X}" + ) + + items = all_symbols_dict.get(addr) + if items is not None: + for item in items: + have_same_rom_addresses = sym.rom == item.rom + same_segment = sym.segment == item.segment + + if have_same_rom_addresses and same_segment: + if not sym.allow_duplicated or not item.allow_duplicated: + log.parsing_error_preamble(path, line_num, line) + log.error( + f"Duplicate symbol detected! {sym.name} clashes with {item.name} defined at vram 0x{addr:08X}.\n If this is intended, specify either a segment or a rom address for this symbol" + ) + + if len(sym.filename) > 253 or any( + c in ILLEGAL_FILENAME_CHARS for c in sym.filename + ): + log.parsing_error_preamble(path, line_num, line) + log.error( + # sym.name is written on its own line so reading the error message is nicer because the sym name will be very long. + # Other lines have two spaces to make identation nicer and consistent + f"Ilegal symbol filename detected!\n" + f" The symbol\n" + f" {sym.name}\n" + f" exceeds the 255 bytes filename limit that most OS imposes or uses illegal characters,\n" + f" which will be a problem when writing the symbol to its own file.\n" + f" To fix this specify a `filename` for this symbol, like `filename:func_{sym.vram_start:08X}`.\n" + f" Make sure the filename does not exceed 253 bytes nor it contains any of the following characters:\n" + f" {ILLEGAL_FILENAME_CHARS}" + ) - seen_symbols[sym.name] = sym + seen_symbols[sym.name] = sym - add_symbol(sym) + add_symbol(sym, all_symbols_dict) def initialize(all_segments: "List[Segment]"): global all_symbols - global all_symbols_dict - global all_symbols_ranges all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() # Manual list of func name / addrs for path in options.opts.symbol_addrs_paths: @@ -349,155 +387,71 @@ def initialize(all_segments: "List[Segment]"): handle_sym_addrs(path, sym_addrs_lines, all_segments) -def initialize_spim_context(all_segments: "List[Segment]") -> None: - global_vrom_start = None - global_vrom_end = None - global_vram_start = options.opts.global_vram_start - global_vram_end = options.opts.global_vram_end - overlay_segments: Set[spimdisasm.common.SymbolsSegment] = set() - +def initialize_spim_context(manager: "SegmentManager") -> None: spim_context.bannedSymbols |= ignored_addresses - from ..segtypes.common.code import CommonSegCode - - global_segments_after_overlays: List[CommonSegCode] = [] - - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - if segment.special_vram_segment: - # Special segments which should not be accounted in the global VRAM calculation, like N64's IPL3 - continue - - if ( - not isinstance(segment.vram_start, int) - or not isinstance(segment.vram_end, int) - or not isinstance(segment.rom_start, int) - or not isinstance(segment.rom_end, int) - ): - continue - - ram_id = segment.get_exclusive_ram_id() - - if ram_id is None: - if global_vram_start is None: - global_vram_start = segment.vram_start - elif segment.vram_start < global_vram_start: - global_vram_start = segment.vram_start - - if global_vram_end is None: - global_vram_end = segment.vram_end - elif global_vram_end < segment.vram_end: - global_vram_end = segment.vram_end - - if len(overlay_segments) > 0: - # Global segment *after* overlay segments? - global_segments_after_overlays.append(segment) - - if global_vrom_start is None: - global_vrom_start = segment.rom_start - elif segment.rom_start < global_vrom_start: - global_vrom_start = segment.rom_start - - if global_vrom_end is None: - global_vrom_end = segment.rom_end - elif global_vrom_end < segment.rom_end: - global_vrom_end = segment.rom_end - - elif segment.vram_start != segment.vram_end: - # Do not tell to spimdisasm about zero-sized segments. - - spim_segment = spim_context.addOverlaySegment( - ram_id, - segment.rom_start, - segment.rom_end, - segment.vram_start, - segment.vram_end, - ) - # Add the segment-specific symbols first - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_segment, sym) - - overlay_segments.add(spim_segment) - + # Initialize spimdisasm's global ranges. if ( - global_vram_start is not None - and global_vram_end is not None - and global_vrom_start is not None - and global_vrom_end is not None + manager.global_rom_start is not None + and manager.global_rom_end is not None + and manager.global_vram_start is not None + and manager.global_vram_end is not None ): spim_context.changeGlobalSegmentRanges( - global_vrom_start, global_vrom_end, global_vram_start, global_vram_end + manager.global_rom_start, + manager.global_rom_end, + manager.global_vram_start, + manager.global_vram_end, ) - overlaps_found = False - # Check the vram range of the global segment does not overlap with any overlay segment - for ovl_segment in overlay_segments: - assert ovl_segment.vramStart <= ovl_segment.vramEnd, ( - f"{ovl_segment.vramStart:08X} {ovl_segment.vramEnd:08X}" - ) - if ( - ovl_segment.vramEnd > global_vram_start - and global_vram_end > ovl_segment.vramStart - ): - log.write( - f"Error: the vram range ([0x{ovl_segment.vramStart:08X}, 0x{ovl_segment.vramEnd:08X}]) of the non-global segment at rom address 0x{ovl_segment.vromStart:X} overlaps with the global vram range ([0x{global_vram_start:08X}, 0x{global_vram_end:08X}])", - status="warn", - ) - overlaps_found = True - if overlaps_found: - log.write( - "Many overlaps between non-global and global segments were found.", - ) - log.write( - "This is usually caused by missing `exclusive_ram_id` tags on segments that have a higher vram address than other `exclusive_ram_id`-tagged segments" + # Pass absolute symbols. + for sym in manager.absolute_segment.symbols.values(): + add_symbol_to_spimdisasm_segment(spim_context.absoluteSegment, sym) + + all_segment_names = {seg.name for seg in manager.global_segments} | { + seg.name + for overlay_cat in manager.overlay_segments.values() + for seg in overlay_cat.segments.values() + } + + # Pass global symbols + for seg_meta in manager.global_segments: + for sym in seg_meta.symbols.values(): + for prioritized_seg in seg_meta.prioritized_segments: + if prioritized_seg not in all_segment_names: + log.error( + f"\nError: The segment '{seg_meta.name}' references non-existing '{prioritized_seg}' segment. Stopping." + ) + spim_context.globalSegment.addPrioritizedSegment(prioritized_seg) + add_symbol_to_spimdisasm_segment(spim_context.globalSegment, sym) + + # Create overlays and pass their symbols. + for ovl_id, segments_per_rom in manager.overlay_segments.items(): + for seg_meta in segments_per_rom.segments.values(): + spimdisasm_segment = spim_context.addOverlaySegment( + ovl_id, + seg_meta.rom_start, + seg_meta.rom_end, + seg_meta.vram_start, + seg_meta.vram_end, + seg_meta.name, ) - if len(global_segments_after_overlays) > 0: - log.write( - "These segments are the main suspects for missing a `exclusive_ram_id` tag:", - status="warn", - ) - for seg in global_segments_after_overlays: - log.write(f" '{seg.name}', rom: 0x{seg.rom_start:06X}") - else: - log.write("No suspected segments??", status="warn") - log.error("Stopping due to the above errors") - - # pass the global symbols to spimdisasm - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - ram_id = segment.get_exclusive_ram_id() - if ram_id is not None: - continue - - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_context.globalSegment, sym) - - if global_vram_start and global_vram_end: - # Pass global symbols to spimdisasm that are not part of any segment on the binary we are splitting (for psx and psp) - for sym in all_symbols: - if sym.segment is not None: - # We already handled this symbol somewhere else - continue - - if sym.vram_start < global_vram_start or sym.vram_end > global_vram_end: - # Not global - continue + for prioritized_seg in seg_meta.prioritized_segments: + if prioritized_seg not in all_segment_names: + log.error( + f"\nError: The segment '{seg_meta.name}' references non-existing '{prioritized_seg}' segment. Stopping." + ) + spimdisasm_segment.addPrioritizedSegment(prioritized_seg) - add_symbol_to_spim_segment(spim_context.globalSegment, sym) + for sym in seg_meta.symbols.values(): + add_symbol_to_spimdisasm_segment(spimdisasm_segment, sym) -def add_symbol_to_spim_segment( - segment: spimdisasm.common.SymbolsSegment, sym: "Symbol" +def add_symbol_to_spimdisasm_segment( + segment: spimdisasm.common.SymbolsSegment, + sym: "Symbol", ) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": + if sym.type == "func" or sym.type == "alabel": context_sym = segment.addFunction( sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom ) @@ -554,102 +508,55 @@ def add_symbol_to_spim_segment( return context_sym -def add_symbol_to_spim_section( - section: spimdisasm.mips.sections.SectionBase, sym: "Symbol" -) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": - context_sym = section.addFunction( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl": - context_sym = section.addJumpTable( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl_label": - context_sym = section.addJumpTableLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "label": - context_sym = section.addBranchLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - else: - context_sym = section.addSymbol( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - if sym.type is not None: - context_sym.type = sym.type - - if sym.user_declared: - context_sym.isUserDeclared = True - if sym.defined: - context_sym.isDefined = True - if sym.rom is not None: - context_sym.vromAddress = sym.rom - if sym.given_size is not None: - context_sym.size = sym.size - if sym.force_migration: - context_sym.forceMigration = True - if sym.force_not_migration: - context_sym.forceNotMigration = True - context_sym.functionOwnerForMigration = sym.function_owner - context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) - if sym.given_name_end: - context_sym.nameEnd = sym.given_name_end - if sym.given_visibility: - context_sym.visibility = sym.given_visibility - if sym.given_align: - context_sym.setAlignment(sym.given_align) - if sym.use_non_matching_label is not None: - context_sym.useNonMatchingLabel = sym.use_non_matching_label - - return context_sym - - -# force_in_segment=True when the symbol belongs to this specific segment. +# force_in_segment=True when the symbol belongs to this specific parent segment. # force_in_segment=False when this symbol is just a reference. def create_symbol_from_spim_symbol( - segment: "Segment", + most_parent: "Segment", + current_segment: "Segment", context_sym: spimdisasm.common.ContextSymbol, *, force_in_segment: bool, ) -> "Symbol": - in_segment = False - sym_type = None if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: - in_segment = True sym_type = "jtbl" elif context_sym.type == spimdisasm.common.SymbolSpecialType.function: sym_type = "func" elif context_sym.type == spimdisasm.common.SymbolSpecialType.branchlabel: - in_segment = True sym_type = "label" elif context_sym.type == spimdisasm.common.SymbolSpecialType.jumptablelabel: - in_segment = True sym_type = "jtbl_label" - if not in_segment: - if ( - context_sym.overlayCategory is None - and segment.get_exclusive_ram_id() is None - ): - in_segment = segment.contains_vram(context_sym.vram) - elif context_sym.overlayCategory == segment.get_exclusive_ram_id(): - if context_sym.vromAddress is not None: - in_segment = segment.contains_rom(context_sym.vromAddress) - else: - in_segment = segment.contains_vram(context_sym.vram) + in_segment = False + if ( + context_sym.overlayCategory is None + and most_parent.get_exclusive_ram_id() is None + ): + in_segment = most_parent.contains_vram(context_sym.vram) + elif context_sym.overlayCategory == most_parent.get_exclusive_ram_id(): + if context_sym.vromAddress is not None: + in_segment = most_parent.contains_rom(context_sym.vromAddress) + else: + in_segment = most_parent.contains_vram(context_sym.vram) - sym = segment.create_symbol( - context_sym.vram, force_in_segment or in_segment, type=sym_type, reference=True + sym = most_parent.create_symbol( + context_sym.vram, + force_in_segment or in_segment, + type=sym_type, + reference=True, ) + # Avoid overriding names for user declared symbols + if sym.given_name is not None and context_sym.name is None: + context_sym.name = sym.given_name if sym.given_name is None and context_sym.name is not None: sym.given_name = context_sym.name # To keep the symbol name in sync between splat and spimdisasm - context_sym.setNameGetCallback(lambda _: sym.name) + if in_segment: + context_sym.setNameGetCallback(lambda _: sym.name) + else: + context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) if context_sym.size is not None: sym.given_size = context_sym.getSize() @@ -660,6 +567,17 @@ def create_symbol_from_spim_symbol( if context_sym.referenceCounter > 0: sym.referenced = True + # Void the autodetected symbol type if it is a branch target, but the + # symbol isn't part of a text section. + # This may happen on handwritten asm where the function jumps to some + # data/bss symbol. This can be seen on libultra's monoutil.s (__isExp). + if ( + not current_segment.is_text() + and sym_type is None + and sym.type in ("label", "jtbl_label") + ): + sym.type = None + return sym @@ -710,6 +628,10 @@ class Symbol: use_non_matching_label: Optional[bool] = None + unknown_segment: bool = False + absolute: bool = False + seg_meta: Optional["SegmentMetadata"] = None + _generated_default_name: Optional[str] = None _last_type: Optional[str] = None @@ -814,12 +736,14 @@ def get_all_symbols(): def reset_symbols(): global all_symbols - global all_symbols_dict - global all_symbols_ranges global ignored_addresses global to_mark_as_defined + global spim_context all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() ignored_addresses = set() to_mark_as_defined = set() + spim_context = spimdisasm.common.Context() + + from .metadata import segment_manager + + segment_manager.reset() diff --git a/test.py b/test.py index df4e59be..c06c7b25 100755 --- a/test.py +++ b/test.py @@ -10,7 +10,7 @@ from src.splat import __version__ from src.splat.disassembler import disassembler_instance from src.splat.scripts.split import main -from src.splat.util import symbols, options +from src.splat.util import symbols, options, metadata from src.splat.segtypes.common.rodata import CommonSegRodata from src.splat.segtypes.common.code import CommonSegCode from src.splat.segtypes.common.c import CommonSegC @@ -188,7 +188,7 @@ def test_check_valid_type(self): for type in spim_types: assert symbols.check_valid_type(type) - def test_add_symbol_to_spim_segment(self): + def test_add_symbol_to_spimdisasm_segment(self): segment = spimdisasm.common.SymbolsSegment( context=spimdisasm.common.Context(), vromStart=0x0, @@ -201,28 +201,7 @@ def test_add_symbol_to_spim_segment(self): sym.defined = True sym.rom = 0x0 sym.type = "func" - result = symbols.add_symbol_to_spim_segment(segment, sym) - assert result.type == spimdisasm.common.SymbolSpecialType.function - assert sym.user_declared == result.isUserDeclared - assert sym.defined == result.isDefined - - def test_add_symbol_to_spim_section(self): - section = spimdisasm.mips.sections.SectionBase( - context=spimdisasm.common.Context(), - vromStart=0x0, - vromEnd=0x10, - vram=0x40000000, - filename="test", - words=[], - sectionType=spimdisasm.common.FileSectionType.Text, - segmentVromStart=0x0, - overlayCategory=None, - ) - sym = symbols.Symbol(0x100) - sym.type = "func" - sym.user_declared = False - sym.defined = True - result = symbols.add_symbol_to_spim_section(section, sym) + result = symbols.add_symbol_to_spimdisasm_segment(segment, sym) assert result.type == spimdisasm.common.SymbolSpecialType.function assert sym.user_declared == result.isUserDeclared assert sym.defined == result.isDefined @@ -253,7 +232,7 @@ def test_create_symbol_from_spim_symbol(self): segment=spimdisasm_segment, ) result = symbols.create_symbol_from_spim_symbol( - segment, context_sym, force_in_segment=False + segment, segment, context_sym, force_in_segment=False ) assert result.referenced assert result.extract @@ -273,7 +252,7 @@ def get_yaml(): class Rodata(unittest.TestCase): - def test_disassemble_data(self): + def test_disassemble_rodata(self): test_init() common_seg_rodata = CommonSegRodata( rom_start=0x0, @@ -284,13 +263,17 @@ def test_disassemble_data(self): args=[], yaml={}, ) + metadata.segment_manager.initialize([common_seg_rodata], []) + symbols.initialize_spim_context(metadata.segment_manager.manager) + rom_data = list(range(0x100)) common_seg_rodata.disassemble_data(bytes(rom_data)) assert common_seg_rodata.spim_section is not None assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203 - assert symbols.get_all_symbols()[0].vram_start == 0x400 - assert symbols.get_all_symbols()[0].segment == common_seg_rodata - assert symbols.get_all_symbols()[0].linker_section == ".rodata" + sym = symbols.get_all_symbols()[0] + assert sym.vram_start == 0x400, f"{sym.vram_start=}" + assert sym.segment == common_seg_rodata, f"{sym.segment=}" + assert sym.linker_section == ".rodata", f"{sym.linker_section=}" def test_get_possible_text_subsegment_for_symbol(self): context = spimdisasm.common.Context() @@ -435,7 +418,7 @@ def test_attrs(self): test_init() sym_addrs_lines = [ - "func_1 = 0x100; // type:func size:10 rom:100 segment:test_segment name_end:the_name_end " + "func_1 = 0x300; // type:func size:10 rom:0x100 segment:test_segment name_end:the_name_end " ] all_segments = [ @@ -454,7 +437,7 @@ def test_attrs(self): assert symbols.all_symbols[0].given_name == "func_1" assert symbols.all_symbols[0].type == "func" assert symbols.all_symbols[0].given_size == 10 - assert symbols.all_symbols[0].rom == 100 + assert symbols.all_symbols[0].rom == 0x100 assert symbols.all_symbols[0].segment == all_segments[0] assert symbols.all_symbols[0].given_name_end == "the_name_end" @@ -545,7 +528,9 @@ def test_overlay(self): # force this since it's hard to set up all_segments[0].exclusive_ram_id = "overlay" - symbols.initialize_spim_context(all_segments) + metadata.segment_manager.initialize(all_segments, symbols.all_symbols) + + symbols.initialize_spim_context(metadata.segment_manager.manager) # spim should have added something to overlaySegments assert ( type(symbols.spim_context.overlaySegments["overlay"][0x1000]) @@ -584,9 +569,11 @@ def test_global(self): ) ] + metadata.segment_manager.initialize(all_segments, symbols.all_symbols) + assert symbols.spim_context.globalSegment.vramStart == 0x80000000 assert symbols.spim_context.globalSegment.vramEnd == 0x80001000 - symbols.initialize_spim_context(all_segments) + symbols.initialize_spim_context(metadata.segment_manager.manager) assert symbols.spim_context.globalSegment.vramStart == 0x100 assert symbols.spim_context.globalSegment.vramEnd == 0x2C0