Merge pull request #30 from Decompollaborate/develop

2.6.0

CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.6.0] - 2024-08-26
+
+### Added
+
+- Add new parameters to `bss_check.printSymbolComparison`.
+  - `printGoods`: Allows toggling printing the GOOD symbols.
+  - `printingStyle`: The style to use to print the symbols, either `"csv"` or
+    `"listing"`.
+  - TODO: port to Rust.
+- New `MapFile.fixupNonMatchingSymbols` method.
+  - Allows to fixup size calculation of symbols messed up by symbols with the
+    same name suffixed with `.NON_MATCHING`.
+- Add support for `.NON_MATCHING` suffix on symbols on progress calculation.
+  - If a symbol exists and has a `.NON_MATCHING`-suffixed counterpart then
+    consider it not mateched yet.
+
+### Changed
+
+- Minor cleanups.
+
 ## [2.5.1] - 2024-08-09
 
 ### Fixed
@@ -372,6 +392,7 @@ Full changes: <https://github.com/Decompollaborate/mapfile_parser/compare/702a73
 - Initial release
 
 [unreleased]: https://github.com/Decompollaborate/mapfile_parser/compare/master...develop
+[2.6.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.1...2.6.0
 [2.5.1]: https://github.com/Decompollaborate/mapfile_parser/compare/2.5.0...2.5.1
 [2.5.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.4.0...2.5.0
 [2.4.0]: https://github.com/Decompollaborate/mapfile_parser/compare/2.3.7...2.4.0

Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "mapfile_parser"
-version = "2.5.1"
+version = "2.6.0"
 edition = "2021"
 rust-version = "1.65.0"
 authors = ["Anghelo Carvajal <[email protected]>"]

README.md

@@ -35,7 +35,7 @@ If you use a `requirements.txt` file in your repository, then you can add
 this library with the following line:
 
 ```txt
-mapfile_parser>=2.5.1,<3.0.0
+mapfile_parser>=2.6.0,<3.0.0
 ```
 
 #### Development version
@@ -74,7 +74,7 @@ cargo add mapfile_parser
 Or add the following line manually to your `Cargo.toml` file:
 
 ```toml
-mapfile_parser = "2.5.1"
+mapfile_parser = "2.6.0"
 ```
 
 ## Versioning and changelog

pyproject.toml

@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.5.1"
+version = "2.6.0"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.8"

src/mapfile_parser/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 5, 1)
+__version_info__ = (2, 6, 0)
 __version__ = ".".join(map(str, __version_info__))# + "-dev0"
 __author__ = "Decompollaborate"
 

src/mapfile_parser/frontends/bss_check.py

@@ -24,7 +24,7 @@ def getComparison(mapPath: Path, expectedMapPath: Path, *, reverseCheck: bool=Tr
 
     return buildMap.compareFilesAndSymbols(expectedMap, checkOtherOnSelf=reverseCheck)
 
-def printSymbolComparison(comparisonInfo: mapfile.MapsComparisonInfo, printAll: bool=False):
+def printSymbolComparisonAsCsv(comparisonInfo: mapfile.MapsComparisonInfo, printAll: bool=False, printGoods: bool=True):
     print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
 
     # If it's bad or missing, don't need to do anything special.
@@ -48,10 +48,64 @@ def printSymbolComparison(comparisonInfo: mapfile.MapsComparisonInfo, printAll:
                     if not printAll:
                         continue
 
+        if not printGoods and symbolState == "GOOD":
+            continue
+
         if buildFile != expectedFile:
             symbolState += " MOVED"
         print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFilePath},{symbolInfo.expectedAddress:X},{expectedFilePath},{symbolInfo.diff:X},{symbolState}")
 
+def printSymbolComparisonAsListing(comparisonInfo: mapfile.MapsComparisonInfo, printAll: bool=False, printGoods: bool=True):
+    # print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
+
+    # If it's bad or missing, don't need to do anything special.
+    # If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
+
+    for symbolInfo in comparisonInfo.comparedList:
+        buildFile = symbolInfo.buildFile
+        expectedFile = symbolInfo.expectedFile
+        buildFilePath = buildFile.filepath if buildFile is not None else None
+        expectedFilePath = expectedFile.filepath if expectedFile is not None else None
+
+        if symbolInfo.diff is None:
+            print(f"Symbol: {symbolInfo.symbol.name} (MISSING)")
+            if symbolInfo.buildAddress != -1:
+                print(f"    Build:      0x{symbolInfo.buildAddress:08X} (file: {buildFilePath})")
+            if symbolInfo.expectedAddress != -1:
+                print(f"    Expected:   0x{symbolInfo.expectedAddress:08X} (file: {expectedFilePath})")
+            continue
+
+        symbolState = "BAD"
+        if symbolInfo.diff == 0:
+            symbolState = "GOOD"
+            if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
+                if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
+                    if not printAll:
+                        continue
+
+        if not printGoods and symbolState == "GOOD":
+            continue
+
+        if buildFile != expectedFile:
+            symbolState += " MOVED"
+
+        if symbolInfo.diff < 0:
+            diffStr = f"-0x{-symbolInfo.diff:02X}"
+        else:
+            diffStr = f"0x{symbolInfo.diff:02X}"
+
+        print(f"Symbol: {symbolInfo.symbol.name} ({symbolState}) (diff: {diffStr})")
+        print(f"    Build:      0x{symbolInfo.buildAddress:08X} (file: {buildFilePath})")
+        print(f"    Expected:   0x{symbolInfo.expectedAddress:08X} (file: {expectedFilePath})")
+
+def printSymbolComparison(comparisonInfo: mapfile.MapsComparisonInfo, printAll: bool=False, printGoods: bool=True, printingStyle: str="csv"):
+    if printingStyle == "csv":
+        printSymbolComparisonAsCsv(comparisonInfo, printAll, printGoods)
+    elif printingStyle == "listing":
+        printSymbolComparisonAsListing(comparisonInfo, printAll, printGoods)
+    else:
+        printSymbolComparisonAsListing(comparisonInfo, printAll, printGoods)
+
 def printFileComparison(comparisonInfo: mapfile.MapsComparisonInfo):
     utils.eprint("")
 

src/mapfile_parser/frontends/jsonify.py

@@ -12,13 +12,15 @@
 from .. import mapfile
 
 
-def doJsonify(mapPath: Path, outputPath: Path|None, humanReadable: bool=True) -> int:
+def doJsonify(mapPath: Path, outputPath: Path|None, humanReadable: bool=True, applyFixes: bool=False) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
     mapFile = mapfile.MapFile()
     mapFile.readMapFile(mapPath)
+    if applyFixes:
+        mapFile = mapFile.fixupNonMatchingSymbols()
 
     jsonStr = json.dumps(mapFile.toJson(humanReadable=humanReadable), indent=4)
 
@@ -35,14 +37,16 @@ def processArguments(args: argparse.Namespace):
     mapPath: Path = args.mapfile
     outputPath: Path|None = Path(args.output) if args.output is not None else None
     machine: bool = args.machine
+    applyFixes: bool = args.applyFixes
 
-    exit(doJsonify(mapPath, outputPath, humanReadable=not machine))
+    exit(doJsonify(mapPath, outputPath, humanReadable=not machine, applyFixes=applyFixes))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("jsonify", help="Converts a mapfile into a json format.")
 
     parser.add_argument("mapfile", help="Path to a map file", type=Path)
     parser.add_argument("-o", "--output", help="Output path of for the generated json. If omitted then stdout is used instead.")
     parser.add_argument("-m", "--machine", help="Emit numbers as numbers instead of outputting them as pretty strings.", action="store_true")
+    parser.add_argument("-f", "--apply-fixes", help="Apply certain fixups, like fixing size calculation of because of the existence of fake `.NON_MATCHING` symbols.", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/frontends/progress.py

@@ -12,20 +12,20 @@
 from .. import progress_stats
 
 
-def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]:
+def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]:
     mapFile = mapfile.MapFile()
 
     mapFile.debugging = debugging
     mapFile.readMapFile(mapPath)
 
-    return mapFile.filterBySectionType(".text").getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex)
+    return mapFile.filterBySectionType(".text").fixupNonMatchingSymbols().getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles)
 
-def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> int:
+def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
-    totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging)
+    totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging)
 
     progress_stats.printStats(totalStats, progressPerFolder)
     return 0
@@ -36,9 +36,10 @@ def processArguments(args: argparse.Namespace):
     asmPath: Path = args.asmpath
     nonmatchingsPath: Path = args.nonmatchingspath
     pathIndex: int = args.path_index
+    checkFunctionFiles: bool = not args.avoid_function_files
     debugging: bool = args.debugging #! @deprecated
 
-    exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging))
+    exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("progress", help="Computes current progress of the matched functions. Relies on a splat (https://github.com/ethteck/splat) folder structure and matched functions not longer having a file.")
@@ -47,6 +48,7 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser])
     parser.add_argument("asmpath", help="Path to asm folder", type=Path)
     parser.add_argument("nonmatchingspath", help="Path to nonmatchings folder", type=Path)
     parser.add_argument("-i", "--path-index", help="Specify the index to start reading the file paths. Defaults to 2", type=int, default=2)
+    parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true")
     parser.add_argument("-d", "--debugging", help="Enable debugging prints. This option is deprecated", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/frontends/upload_frogress.py

@@ -52,12 +52,12 @@ def uploadEntriesToFrogress(entries: dict[str, int], category: str, url: str, ap
     return 0
 
 
-def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False) -> int:
+def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False, checkFunctionFiles: bool=True) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
-    totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath)
+    totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath, checkFunctionFiles=checkFunctionFiles)
 
     entries: dict[str, int] = getFrogressEntriesFromStats(totalStats, progressPerFolder, verbose)
 
@@ -75,8 +75,9 @@ def processArguments(args: argparse.Namespace):
     baseurl: str = args.baseurl
     apikey: str|None = args.apikey
     verbose: bool = args.verbose
+    checkFunctionFiles: bool = not args.avoid_function_files
 
-    exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose))
+    exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose, checkFunctionFiles))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("upload_frogress", help="Uploads current progress of the matched functions to frogress (https://github.com/decompals/frogress).")
@@ -90,5 +91,6 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser])
     parser.add_argument("--baseurl", help="API base URL", default="https://progress.deco.mp")
     parser.add_argument("--apikey", help="API key. Dry run is performed if this option is omitted")
     parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/mapfile.py

@@ -146,6 +146,10 @@ def toJson(self, humanReadable: bool=True) -> dict[str, Any]:
         return result
 
 
+    def clone(self) -> Symbol:
+        return Symbol(self.name, self.vram, self.size, self.vrom, self.align)
+
+
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Symbol):
             return False
@@ -302,6 +306,13 @@ def appendSymbol(self, sym: Symbol) -> None:
         self._symbols.append(sym)
 
 
+    def clone(self) -> File:
+        f = File(self.filepath, self.vram, self.size, self.sectionType, self.vrom, self.align)
+        for sym in self._symbols:
+            f._symbols.append(sym.clone())
+        return f
+
+
     def __iter__(self) -> Generator[Symbol, None, None]:
         for sym in self._symbols:
             yield sym
@@ -476,6 +487,13 @@ def appendFile(self, file: File) -> None:
         self._filesList.append(file)
 
 
+    def clone(self) -> Segment:
+        s = Segment(self.name, self.vram, self.size, self.vrom, self.align)
+        for f in self._filesList:
+            s._filesList.append(f.clone())
+        return s
+
+
     def __iter__(self) -> Generator[File, None, None]:
         for file in self._filesList:
             yield file
@@ -672,7 +690,21 @@ def mixFolders(self) -> MapFile:
 
         return newMapFile
 
-    def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[ProgressStats, dict[str, ProgressStats]]:
+    def fixupNonMatchingSymbols(self) -> MapFile:
+        newMapFile = self.clone()
+
+        for segment in newMapFile._segmentsList:
+            for file in segment._filesList:
+                for sym in file._symbols:
+                    if sym.name.endswith(".NON_MATCHING") and sym.size != 0:
+                        realSym = file.findSymbolByName(sym.name.replace(".NON_MATCHING", ""))
+                        if realSym is not None and realSym.size == 0:
+                            realSym.size = sym.size
+                            sym.size = 0
+
+        return newMapFile
+
+    def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2, checkFunctionFiles: bool=True) -> tuple[ProgressStats, dict[str, ProgressStats]]:
         totalStats = ProgressStats()
         progressPerFolder: dict[str, ProgressStats] = dict()
 
@@ -710,6 +742,9 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]
                     utils.eprint(f"  whole file is undecomped: {wholeFileIsUndecomped}")
 
                 for func in file:
+                    if func.name.endswith(".NON_MATCHING"):
+                        continue
+
                     funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s"
 
                     symSize = 0
@@ -724,7 +759,12 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]
                         progressPerFolder[folder].undecompedSize += symSize
                         if self.debugging:
                             utils.eprint(f" the whole file is undecomped (no individual function files exist yet)")
-                    elif funcAsmPath.exists():
+                    elif self.findSymbolByName(f"{func.name}.NON_MATCHING") is not None:
+                        totalStats.undecompedSize += symSize
+                        progressPerFolder[folder].undecompedSize += symSize
+                        if self.debugging:
+                            utils.eprint(f" the function hasn't been matched yet (there's a `.NON_MATCHING` symbol with the same name)")
+                    elif checkFunctionFiles and funcAsmPath.exists():
                         totalStats.undecompedSize += symSize
                         progressPerFolder[folder].undecompedSize += symSize
                         if self.debugging:
@@ -810,6 +850,14 @@ def appendSegment(self, segment: Segment) -> None:
         self._segmentsList.append(segment)
 
 
+    def clone(self) -> MapFile:
+        m = MapFile()
+        m.debugging = self.debugging
+        for s in self._segmentsList:
+            m._segmentsList.append(s.clone())
+        return m
+
+
     def __iter__(self) -> Generator[Segment, None, None]:
         for file in self._segmentsList:
             yield file