From ca8b872fe95b6e3933a5617659c19dce643df068 Mon Sep 17 00:00:00 2001 From: angie Date: Mon, 26 Aug 2024 14:56:46 -0400 Subject: [PATCH] Add support for `.NON_MATCHING` suffix on symbols on progress calculation --- CHANGELOG.md | 5 +++- src/mapfile_parser/frontends/progress.py | 12 ++++++---- .../frontends/upload_frogress.py | 8 ++++--- src/mapfile_parser/mapfile.py | 12 ++++++++-- src/rs/mapfile.rs | 23 ++++++++++++++++--- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99cf68b..7334ccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,10 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - New `MapFile.fixupNonMatchingSymbols` method. - Allows to fixup size calculation of symbols messed up by symbols with the same name suffixed with `.NON_MATCHING`. +- Add support for `.NON_MATCHING` suffix on symbols on progress calculation. + - If a symbol exists and has a `.NON_MATCHING`-suffixed counterpart then + consider it not mateched yet. ### Changed -- Minor cleanups +- Minor cleanups. ## [2.5.1] - 2024-08-09 diff --git a/src/mapfile_parser/frontends/progress.py b/src/mapfile_parser/frontends/progress.py index 6c538bc..17b286e 100644 --- a/src/mapfile_parser/frontends/progress.py +++ b/src/mapfile_parser/frontends/progress.py @@ -12,20 +12,20 @@ from .. import progress_stats -def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]: +def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]: mapFile = mapfile.MapFile() mapFile.debugging = debugging mapFile.readMapFile(mapPath) - return mapFile.filterBySectionType(".text").fixupNonMatchingSymbols().getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex) + return mapFile.filterBySectionType(".text").fixupNonMatchingSymbols().getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles) -def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> int: +def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> int: if not mapPath.exists(): print(f"Could not find mapfile at '{mapPath}'") return 1 - totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging) + totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging) progress_stats.printStats(totalStats, progressPerFolder) return 0 @@ -36,9 +36,10 @@ def processArguments(args: argparse.Namespace): asmPath: Path = args.asmpath nonmatchingsPath: Path = args.nonmatchingspath pathIndex: int = args.path_index + checkFunctionFiles: bool = not args.avoid_function_files debugging: bool = args.debugging #! @deprecated - exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging)) + exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging)) def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]): parser = subparser.add_parser("progress", help="Computes current progress of the matched functions. Relies on a splat (https://github.com/ethteck/splat) folder structure and matched functions not longer having a file.") @@ -47,6 +48,7 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]) parser.add_argument("asmpath", help="Path to asm folder", type=Path) parser.add_argument("nonmatchingspath", help="Path to nonmatchings folder", type=Path) parser.add_argument("-i", "--path-index", help="Specify the index to start reading the file paths. Defaults to 2", type=int, default=2) + parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true") parser.add_argument("-d", "--debugging", help="Enable debugging prints. This option is deprecated", action="store_true") parser.set_defaults(func=processArguments) diff --git a/src/mapfile_parser/frontends/upload_frogress.py b/src/mapfile_parser/frontends/upload_frogress.py index 0b84b58..9ef13df 100644 --- a/src/mapfile_parser/frontends/upload_frogress.py +++ b/src/mapfile_parser/frontends/upload_frogress.py @@ -52,12 +52,12 @@ def uploadEntriesToFrogress(entries: dict[str, int], category: str, url: str, ap return 0 -def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False) -> int: +def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False, checkFunctionFiles: bool=True) -> int: if not mapPath.exists(): print(f"Could not find mapfile at '{mapPath}'") return 1 - totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath) + totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath, checkFunctionFiles=checkFunctionFiles) entries: dict[str, int] = getFrogressEntriesFromStats(totalStats, progressPerFolder, verbose) @@ -75,8 +75,9 @@ def processArguments(args: argparse.Namespace): baseurl: str = args.baseurl apikey: str|None = args.apikey verbose: bool = args.verbose + checkFunctionFiles: bool = not args.avoid_function_files - exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose)) + exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose, checkFunctionFiles)) def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]): parser = subparser.add_parser("upload_frogress", help="Uploads current progress of the matched functions to frogress (https://github.com/decompals/frogress).") @@ -90,5 +91,6 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]) parser.add_argument("--baseurl", help="API base URL", default="https://progress.deco.mp") parser.add_argument("--apikey", help="API key. Dry run is performed if this option is omitted") parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true") parser.set_defaults(func=processArguments) diff --git a/src/mapfile_parser/mapfile.py b/src/mapfile_parser/mapfile.py index 1d588b7..34c03d1 100644 --- a/src/mapfile_parser/mapfile.py +++ b/src/mapfile_parser/mapfile.py @@ -704,7 +704,7 @@ def fixupNonMatchingSymbols(self) -> MapFile: return newMapFile - def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[ProgressStats, dict[str, ProgressStats]]: + def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2, checkFunctionFiles: bool=True) -> tuple[ProgressStats, dict[str, ProgressStats]]: totalStats = ProgressStats() progressPerFolder: dict[str, ProgressStats] = dict() @@ -742,6 +742,9 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str] utils.eprint(f" whole file is undecomped: {wholeFileIsUndecomped}") for func in file: + if func.name.endswith(".NON_MATCHING"): + continue + funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s" symSize = 0 @@ -756,7 +759,12 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str] progressPerFolder[folder].undecompedSize += symSize if self.debugging: utils.eprint(f" the whole file is undecomped (no individual function files exist yet)") - elif funcAsmPath.exists(): + elif self.findSymbolByName(f"{func.name}.NON_MATCHING") is not None: + totalStats.undecompedSize += symSize + progressPerFolder[folder].undecompedSize += symSize + if self.debugging: + utils.eprint(f" the function hasn't been matched yet (there's a `.NON_MATCHING` symbol with the same name)") + elif checkFunctionFiles and funcAsmPath.exists(): totalStats.undecompedSize += symSize progressPerFolder[folder].undecompedSize += symSize if self.debugging: diff --git a/src/rs/mapfile.rs b/src/rs/mapfile.rs index e9d3e25..2a90ff0 100644 --- a/src/rs/mapfile.rs +++ b/src/rs/mapfile.rs @@ -616,6 +616,7 @@ impl MapFile { nonmatchings: &Path, aliases: &HashMap, path_index: usize, + check_function_files: bool, ) -> ( progress_stats::ProgressStats, HashMap, @@ -661,13 +662,22 @@ impl MapFile { let whole_file_is_undecomped = full_asm_file.exists(); for func in &file.symbols { + if func.name.ends_with(".NON_MATCHING") { + continue; + } + let func_asm_path = nonmatchings .join(extensionless_file_path.clone()) .join(func.name.clone() + ".s"); let sym_size = func.size.unwrap_or(0) as usize; - if whole_file_is_undecomped || func_asm_path.exists() { + if whole_file_is_undecomped + || self + .find_symbol_by_name(&format!("{}.NON_MATCHING", func.name)) + .is_some() + || (check_function_files && func_asm_path.exists()) + { total_stats.undecomped_size += sym_size; folder_progress.undecomped_size += sym_size; } else { @@ -882,18 +892,25 @@ pub(crate) mod python_bindings { self.fixup_non_matching_symbols() } - #[pyo3(signature = (asm_path, nonmatchings, aliases=HashMap::new(), path_index=2))] + #[pyo3(signature = (asm_path, nonmatchings, aliases=HashMap::new(), path_index=2, check_function_files=true))] fn getProgress( &self, asm_path: PathBuf, nonmatchings: PathBuf, aliases: HashMap, path_index: usize, + check_function_files: bool, ) -> ( progress_stats::ProgressStats, HashMap, ) { - self.get_progress(&asm_path, &nonmatchings, &aliases, path_index) + self.get_progress( + &asm_path, + &nonmatchings, + &aliases, + path_index, + check_function_files, + ) } #[pyo3(signature=(other_map_file, *, check_other_on_self=true))]