Add support for .NON_MATCHING suffix on symbols on progress calcula…

…tion

CHANGELOG.md

@@ -17,10 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - New `MapFile.fixupNonMatchingSymbols` method.
   - Allows to fixup size calculation of symbols messed up by symbols with the
     same name suffixed with `.NON_MATCHING`.
+- Add support for `.NON_MATCHING` suffix on symbols on progress calculation.
+  - If a symbol exists and has a `.NON_MATCHING`-suffixed counterpart then
+    consider it not mateched yet.
 
 ### Changed
 
-- Minor cleanups
+- Minor cleanups.
 
 ## [2.5.1] - 2024-08-09
 

src/mapfile_parser/frontends/progress.py

@@ -12,20 +12,20 @@
 from .. import progress_stats
 
 
-def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]:
+def getProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> tuple[progress_stats.ProgressStats, dict[str, progress_stats.ProgressStats]]:
     mapFile = mapfile.MapFile()
 
     mapFile.debugging = debugging
     mapFile.readMapFile(mapPath)
 
-    return mapFile.filterBySectionType(".text").fixupNonMatchingSymbols().getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex)
+    return mapFile.filterBySectionType(".text").fixupNonMatchingSymbols().getProgress(asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles)
 
-def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, debugging: bool=False) -> int:
+def doProgress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, pathIndex: int=2, checkFunctionFiles: bool=True, debugging: bool=False) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
-    totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging)
+    totalStats, progressPerFolder = getProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging)
 
     progress_stats.printStats(totalStats, progressPerFolder)
     return 0
@@ -36,9 +36,10 @@ def processArguments(args: argparse.Namespace):
     asmPath: Path = args.asmpath
     nonmatchingsPath: Path = args.nonmatchingspath
     pathIndex: int = args.path_index
+    checkFunctionFiles: bool = not args.avoid_function_files
     debugging: bool = args.debugging #! @deprecated
 
-    exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, debugging=debugging))
+    exit(doProgress(mapPath, asmPath, nonmatchingsPath, pathIndex=pathIndex, checkFunctionFiles=checkFunctionFiles, debugging=debugging))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("progress", help="Computes current progress of the matched functions. Relies on a splat (https://github.com/ethteck/splat) folder structure and matched functions not longer having a file.")
@@ -47,6 +48,7 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser])
     parser.add_argument("asmpath", help="Path to asm folder", type=Path)
     parser.add_argument("nonmatchingspath", help="Path to nonmatchings folder", type=Path)
     parser.add_argument("-i", "--path-index", help="Specify the index to start reading the file paths. Defaults to 2", type=int, default=2)
+    parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true")
     parser.add_argument("-d", "--debugging", help="Enable debugging prints. This option is deprecated", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/frontends/upload_frogress.py

@@ -52,12 +52,12 @@ def uploadEntriesToFrogress(entries: dict[str, int], category: str, url: str, ap
     return 0
 
 
-def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False) -> int:
+def doUploadFrogress(mapPath: Path, asmPath: Path, nonmatchingsPath: Path, project: str, version: str, category: str, baseurl: str, apikey: str|None=None, verbose: bool=False, checkFunctionFiles: bool=True) -> int:
     if not mapPath.exists():
         print(f"Could not find mapfile at '{mapPath}'")
         return 1
 
-    totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath)
+    totalStats, progressPerFolder = progress.getProgress(mapPath, asmPath, nonmatchingsPath, checkFunctionFiles=checkFunctionFiles)
 
     entries: dict[str, int] = getFrogressEntriesFromStats(totalStats, progressPerFolder, verbose)
 
@@ -75,8 +75,9 @@ def processArguments(args: argparse.Namespace):
     baseurl: str = args.baseurl
     apikey: str|None = args.apikey
     verbose: bool = args.verbose
+    checkFunctionFiles: bool = not args.avoid_function_files
 
-    exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose))
+    exit(doUploadFrogress(mapPath, asmPath, nonmatchingsPath, project, version, category, baseurl, apikey, verbose, checkFunctionFiles))
 
 def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
     parser = subparser.add_parser("upload_frogress", help="Uploads current progress of the matched functions to frogress (https://github.com/decompals/frogress).")
@@ -90,5 +91,6 @@ def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser])
     parser.add_argument("--baseurl", help="API base URL", default="https://progress.deco.mp")
     parser.add_argument("--apikey", help="API key. Dry run is performed if this option is omitted")
     parser.add_argument("-v", "--verbose", action="store_true")
+    parser.add_argument("-f", "--avoid-function-files", help="Avoid checking if the assembly file for a function exists as a way to determine if the function has been matched or not", action="store_true")
 
     parser.set_defaults(func=processArguments)

src/mapfile_parser/mapfile.py

@@ -704,7 +704,7 @@ def fixupNonMatchingSymbols(self) -> MapFile:
 
         return newMapFile
 
-    def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[ProgressStats, dict[str, ProgressStats]]:
+    def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2, checkFunctionFiles: bool=True) -> tuple[ProgressStats, dict[str, ProgressStats]]:
         totalStats = ProgressStats()
         progressPerFolder: dict[str, ProgressStats] = dict()
 
@@ -742,6 +742,9 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]
                     utils.eprint(f"  whole file is undecomped: {wholeFileIsUndecomped}")
 
                 for func in file:
+                    if func.name.endswith(".NON_MATCHING"):
+                        continue
+
                     funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s"
 
                     symSize = 0
@@ -756,7 +759,12 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]
                         progressPerFolder[folder].undecompedSize += symSize
                         if self.debugging:
                             utils.eprint(f" the whole file is undecomped (no individual function files exist yet)")
-                    elif funcAsmPath.exists():
+                    elif self.findSymbolByName(f"{func.name}.NON_MATCHING") is not None:
+                        totalStats.undecompedSize += symSize
+                        progressPerFolder[folder].undecompedSize += symSize
+                        if self.debugging:
+                            utils.eprint(f" the function hasn't been matched yet (there's a `.NON_MATCHING` symbol with the same name)")
+                    elif checkFunctionFiles and funcAsmPath.exists():
                         totalStats.undecompedSize += symSize
                         progressPerFolder[folder].undecompedSize += symSize
                         if self.debugging:

src/rs/mapfile.rs

@@ -616,6 +616,7 @@ impl MapFile {
         nonmatchings: &Path,
         aliases: &HashMap<String, String>,
         path_index: usize,
+        check_function_files: bool,
     ) -> (
         progress_stats::ProgressStats,
         HashMap<String, progress_stats::ProgressStats>,
@@ -661,13 +662,22 @@ impl MapFile {
                 let whole_file_is_undecomped = full_asm_file.exists();
 
                 for func in &file.symbols {
+                    if func.name.ends_with(".NON_MATCHING") {
+                        continue;
+                    }
+
                     let func_asm_path = nonmatchings
                         .join(extensionless_file_path.clone())
                         .join(func.name.clone() + ".s");
 
                     let sym_size = func.size.unwrap_or(0) as usize;
 
-                    if whole_file_is_undecomped || func_asm_path.exists() {
+                    if whole_file_is_undecomped
+                        || self
+                            .find_symbol_by_name(&format!("{}.NON_MATCHING", func.name))
+                            .is_some()
+                        || (check_function_files && func_asm_path.exists())
+                    {
                         total_stats.undecomped_size += sym_size;
                         folder_progress.undecomped_size += sym_size;
                     } else {
@@ -882,18 +892,25 @@ pub(crate) mod python_bindings {
             self.fixup_non_matching_symbols()
         }
 
-        #[pyo3(signature = (asm_path, nonmatchings, aliases=HashMap::new(), path_index=2))]
+        #[pyo3(signature = (asm_path, nonmatchings, aliases=HashMap::new(), path_index=2, check_function_files=true))]
         fn getProgress(
             &self,
             asm_path: PathBuf,
             nonmatchings: PathBuf,
             aliases: HashMap<String, String>,
             path_index: usize,
+            check_function_files: bool,
         ) -> (
             progress_stats::ProgressStats,
             HashMap<String, progress_stats::ProgressStats>,
         ) {
-            self.get_progress(&asm_path, &nonmatchings, &aliases, path_index)
+            self.get_progress(
+                &asm_path,
+                &nonmatchings,
+                &aliases,
+                path_index,
+                check_function_files,
+            )
         }
 
         #[pyo3(signature=(other_map_file, *, check_other_on_self=true))]