unpack.py: add dynamic dependecies to the unpack fetcher

Enable building the build.ninja even when the moulin YAML configuration
specifies an archive that is not present in the file system.

Detection of the missing archive error will now occur during the ninja
invocation, specifically when building a ninja target that unpacks the
archive

Signed-off-by: Mykhailo Androsiuk <[email protected]>

moulin/fetchers/unpack.py

@@ -3,7 +3,6 @@
 """Unpacker fetcher module"""
 
 import os.path
-import subprocess
 from typing import List
 from moulin.yaml_helpers import YAMLProcessingError
 from moulin.yaml_wrapper import YamlValue
@@ -18,11 +17,23 @@ def get_fetcher(conf: YamlValue, build_dir: str, generator: ninja_syntax.Writer)
 def gen_build_rules(generator: ninja_syntax.Writer):
     """Generate build rules using Ninja generator"""
     generator.rule("tar_unpack",
-                   command="mkdir -p $out_dir && tar -m -C $out_dir -xf $in",
+                   command="mkdir -p $out_dir && tar -m -C $out_dir -xf $in && touch $out",
                    description="Unpack $in with tar")
+    generator.rule("tar_scan",
+                   command='bash -c "out_dir_escaped=$$(echo $out_dir | sed \'s/\\//\\\\\\//g\'); '
+                           'echo -e \\\"ninja_dyndep_version = 1\\nbuild $stamp | $$(tar --list -f $in | '
+                           'tr \'\\n\' \' \' | sed \'s/ \\([^ ]\\)/ $$out_dir_escaped\\/\\1/g; '
+                           's/^/$$out_dir_escaped\\//\'): dyndep\\n  restat = 1\\\" > $out\" ',
+                   description="Scan $in tar archive and create .dd file")
     generator.rule("zip_unpack",
-                   command="mkdir -p $out_dir && unzip -DD -n $in -d $out_dir",
+                   command="mkdir -p $out_dir && unzip -DD -n $in -d $out_dir && touch $out",
                    description="Unpack $in with unzip")
+    generator.rule("zip_scan",
+                   command='bash -c "out_dir_escaped=$$(echo $out_dir | sed \'s/\\//\\\\\\//g\'); '
+                           'echo -e \\\"ninja_dyndep_version = 1\\nbuild $stamp | $$(unzip -Z -2 $in | '
+                           'tr \'\\n\' \' \' | sed \'s/ \\([^ ]\\)/ $$out_dir_escaped\\/\\1/g; '
+                           's/^/$$out_dir_escaped\\//\'): dyndep\\n  restat = 1\\\" > $out\" ',
+                   description="Scan $in zip archive and create .dd file")
     generator.newline()
 
 
@@ -40,33 +51,20 @@ def __init__(self, conf: YamlValue, build_dir: str, generator: ninja_syntax.Writ
         dirname = conf.get("dir", default=".").as_str
         self.out_dir = os.path.join(build_dir, dirname)
         self.type = conf["archive_type"].as_str
-
         known_types = ["tar", "zip"]
         if self.type not in known_types:
             raise YAMLProcessingError(f"Unkown archive type: {self.type}",
                                       self.conf["archive_type"].mark)
 
-        if not os.path.exists(self.fname):
-            raise YAMLProcessingError(f"File \"{self.fname}\" does not exist",
-                                      self.conf["file"].mark)
-
     def gen_fetch(self) -> List[str]:
         """Generate instructions to unpack archive"""
-        rule_name = f"{self.type}_unpack"
-        files = self.get_file_list()
-        self.generator.build(files, rule_name, self.fname, variables={"out_dir": self.out_dir})
-
-        return files
+        unpack_rule_name = f"{self.type}_unpack"
+        scan_rule_name = f"{self.type}_scan"
+        stamp = ".stamps/" + self.fname + ".stamp"
+        dyndep = ".stamps/" + self.fname + ".dd"
 
-    def get_file_list(self) -> List[str]:
-        "Get list of files in archive"
+        self.generator.build(stamp, unpack_rule_name, self.fname, order_only=dyndep,
+                             variables={"dyndep": dyndep, "out_dir": self.out_dir})
+        self.generator.build(dyndep, scan_rule_name, self.fname, variables={"stamp": stamp, "out_dir": self.out_dir})
 
-        commands = {
-            "tar": ["tar", "--list", "-f", f"{self.fname}"],
-            "zip": ["unzip", "-Z", "-2", f"{self.fname}"]
-        }
-        ret = subprocess.run(commands[self.type],
-                             check=True,
-                             stdout=subprocess.PIPE,
-                             universal_newlines=True)
-        return [os.path.join(self.out_dir, x) for x in ret.stdout.split("\n") if len(x) > 0]
+        return stamp