From cab499cc0c36a7c5195a834e1c0242b2d3d0b994 Mon Sep 17 00:00:00 2001
From: Mykhailo Androsiuk <mykhailo_androsiuk@epam.com>
Date: Wed, 17 Jan 2024 23:50:11 +0200
Subject: [PATCH] unpack.py: add dynamic dependecies to the unpack fetcher

Enable building the build.ninja even when the moulin YAML configuration
specifies an archive that is not present in the file system.

Detection of the missing archive error will now occur during the ninja
invocation, specifically when building a ninja target that unpacks the
archive

Signed-off-by: Mykhailo Androsiuk <mykhailo_androsiuk@epam.com>
---
 moulin/fetchers/unpack.py | 46 +++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/moulin/fetchers/unpack.py b/moulin/fetchers/unpack.py
index 2393fa6..4a7db05 100644
--- a/moulin/fetchers/unpack.py
+++ b/moulin/fetchers/unpack.py
@@ -3,7 +3,6 @@
 """Unpacker fetcher module"""
 
 import os.path
-import subprocess
 from typing import List
 from moulin.yaml_helpers import YAMLProcessingError
 from moulin.yaml_wrapper import YamlValue
@@ -18,11 +17,23 @@ def get_fetcher(conf: YamlValue, build_dir: str, generator: ninja_syntax.Writer)
 def gen_build_rules(generator: ninja_syntax.Writer):
     """Generate build rules using Ninja generator"""
     generator.rule("tar_unpack",
-                   command="mkdir -p $out_dir && tar -m -C $out_dir -xf $in",
+                   command="mkdir -p $out_dir && tar -m -C $out_dir -xf $in && touch $out",
                    description="Unpack $in with tar")
+    generator.rule("tar_scan",
+                   command='bash -c "out_dir_escaped=$$(echo $out_dir | sed \'s/\\//\\\\\\//g\'); '
+                           'echo -e \\\"ninja_dyndep_version = 1\\nbuild $stamp | $$(tar --list -f $in | '
+                           'tr \'\\n\' \' \' | sed \'s/ \\([^ ]\\)/ $$out_dir_escaped\\/\\1/g; '
+                           's/^/$$out_dir_escaped\\//\'): dyndep\\n  restat = 1\\\" > $out\" ',
+                   description="Scan $in tar archive and create .dd file")
     generator.rule("zip_unpack",
-                   command="mkdir -p $out_dir && unzip -DD -n $in -d $out_dir",
+                   command="mkdir -p $out_dir && unzip -DD -n $in -d $out_dir && touch $out",
                    description="Unpack $in with unzip")
+    generator.rule("zip_scan",
+                   command='bash -c "out_dir_escaped=$$(echo $out_dir | sed \'s/\\//\\\\\\//g\'); '
+                           'echo -e \\\"ninja_dyndep_version = 1\\nbuild $stamp | $$(unzip -Z -2 $in | '
+                           'tr \'\\n\' \' \' | sed \'s/ \\([^ ]\\)/ $$out_dir_escaped\\/\\1/g; '
+                           's/^/$$out_dir_escaped\\//\'): dyndep\\n  restat = 1\\\" > $out\" ',
+                   description="Scan $in zip archive and create .dd file")
     generator.newline()
 
 
@@ -40,33 +51,20 @@ def __init__(self, conf: YamlValue, build_dir: str, generator: ninja_syntax.Writ
         dirname = conf.get("dir", default=".").as_str
         self.out_dir = os.path.join(build_dir, dirname)
         self.type = conf["archive_type"].as_str
-
         known_types = ["tar", "zip"]
         if self.type not in known_types:
             raise YAMLProcessingError(f"Unkown archive type: {self.type}",
                                       self.conf["archive_type"].mark)
 
-        if not os.path.exists(self.fname):
-            raise YAMLProcessingError(f"File \"{self.fname}\" does not exist",
-                                      self.conf["file"].mark)
-
     def gen_fetch(self) -> List[str]:
         """Generate instructions to unpack archive"""
-        rule_name = f"{self.type}_unpack"
-        files = self.get_file_list()
-        self.generator.build(files, rule_name, self.fname, variables={"out_dir": self.out_dir})
-
-        return files
+        unpack_rule_name = f"{self.type}_unpack"
+        scan_rule_name = f"{self.type}_scan"
+        stamp = ".stamps/" + self.fname + ".stamp"
+        dyndep = ".stamps/" + self.fname + ".dd"
 
-    def get_file_list(self) -> List[str]:
-        "Get list of files in archive"
+        self.generator.build(stamp, unpack_rule_name, self.fname, order_only=dyndep,
+                             variables={"dyndep": dyndep, "out_dir": self.out_dir})
+        self.generator.build(dyndep, scan_rule_name, self.fname, variables={"stamp": stamp, "out_dir": self.out_dir})
 
-        commands = {
-            "tar": ["tar", "--list", "-f", f"{self.fname}"],
-            "zip": ["unzip", "-Z", "-2", f"{self.fname}"]
-        }
-        ret = subprocess.run(commands[self.type],
-                             check=True,
-                             stdout=subprocess.PIPE,
-                             universal_newlines=True)
-        return [os.path.join(self.out_dir, x) for x in ret.stdout.split("\n") if len(x) > 0]
+        return stamp