Merge pull request #1205 from kapicorp/test/fork

Add mp-method arg

kapitan/cli.py

@@ -101,6 +101,7 @@ def trigger_compile(args):
 def build_parser():
     parser = argparse.ArgumentParser(prog=PROJECT_NAME, description=DESCRIPTION)
     parser.add_argument("--version", action="version", version=VERSION)
+    parser.add_argument("--mp-method", action="store", default=from_dot_kapitan("global", "mp-method", "spawn"), help="set multiprocessing start method", choices=["spawn", "fork", "forkserver"])
     subparser = parser.add_subparsers(help="commands", dest="subparser_name")
 
     inventory_backend_parser = argparse.ArgumentParser(add_help=False)
@@ -601,16 +602,17 @@ def build_parser():
 
 def main():
     """main function for command line usage"""
+
+    parser = build_parser()
+    args = parser.parse_args()
+
     try:
-        multiprocessing.set_start_method("spawn")
+        multiprocessing.set_start_method(args.mp_method)
     # main() is explicitly multiple times in tests
     # and will raise RuntimeError
     except RuntimeError:
         pass
-
-    parser = build_parser()
-    args = parser.parse_args()
-
+
     if getattr(args, "func", None) == generate_inventory and args.pattern and args.target_name == "":
         parser.error("--pattern requires --target_name")
 

kapitan/resources.py

@@ -22,6 +22,7 @@
 
 import kapitan.cached as cached
 from kapitan import __file__ as kapitan_install_path
+from kapitan.inputs.kadet import Dict
 from kapitan.errors import CompileError, InventoryError, KapitanError
 from kapitan.inventory import Inventory, ReclassInventory, AVAILABLE_BACKENDS
 from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string
@@ -334,6 +335,13 @@ def get_inventory(inventory_path, ignore_class_not_found: bool = False) -> Inven
 
     cached.inv = inventory_backend
     cached.global_inv = cached.inv.inventory
+
+    # if we use forked processes, we need to load the inventory for kadet once 
+    # and pass it to the children, to avoid re-reading the inventory for each child
+    # TODO(adenaria): Improve to only do it for kadet
+    if cached.args.mp_method != "spawn":
+        cached.inventory_global_kadet = Dict(cached.global_inv)
+
     # migrate inventory to selected inventory backend
     if hasattr(cached.args, "migrate") and cached.args.migrate:
         inventory_backend.migrate()