Add jinja2 support via a build-config.yml and/or command-line --build…

…-config=VAR(.*)=VAL

conda_concourse_ci/cli.py

@@ -22,7 +22,8 @@ def parse_args(parse_this=None):
     submit_parser.add_argument('--pipeline-file', default='plan_director.yml',
                                help="path to pipeline .yml file containing plan")
     submit_parser.add_argument('--config-root-dir',
-                               help="path containing config.yml and matrix definitions")
+                               help="path containing build-config.yml (optional), config.yml and matrix definitions")
+
     submit_parser.add_argument('--src-dir', help="folder where git repo of source code lives",
                                default=os.getcwd())
     submit_parser.add_argument('--private', action='store_false',
@@ -38,6 +39,8 @@ def parse_args(parse_this=None):
                                    help="submit local recipes and plan to configured server")
     one_off_parser.add_argument('pipeline_label',
                                 help="name of your project, to distinguish it from other projects")
+    one_off_parser.add_argument('--build-config', nargs="+",
+                                help=("Specify VAR=VAL to override values defined in build-config.yml"))
     one_off_parser.add_argument('folders', nargs="+",
                                 help=("Specify folders, relative to --recipe-root-dir, to upload "
                                       "and build"))

conda_concourse_ci/execute.py

@@ -41,9 +41,9 @@
 yaml.add_representer(tuple, yaml.representer.SafeRepresenter.represent_list)
 
 
-def parse_platforms(matrix_base_dir, platform_filters):
+def parse_platforms(matrix_base_dir, platform_filters, build_config_vars):
     platform_dir = os.path.join(matrix_base_dir, 'build_platforms.d')
-    platforms = load_yaml_config_dir(platform_dir, platform_filters)
+    platforms = load_yaml_config_dir(platform_dir, platform_filters, build_config_vars)
     log.debug("Platforms found:")
     log.debug(platforms)
     return platforms
@@ -80,6 +80,7 @@ def collect_tasks(
         append_sections_file=None,
         pass_throughs=None,
         skip_existing=True,
+        build_config_vars={}
         ):
     """ Return a graph of build tasks """
     task_graph = nx.DiGraph()
@@ -91,7 +92,7 @@ def collect_tasks(
         **parsed_cli_args,
     )
     platform_filters = ensure_list(platform_filters) if platform_filters else ['*']
-    platforms = parse_platforms(matrix_base_dir, platform_filters)
+    platforms = parse_platforms(matrix_base_dir, platform_filters, build_config_vars)
     # loop over platforms here because each platform may have different dependencies
     # each platform will be submitted with a different label
     for platform in platforms:
@@ -519,6 +520,7 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None,
                    worker_tags=None, clobber_sections_file=None, append_sections_file=None,
                    pass_throughs=None, skip_existing=True,
                    use_repo_access=False, use_staging_channel=False, **kw):
+    build_config = kw.get('build_config', []) or []
     if kw.get('stage_for_upload', False):
         if kw.get('commit_msg') is None:
             raise ValueError(
@@ -542,6 +544,33 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None,
 
     repo_commit = ''
     git_identifier = ''
+
+    build_config_yml = os.path.join(matrix_base_dir, 'build-config.yml')
+    matched = {}
+    build_config_vars = {}
+    try:
+        with open(build_config_yml) as build_config_file:
+            build_config_vars = yaml.safe_load(build_config_file)
+    except (OSError, IOError) as e:
+        print('WARNING :: open(build_config_yml={}) failed'.format(build_config_yml))
+        pass
+    for bcv in build_config_vars:
+        for var_val in build_config:
+            var = var_val.split('=', 1)[0]
+            val = var_val.split('=', 1)[1]
+            if fnmatch(bcv, var):
+                matched[var_val] = 1
+                log.info("Overriding build-config.yaml with {}={}".format(var, val))
+                build_config_vars[bcv] = val
+    for var_val in build_config:
+        if var_val not in matched:
+            var = var_val.split('=', 1)[0]
+            if '*' in var:
+                log.warning("Did not find match for --build-config={} (it has no effect)".format(var_val))
+            else:
+                val = var_val.split('=', 1)[1]
+                log.info("Adding {}={} to build configuration".format(var, val))
+                build_config_vars[var] = val
     task_graph = collect_tasks(
         path,
         folders=folders,
@@ -555,7 +584,8 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None,
         append_sections_file=append_sections_file,
         clobber_sections_file=clobber_sections_file,
         pass_throughs=pass_throughs,
-        skip_existing=skip_existing
+        skip_existing=skip_existing,
+        build_config_vars=build_config_vars
     )
 
     with open(os.path.join(matrix_base_dir, 'config.yml')) as src:

conda_concourse_ci/utils.py

@@ -6,6 +6,7 @@
 import yaml
 
 from conda_build.utils import HashableDict  # NOQA
+from jinja2 import Environment, FileSystemLoader
 
 
 def ensure_list(arg):
@@ -17,10 +18,23 @@ def ensure_list(arg):
     return arg
 
 
-def load_yaml_config_dir(platforms_dir, platform_filters):
+def load_yaml_config_dir(platforms_dir, platform_filters, build_config_vars):
     platforms = []
+    # for f in os.listdir(platforms_dir):
+    #     if f.endswith('.yml') and any(fnmatch.fnmatch(f[:-len('.yml')], pat) for pat in platform_filters):
+    #         with open(os.path.join(platforms_dir, f)) as buff:
+    #             env = Environment(loader=DictLoader({'platform.yml': buff.read()}), trim_blocks=True,
+    #                               lstrip_blocks=True)
+    #             rendered = env.get_template('platform.yml').render(**build_config_vars)
+    #             platforms.append(yaml.load(rendered, Loader=yaml.BaseLoader))
+    print("Using the following c3i build config vars:\n{}".format(build_config_vars))
     for f in os.listdir(platforms_dir):
-        if f.endswith('.yml') and any(fnmatch.fnmatch(f, pat) for pat in platform_filters):
-            with open(os.path.join(platforms_dir, f)) as buff:
-                platforms.append(yaml.load(buff, Loader=yaml.BaseLoader))
+        if f.endswith('.yml') and os.sep not in f and any(
+                fnmatch.fnmatch(f[:-len('.yml')], pat) for pat in platform_filters):
+            path = os.path.join(platforms_dir, f)
+            env = Environment(loader=FileSystemLoader(os.path.dirname(path)), trim_blocks=True,
+                              lstrip_blocks=True)
+            rendered = env.get_template(os.path.basename(path)).render(**build_config_vars)
+            platforms.append(yaml.load(rendered, Loader=yaml.BaseLoader))
+
     return platforms