mixin-update

#!/usr/bin/env python

import argparse
from six.moves import configparser
import sys
import os
import re
import argparse
import collections
sys.path.insert(1,'external/pystache')
import pystache
import shutil
import tempfile
import stat
import logging
import filecmp
from filecmp import dircmp
pystache.defaults.MISSING_TAGS = 'strict'

class Configurer:
    # These are the set of product configuration files that are modified by mixins.
    # If they are named something else in the actual product directory, the mixin spec
    # file should setup the mapping in the "mapping" section
    FILE_LIST = ["BoardConfig.mk", "init.rc", "init.recovery.rc", "fstab", "fstab.recovery", "product.mk",
              "ueventd.rc", "AndroidBoard.mk", "gpt.ini", "flashfiles.ini"]
    RENDER_BLACK_LIST = [".bin"]
    TEMPLATE_GROUPS = ["template-simple", "template-complex"]
    PRODUCT_SPEC_FN = "mixins.spec"
    SPEC_SUFFIX = ".spec"
    ANDROID_PRODUCTS_MK = "AndroidProducts.mk"
    GROUP_SPEC_FN = "mixinfo.spec"
    OPTION_SPEC_FN = "option.spec"
    FILES_SPEC_FN = "files.spec"
    DOC_SPEC_FN = "doc.spec"
    EXTRA_DIR = "extra_files"
    EXTRA_DIR_KEY = "_extra_dir"
    INTERNAL_TREE = "device/intel/mixins/groups"
    INTERNAL_DEVICE_DIR = "device/intel"
    RELEASE_INFO_FN = "mixin-groups.doc"
    CHANGE_LOG_FN = "ChangeLog"
    EXEMPT_FN = "exempt_groups.txt"
    RULE_FN = "new_group_creation_rule.txt"
    VALID_TARGETS_FN = "valid_targets.spec"
    EMBARGOED_VALID_TARGETS_FN = "../../../vendor/intel/utils_priv/embargoed_valid_targets.spec"
    SCRIPT_FN = "mixin-update"
    VERSION = "3.2.3"


class ReadTrackingDict(dict):
    def __init__(self, *args, **kwargs):
        super(ReadTrackingDict, self).__init__(*args, **kwargs)
        self.reads = collections.defaultdict(int)
    def __getitem__(self, key):
        self.reads[key] += 1
        return super(ReadTrackingDict, self).__getitem__(key)
    def unused_keys(self):
        return list(set(self.keys()) - set(self.reads.keys()))


class Logger():
    def __init__(self, abort_on_errors, tag):
        self.abort_on_errors = abort_on_errors
        self.errors_found = False
        self.tag = tag
        self.logger = logging.getLogger(tag)

    def debug(self, s):
        self.logger.debug(s)

    def info(self, s):
        self.logger.info(s)

    def warning(self, s):
        self.logger.warning(s)

    def error(self, s):
        sys.stderr.write("ERROR: ")
        self.warning(s)
        if self.abort_on_errors:
            sys.exit(1)
        else:
            self.errors_found = True


class FileHelper(Logger):
    """ This class is used to process file/folder related operations."""
    def __init__(self, abort_on_errors, tag):
        Logger.__init__(self, abort_on_errors, tag)
        self.tmp_fn = tempfile.mkstemp(prefix='mixins-', text=True)[1]
        # track all files processed by this class
        self.update_file_set = set()

    def _is_different_dir(self, dcmp):
        if dcmp.diff_files or dcmp.left_only or dcmp.right_only:
            return True

        for sub_dcmp in dcmp.subdirs.values():
            if self._is_different_dir(sub_dcmp):
                return True

        return False

    def __del__(self):
        if os.path.exists(self.tmp_fn):
            os.remove(self.tmp_fn)

    def is_different_dir(self, src_dir, dst_dir):
        dcmp = dircmp(src_dir, dst_dir)
        return self._is_different_dir(dcmp)


    def render_file_content(self, fn, params):
        """ return a rendered file"""
        if not os.path.splitext(fn)[1] in Configurer.RENDER_BLACK_LIST:
            with open(fn,'rb') as sfile:
                # Avoid read large non-text file as it'll cost much time
                sfile.seek(-1, 2)
                c = sfile.read(1)
                if c == b'\n':
                    sfile.seek(0, 0)
                    src_contents = sfile.read()
                    if b"{{" in src_contents:
                        try:
                            slines = pystache.render(src_contents, params)
                        except pystache.context.KeyNotFoundError as e:
                            self.error("{} depends on undefined mixin parameter '{}'".format(fn, e.key))
                        with open(self.tmp_fn, "w") as dfile:
                            dfile.writelines(slines)
                        return self.tmp_fn

        return fn

    def update_file_with_params(self, src_fn, dst_fn, dry_run=False, params=None):

        retval = True

        if not os.path.exists(src_fn):
            return retval

        source_file = src_fn
        # replace mixins parameters
        if params:
            source_file = self.render_file_content(src_fn, params)
        if os.path.exists(dst_fn):
            if not filecmp.cmp(source_file, dst_fn):
                if dry_run:
                    self.warning("{} is out of date".format(dst_fn))
                    retval = False
                else:
                    self.info("updating file: {}".format(dst_fn))
                    shutil.copy(source_file, dst_fn)
        else:
            dirname = os.path.dirname(dst_fn)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            if dry_run:
                self.warning("{} is out of date".format(dst_fn))
                retval = False
            else:
                self.info("updating file: {}".format(dst_fn))
                shutil.copy(source_file, dst_fn)

        if not dry_run:
            src_mode = os.stat(src_fn).st_mode
            dst_mode = os.stat(dst_fn).st_mode
            src_chmod = stat.S_IMODE(src_mode)
            dst_chmod = stat.S_IMODE(dst_mode)
            if src_chmod != dst_chmod:
                self.info("chmod {} {} ".format(oct(src_chmod), dst_fn))
                os.chmod(dst_fn,src_chmod)

        if dst_fn not in self.update_file_set:
            self.update_file_set.add(dst_fn)

        return retval


    def update_file_with_lines(self, dlines, dst_fn, dry_run=False):

        retval = True

        orig_dlines = []
        if os.path.exists(dst_fn):
            with open(dst_fn) as dfile:
                orig_dlines = dfile.readlines()

        if ((orig_dlines > dlines) - (orig_dlines < dlines)) != 0:
            if dry_run:
                self.warning("{} is out of date".format(dst_fn))
                retval = False
            else:
                self.info("updating file {}".format(dst_fn))
                with open(dst_fn, "w") as fp:
                    fp.writelines(dlines)

        if dst_fn not in self.update_file_set:
            self.update_file_set.add(dst_fn)

        return retval


    def update_directory_with_params(self, src_dir, dst_dir, dry_run=False, params=None):

        retval = True
        source_dir = src_dir
        # replace mixins parameters
        if params:
            source_dir = os.path.join(tempfile.mkdtemp(), "mixins-dir-")
            shutil.copytree(src_dir, source_dir, symlinks=True)
            for root, dirs, files in os.walk(source_dir):
                if files:
                    for f in files:
                        fn = os.path.join(root, f)
                        tmp_fn = self.render_file_content(fn, params)
                        if not filecmp.cmp(fn, tmp_fn):
                            shutil.copy(tmp_fn, fn)

        if os.path.exists(dst_dir):
            if self.is_different_dir(source_dir, dst_dir):
                if dry_run:
                    self.warning("{} is out of date".format(dst_dir))
                    retval = False
                else:
                    self.info("updating dir: {}".format(dst_dir))
                    shutil.rmtree(dst_dir)
                    shutil.copytree(source_dir, dst_dir, symlinks=True)
        else:
            if dry_run:
                self.warning("{} is out of date".format(dst_dir))
                retval = False
            else:
                self.info("updating dir: {}".format(dst_dir))
                shutil.copytree(source_dir, dst_dir, symlinks=True)

        if source_dir != src_dir:
            shutil.rmtree(source_dir)

        for root, dirs, files in os.walk(dst_dir):
            if files:
                for f in files:
                    fn = os.path.join(root, f)
                    if fn not in self.update_file_set:
                        self.update_file_set.add(fn)

        return retval


class Checker(Logger):
    checked_list = set()

    def __init__(self, abort_on_errors, tag):
        Logger.__init__(self, abort_on_errors, "Checker-{}".format(tag))


    def _check_doc(self, group_tree):
        doc_fn = group_tree["doc"]["fn"]
        doc_tree = group_tree["doc"]["tree"]
        sections = ["Overview", "Options"]
        for section in sections:
            if section not in doc_tree.keys():
                self.policy_error("{} lacks {} section description or {} has syntax error".format(doc_fn, section, section))

            if section == "Options":
                if not doc_tree["Options"].keys():
                    self.policy_error("Options section missing valid option description: {}".format(doc_fn))
                else:
                    delta = list(set(group_tree["options"].keys()) - set(doc_tree["Options"].keys()))
                    if delta:
                        self.policy_error("{} lacks options:{} description".format(doc_fn, delta))
                    else:
                        for option in group_tree["options"].keys():
                            if len(doc_tree["Options"][option]) < 1:
                                self.policy_error("{} lacks option {} detail description".format(doc_fn, option))

            if section == "Overview" and len(doc_tree["Overview"]) < 1:
                self.policy_error("{} lacks {} description".format(doc_fn, section))


    def policy_error(self, s):
        s = s + "\n" + "please refer to {} file for new group creation.".format(os.path.join(os.path.dirname(sys.argv[0]), Configurer.RULE_FN))
        self.error(s)


    def _check_name(self, name):
        forbid_names_prefix = ["disable", "enable", "none"]
        basename = os.path.basename(name).lower()
        for prefix in forbid_names_prefix:
            if basename.startswith(prefix):
                self.policy_error("basename {}: don't prefix names in {}, you can use true/false instead.".format(name, forbid_names_prefix))
        if len(basename) > 20:
            self.policy_error("basename {}: exceeds 20 characters.".format(name))
        if '_' in basename:
            self.policy_error("basename {}: don't use '_' as separator, use '-' instead.'".format(name))

    def check_rules(self, parser):

        mixins_tree_dir = ""
        if os.path.exists(parser.internal_mixins_tree_dir):
            if parser.internal_mixins_tree_dir in Checker.checked_list:
                self.debug("Skip as {} already checked.".format(parser.internal_mixins_tree_dir))
                return
            mixins_tree_dir = parser.internal_mixins_tree_dir
            Checker.checked_list.add(mixins_tree_dir)
            mixins_tree = parser.internal_mixins_tree
        else:
            if parser.mixins_tree_dir in Checker.checked_list:
                self.debug("Skip as {} already checked".format(parser.mixins_tree_dir))
                return
            mixins_tree_dir = parser.mixins_tree_dir
            Checker.checked_list.add(mixins_tree_dir)
            mixins_tree = parser.mixins_tree

        self.debug("check {}".format(mixins_tree_dir))

        exempt_groups = []
        exempt_fn = os.path.join(os.path.dirname(sys.argv[0]), Configurer.EXEMPT_FN)
        if os.path.exists(exempt_fn):
            with open(exempt_fn) as dfile:
                dlines = dfile.readlines()
            for i in dlines:
                if not i.startswith('#'):
                    exempt_groups.append(i.replace('\n', ''))

        for group in mixins_tree.keys():
            if group not in exempt_groups:
                self._check_name(mixins_tree[group]["group_dir"])
                options = mixins_tree[group]["options"].keys()
                for option in options:
                    self._check_name(mixins_tree[group]["options"][option]["optiondir"])
                # default and false are mandatory options
                for option in ["default", "false"]:
                    if option not in options:
                        self.policy_error("group {}: missing {} option".format(group, option))
                    if option == "default" :
                        optdir = mixins_tree[group]["options"][option]["optiondir"]
                        if not os.path.islink(optdir) :
                            self.policy_error("group {}: default option should be a symbolic file which links to false option".format(group))
                        else:
                            opt = os.readlink(optdir)
                            if opt not in ["false", "false/"]:
                                self.policy_error("group {}: default option should link to false option".format(group))

                # doc.spec rule check
                if os.path.exists(mixins_tree[group]["doc"]["fn"]):
                    self._check_doc(mixins_tree[group])
                else:
                    self.policy_error("{}: lack doc.spec file to describle your group.".format(group))
        self.debug("Pass")


class Releaser(FileHelper):

    def __init__(self, dry_run, abort_on_errors, tag):
        FileHelper.__init__(self, abort_on_errors, "Releaser-{}".format(tag))
        self.dry_run = dry_run


    def _is_builtin_file(self, fn):

        for f in Configurer.FILE_LIST + [Configurer.OPTION_SPEC_FN, Configurer.FILES_SPEC_FN]:
            if fn.startswith(f):
                return True

        return False

    def _release_one_option(self, opt_src, opt_dst, option_tree):

        if not os.path.exists(opt_dst):
            os.makedirs(opt_dst)
        # Create default option in external mixins group if internal doesn't include
        if not os.path.exists(opt_src) and os.path.basename(opt_src) in ["default", "false"] :
            open("{}/empty_dir".format(opt_dst), "w").close()
            return

        ### Copy builtin files
        builtin_files = [i for i in os.listdir(opt_src) if self._is_builtin_file(i)]
        for f in builtin_files:
            self.update_file_with_params(os.path.join(opt_src, f), os.path.join(opt_dst, f))

        # remove redundant files in target's mixin tree
        # TODO: remove extra directories ?
        opt_dst_f_set = set([i for i in os.listdir(opt_dst) if self._is_builtin_file(i)])
        redundant_files = opt_dst_f_set - set(builtin_files)
        for f in redundant_files:
            fn_dst = os.path.join(opt_dst, f)
            os.remove(fn_dst)

        ### Copy extrafiles and devicefiles
        for f in option_tree["extrafiles"] + option_tree["devicefiles"]:
            src_fn = os.path.join(opt_src, f)
            dst_fn = os.path.join(opt_dst, f)
            if os.path.isdir(src_fn):
                self.update_directory_with_params(src_fn, dst_fn)
            else:
                self.update_file_with_params(src_fn, dst_fn)

        # Don't add empty folder as mixins will delete empty folder when mixinsctl is set.
        files = os.listdir(opt_dst)
        if len(files) == 0:
            open("{}/empty_dir".format(opt_dst), "w").close()


    def _release_valid_options(self, group_src, group_dst, options_copy, group_tree):

        for opt in options_copy:
            opt_src = os.path.join(group_src, opt)
            opt_dst = os.path.join(group_dst, opt)
            option_tree = {}
            if opt in group_tree["options"].keys():
                option_tree = group_tree["options"][opt]
            if os.path.islink(opt_src):
                opt_src_real  = os.path.realpath(opt_src)
                realname = os.path.basename(opt_src_real)
                opt_dst_real = os.path.join(group_dst, realname)
                self._release_one_option(opt_src, opt_dst_real, option_tree)
                if not os.path.exists(opt_dst):
                    os.symlink(realname, opt_dst)
            else:
                self._release_one_option(opt_src, opt_dst, option_tree)


    def _release_option_doc_description(self, group_src, group_dst, option, doc_tree_src, parser):

        docinfo_src = os.path.join(group_src, Configurer.DOC_SPEC_FN)
        if os.path.exists(docinfo_src):
            dlines = []
            for section in doc_tree_src.keys():
                if section == "Options":
                    dlines.append("{} {}\n".format(parser.doc_delims["section"], section))
                    dlines.append("\n{} {}\n".format(parser.doc_delims["option"], option))
                    if option in doc_tree_src["Options"].keys():
                        dlines.extend(doc_tree_src["Options"][option])
                else:
                    dlines.append("{} {}\n".format(parser.doc_delims["section"], section))
                    dlines.extend(doc_tree_src[section])
                dlines.append("\n")

            docinfo_dst = os.path.join(group_dst, Configurer.DOC_SPEC_FN)
            if os.path.exists(docinfo_dst):
                with open(docinfo_dst) as dfile:
                    dst_lines = dfile.readlines()
                doc_tree_dst = parser.read_doc_tree(docinfo_dst)
                if option not in doc_tree_dst["Options"].keys() and option in doc_tree_src["Options"].keys():
                    dst_lines.append("\n{} {}\n".format(parser.doc_delims["option"], option))
                    dst_lines.extend(doc_tree_src["Options"][option])
                    with open(docinfo_dst, "w") as dfile:
                        dfile.writelines(dst_lines)
            else:
                with open(docinfo_dst, "w") as dfile:
                    dfile.writelines(dlines)


    def _release_one_group(self, group, option, params, parser, output_dir):

        group_tree = parser.internal_mixins_tree[group]
        group_src = group_tree["group_dir"]
        group_dst = os.path.join(output_dir, group)

        ### Copy options
        options_copy = {option, "default", "false"}
        self._release_valid_options(group_src, group_dst, options_copy, group_tree)

        ### Copy doc spec file
        doc_tree = group_tree["doc"]["tree"]
        for opt in options_copy:
            self._release_option_doc_description(group_src, group_dst, opt, doc_tree, parser)

        ### Copy group meta file
        mixinfo_src = os.path.join(group_src, Configurer.GROUP_SPEC_FN)
        mixinfo_dst = os.path.join(group_dst, Configurer.GROUP_SPEC_FN)
        if os.path.exists(mixinfo_src):
            self.update_file_with_params(mixinfo_src, mixinfo_dst)



    def _release_mixins_tree(self, parser):


        for group, option, params in parser.internal_selections:
            # Only copy valid default option
            if group not in parser.group_deps and option == "default":
                optdir = os.path.join(Configurer.INTERNAL_TREE, group, option)
                if not set(os.listdir(optdir)) & set(Configurer.FILE_LIST):
                    continue
            self._release_one_group(group, option, params, parser, parser.mixins_tree_dir)

        parser.mixins_tree = parser.read_mixins_tree(parser.mixins_tree_dir)
        selections = parser.spec_selections[:]
        parser.selections = parser.validate_selections(selections, parser.mixins_tree)


    def _release_info_file(self, parser):

        retval = True

        release_info = {}
        release_info["table"] = []
        release_info["contents"] = []
        release_info["table"].append("This document describes every mixins group overview and current target's option.\n")
        release_info["table"].append("\n")
        release_info["table"].append("{} Table of Contents\n".format(parser.doc_delims["section"]))
        release_info["table"].append("\n")
        doc_seq_num = 1
        for group, option, params in parser.selections:
            group_tree = parser.mixins_tree[group]
            doc_spec_fn = group_tree["doc"]["fn"]
            if os.path.exists(doc_spec_fn):
            # table & contents title
                release_info["table"].append("    {} [{}] {}\n".format(parser.doc_delims["section"], doc_seq_num, group))
                release_info["contents"].append("{} [{}] {}\n".format(parser.doc_delims["section"], doc_seq_num, group))
                doc_seq_num += 1
                doc_tree = group_tree["doc"]["tree"]
                for section in doc_tree.keys():
                    if section == "Options":
                        if option in doc_tree["Options"].keys():
                            release_info["contents"].append("{} {}\n".format(parser.doc_delims["option"], option))
                            release_info["contents"].extend(doc_tree["Options"][option])
                            release_info["contents"].append("\n")
                    else:
                        release_info["contents"].extend(doc_tree[section])
                        release_info["contents"].append("\n")
        release_info["table"].append("\n")
        release_info_dlines = release_info["table"] + release_info["contents"]
        release_info_fn = os.path.join(parser.mixins_dir,  Configurer.RELEASE_INFO_FN)
        if not self.update_file_with_lines(release_info_dlines, release_info_fn):
            retval = False

        return retval


    def _release_exempt_file(self, parser):
        src_exempt_fn = os.path.join(parser.internal_mixins_dir, Configurer.EXEMPT_FN)
        dst_exempt_fn = os.path.join(parser.mixins_dir, Configurer.EXEMPT_FN)
        src_exempt_groups = []
        dst_exempt_groups = []
        if os.path.exists(src_exempt_fn):
            with open(src_exempt_fn) as sfile:
                slines = sfile.readlines()

            for i in slines:
                if not i.startswith('#'):
                    src_exempt_groups.append(i.replace('\n', ''))

            for group, option, params in parser.selections:
                if group in src_exempt_groups:
                    dst_exempt_groups.append(group+'\n')

            if os.path.exists(dst_exempt_fn):
                with open(dst_exempt_fn) as dfile:
                    dlines = dfile.readlines()
                delta =  set(dst_exempt_groups) - set(dlines)
                if delta:
                    for i in delta:
                        dlines.append(i)
                self.update_file_with_lines(dlines, dst_exempt_fn)
            else:
                self.update_file_with_lines(dst_exempt_groups, dst_exempt_fn)


    def _release_mixins_files(self, parser):

        # copy directly
        for fn in [Configurer.SCRIPT_FN, Configurer.CHANGE_LOG_FN, Configurer.RULE_FN]:
            self.update_file_with_params(os.path.join(parser.internal_mixins_dir, fn), os.path.join(parser.mixins_dir, fn))

        # copy template groups
        for template in Configurer.TEMPLATE_GROUPS:
            self.update_directory_with_params(os.path.join(parser.internal_mixins_tree_dir, template), os.path.join(parser.mixins_tree_dir, template))

        # release info file
        if parser.product_dir in parser.mixins_tree_dir:
            self._release_info_file(parser)
        # release exempt file
        self._release_exempt_file(parser)


    def release_mixins(self, parser):

        if self.dry_run and os.path.exists(parser.mixins_dir):
            self.debug("backuping origin mixins: {}".format(parser.mixins_dir))
            parser.mixins_backup = os.path.join(tempfile.mkdtemp(), "mixins")
            shutil.copytree(parser.mixins_dir, parser.mixins_backup, symlinks=True)

        if "project-celadon" not in parser.mixins_dir and os.path.exists(parser.mixins_dir):
            shutil.rmtree(parser.mixins_dir)

        self._release_mixins_tree(parser)
        self._release_mixins_files(parser)
        self.debug("Pass")


class Parser(FileHelper):

    internal_mixins_tree_cache = {}
    doc_delims = {
        "section" : "===",
        "option" : "---",
    }

    def __init__(self, sf=None, dry_run=False, abort_on_errors=True):
        self.spec_file = sf
        self.dry_run = dry_run
        self.mixins_control = False
        self.mixins_release = False
        self.spec_selections = []
        # used by Updater
        self.mixins_tree_dir = ""
        self.mixins_dir = ""
        self.selections = []
        self.mixins_tree = {}
        self.product_dir = ""
        if sf != None:
            self.product_name = os.path.basename(os.path.dirname(os.path.abspath(sf)))
            tag = "Parser-{}".format(self.product_name)
        else:
            tag = "Parser"
        FileHelper.__init__(self, abort_on_errors, tag)
        # used by Releaser
        self.mixins_backup = ""
        self.internal_mixins_tree_dir = Configurer.INTERNAL_TREE
        self.internal_mixins_dir = os.path.dirname(Configurer.INTERNAL_TREE)
        self.internal_selections = []
        self.internal_mixins_tree = {}
        self.file_map = {}
        self.group_deps = set()



    def _coerce_boolean(self, d):
        for k in d:
            if type(d[k]) == str and d[k].lower() == 'false':
                d[k] = False

    def _split_lines(self, dlines, delimiter):
        """split lines based on delimiter and return a dic
        """
        d = {}

        delimiter_index = []
        # Find all delimter lines
        for i, line in enumerate(dlines):
            if line.startswith(delimiter):
                delimiter_index.append(i)

        # eg: "=== section_name", space character is needed
        for i, index in enumerate(delimiter_index):
            s = dlines[index].strip()
            if s.find(' ') == -1:
                self.error('space character is needed between delimter and section name: "{}"'.format(s))
                continue
            l = s.split()
            title = l[1]
            if i < len(delimiter_index)-1:
                content = dlines[index+1: delimiter_index[i+1]-1]
            else:
                content = dlines[index+1:]
            d[title] = content

        return d

    def read_doc_tree(self, fn):
        with open(fn) as dfile:
            dlines = dfile.readlines()

        doc_tree = self._split_lines(dlines, self.doc_delims["section"])
        if "Options" in doc_tree.keys():
            d = self._split_lines(doc_tree["Options"], self.doc_delims["option"])
            doc_tree["Options"] = d

        return doc_tree

    def _handle_include(self, sf):
        """ check for include section in a spec file.
        handle multiple include by recursion.
        return the list of file in the order they must be parsed."""

        cp = self.read_spec_file(sf)

        if cp.has_section("include"):
            relative_path = cp.get("include", "file")
            include_path = os.path.join(os.path.dirname(sf), relative_path)
            return self._handle_include(include_path) + [sf]
        return [sf]


    def read_spec_file(self, sf, cp=None, params=None):
        """read a mixin spec file and return a ConfigParser object with its
        definitions. If a ConfigParser is supplied as an argument, it will
        be augmented with the new data"""
        if not cp:
            cp = configparser.ConfigParser()
            cp.optionxform = str

        spec_file = sf
        if params:
            spec_file = self.render_file_content(sf, params)
        try:
            with open(spec_file) as fp:
                cp.read_file(fp)
        except IOError:
            if os.path.islink(spec_file):
                self.error("reading sf {} which is a symlink,"
                         " maybe it is broken".format(spec_file))
            else:
                self.error("reading sf {}".format(spec_file))

        return cp

    def _split_params(self, selections):
        regx_val = re.compile(r"(?<!\\),")  # param's value can contain comma (escaped with backslash)
        res = []
        for group, option_params in selections:
            m = re.match('([^\s(]+)\s*\(([^)]*)\)\s*$', option_params)
            if m is not None:
                option = m.group(1)
                params = dict(map(str.strip, x.replace('\,',',').split('=', 1)) for x in regx_val.split(m.group(2)))
                self._coerce_boolean(params)
                res.append((group, option, ReadTrackingDict(params)))
            else:
                res.append((group, option_params, ReadTrackingDict()))
        return res


    def read_mixins_tree(self, mixins_tree_dir, mixins_tree=None):
        """return a dictionary mapping mixin groups found in a mixins dir with
        a dictionary mapping option names to the directory containing their
        configuration fragments as well as other group-level metadata"""
        if not mixins_tree:
            mixins_tree = {}

        if mixins_tree_dir == Configurer.INTERNAL_TREE and Parser.internal_mixins_tree_cache:
            mixins_tree = Parser.internal_mixins_tree_cache
            # Reset option's extrafiles and devicefiles
            for group in mixins_tree.keys():
                group_tree = mixins_tree[group]
                for option in group_tree["options"].keys():
                    option_tree = group_tree["options"][option]
                    option_tree["extrafiles"] = []
                    option_tree["devicefiles"] = []
                    option_tree["file_map"] = {}
            return mixins_tree


        if not os.path.exists(mixins_tree_dir):
            return mixins_tree

        groups = [i for i in os.listdir(mixins_tree_dir) if not i.startswith(".") and os.path.isdir(os.path.join(mixins_tree_dir, i))]
        for group in groups:
            assert group not in mixins_tree
            mixins_tree[group] = {}
            group_dir = os.path.join(mixins_tree_dir, group)
            mixins_tree[group]["group_dir"] = group_dir
            mixins_tree[group]["options"] = {}
            mixins_tree[group]["deps"] = set()
            mixins_tree[group]["doc"] = {}

            # Check for a mixinfo file in the root of the group directory
            # This is for metadata about the group as a whole. Its presence
            # is optional, many groups won't need it.
            mixinfo_spec_fn = os.path.join(group_dir, Configurer.GROUP_SPEC_FN)
            if os.path.exists(mixinfo_spec_fn):
                cp = self.read_spec_file(mixinfo_spec_fn)

                # "mixinfo.deps" is the set of groups which must be inherited
                # prior to inheriting this mixin, typically because we need the
                # other groups to define certain variables for us
                if cp.has_option("mixinfo", "deps"):
                    mixins_tree[group]["deps"] = set(cp.get("mixinfo", "deps").split())

            mixins_tree[group]["doc"]["tree"] = {}
            mixins_tree[group]["doc"]["fn"] = ""
            doc_spec_fn = os.path.join(group_dir, Configurer.DOC_SPEC_FN)
            if os.path.exists(doc_spec_fn):
                mixins_tree[group]["doc"]["tree"] = self.read_doc_tree(doc_spec_fn)
                mixins_tree[group]["doc"]["fn"] = doc_spec_fn

            options = [i for i in os.listdir(group_dir) if not i.startswith(".") and os.path.isdir(os.path.join(group_dir, i))]
            for option in options:
                assert option not in mixins_tree[group]["options"]

                mixins_tree[group]["options"][option] = {}
                # populated in validation function
                mixins_tree[group]["options"][option]["extrafiles"] = []
                mixins_tree[group]["options"][option]["devicefiles"] = []
                mixins_tree[group]["options"][option]["file_map"] = {}
                cur_opt = mixins_tree[group]["options"][option]

                opt_dir = os.path.join(group_dir, option)
                if os.path.isdir(opt_dir):
                    cur_opt["listdir"] = os.listdir(opt_dir)
                cur_opt["optiondir"] = opt_dir

                cur_opt["deps"] = set()
                cur_opt["defaults"] = {}

                option_spec_fn = os.path.join(cur_opt["optiondir"], Configurer.OPTION_SPEC_FN)
                if option == "default" and os.path.islink(cur_opt["optiondir"]):
                    cur_opt["realname"] = os.path.basename(os.path.realpath(cur_opt["optiondir"]))
                else:
                    cur_opt["realname"] = option

                if os.path.exists(option_spec_fn):
                    cp = self.read_spec_file(option_spec_fn)

                    # Options may define their own dependencies just like at the
                    # group-level
                    if cp.has_option("mixinfo", "deps"):
                        cur_opt["deps"] = set(cp.get("mixinfo", "deps").split())

                    if cp.has_section("defaults"):
                        for k, v in cp.items("defaults"):
                            cur_opt["defaults"][k] = v

        if mixins_tree_dir == Configurer.INTERNAL_TREE:
            Parser.internal_mixins_tree_cache = mixins_tree

        return mixins_tree

    def _sort_selections(self, selections, mixins_tree):
        """sort a group selection based on their dependencies.  Parent groups
        are moved before their childs."""
        sorted_selections = []

        # Re-order the selections so that the "parent" groups appear
        # before their childs. This sort algorithm is stable.
        while selections:
            (group, option, params) = selections.pop(0)
            deps = self._get_deps(mixins_tree, group, option)
            missing_parents = []
            while deps:
                dep = deps.pop()
                parent_group = [x for x in selections if x[0] == dep]
                if not parent_group:
                    continue
                parent_group = parent_group[0]
                missing_parents.append(parent_group)
                for d in self._get_deps(mixins_tree, parent_group[0], parent_group[1]):
                    if d not in deps:
                        deps.append(d)

            while missing_parents:
                parent = missing_parents.pop()
                self.debug("moving %s before %s" % (parent[0], group))
                if parent not in sorted_selections:
                    sorted_selections.append(parent)
                if parent in selections:
                    selections.remove(parent)

            sorted_selections.append((group, option, params))

        return sorted_selections


    def _get_deps(self, mixins_tree, group, option):
        return list(mixins_tree[group]["deps"] | mixins_tree[group]["options"][option]["deps"])

    def _check_section(self, section, cp, sf):
        if not cp.has_section(section):
            self.error('sf missing {} section: {}'.format(section, sf))
            return False
        return True


    def validate_selections(self, selections, mixins_tree):
        """enforce that the set of mixin selections is sane by checking
        the following:
        1) For each group selected, verify that the group exists and that
        the particular selection made within that group also exists
        2) If any groups exist for which there is no selection made, make
        the default selection for that group. If no default exists, report an
        error.
        The selections list may be altered by this function to include
        group default selections"""

        groups_seen = []
        returned_selections = []
        default_groups = []

        # The spec file may have omitted some groups. If so, pull in their
        # default options, or generate an error if there is no default
        selected = []
        for (group, option, params) in selections:
            selected.append(group)
        unspecified_groups = sorted(set(mixins_tree.keys()) - set(selected))
        for group in unspecified_groups:
            default_groups.append(group)
            if "default" not in mixins_tree[group]["options"]:
                self.error("group {} doesn't have a default option!".format(group))
            else:
                params = mixins_tree[group]["options"]["default"]["defaults"]
                self._coerce_boolean(params)
                selections.append((group, "default", ReadTrackingDict(params)))

        selections = self._sort_selections(selections, mixins_tree)

        for (group, option, params) in selections:

            if group in groups_seen:
                self.error("selection already made for group {}".format(group))
                continue

            groups_seen.append(group)

            if group not in mixins_tree:
                self.error("no definition found for group {}".format(group))
                continue

            if option not in mixins_tree[group]["options"]:
                self.error("unknown option {} for group {}".format(option, group))
                continue

            deps = self._get_deps(mixins_tree, group, option)
            if deps:
                self.group_deps |= set(deps)

            for dep in deps:
                if dep not in groups_seen:
                    self.error("group {} option {} requires that group {} be selected first".format(group, option, dep))

            defaults = mixins_tree[group]["options"][option]["defaults"]
            for k, v in defaults.items():
                if k not in params:
                    params[k] = v

            # Inherit the dependency parameters
            for dep_group, dep_option, dep_params in returned_selections:
                if dep_group in deps:
                    dep_params[group] = mixins_tree[group]["options"][option]["realname"]
                    self.group_deps.add(group)
                    self._coerce_boolean(dep_params)
                    for k, v in dep_params.items():
                        if k not in params:
                            params[k] = v
                    params[dep_group] = mixins_tree[dep_group]["options"][dep_option]["realname"]
            extra_dir = os.path.join(Configurer.EXTRA_DIR, group)
            params[Configurer.EXTRA_DIR_KEY] = extra_dir
            self._coerce_boolean(params)
            returned_selections.append((group, option, params))

        # popuplate extrafiles & devicefiles info
        for (group, option, params) in selections:
            cur_opt = mixins_tree[group]["options"][option]
            sf = os.path.join(cur_opt["optiondir"], Configurer.FILES_SPEC_FN)
            if os.path.isfile(sf) or os.path.islink(sf):
                cp = self.read_spec_file(sf, params=params)
                if cp.has_section("mapping"):
                    for fn, mfn in cp.items("mapping"):
                        cur_opt["file_map"][fn] = mfn
                if cp.has_section("extrafiles"):
                    for newfile, comment in cp.items("extrafiles"):
                        cur_opt["extrafiles"].append(newfile)
                if cp.has_section("devicefiles"):
                    for newfile, comment in cp.items("devicefiles"):
                        cur_opt["devicefiles"].append(newfile)

        return returned_selections


    def dump(self):
        self.debug('spec_file: {}'.format(self.spec_file))
        self.debug('dry_run: {}'.format(self.dry_run))
        self.debug('mixins_control: {}'.format(self.mixins_control))
        self.debug('mixins_release: {}'.format(self.mixins_release))
        self.debug('internal_mixins_tree_dir: {}'.format(self.internal_mixins_tree_dir))
        self.debug('internal_mixins_dir: {}'.format(self.internal_mixins_dir))
        self.debug('mixins_tree_dir: {}'.format(self.mixins_tree_dir))
        self.debug('mixins_dir: {}'.format(self.mixins_dir))
        self.debug('mixins_backup: {}'.format(self.mixins_backup))
        self.debug('file_map: {}'.format(self.file_map))
        self.debug('product_dir: {}'.format(self.product_dir))
        self.debug('abort_on_errros: {}'.format(self.abort_on_errors))

    def parse_spec_file(self):

        self.debug("processing {}".format(self.spec_file))
        file_list = self._handle_include(self.spec_file)
        cp = None
        for f in file_list:
            cp = self.read_spec_file(f, cp)

        if False in (
                self._check_section("main", cp, self.spec_file),
                self._check_section("groups", cp, self.spec_file)
                ):
            return

        cp1 = self.read_spec_file(self.spec_file)
        if cp1.has_option("main", "mixinsctl") and cp1.get("main", "mixinsctl").lower() == "true":
            self.mixins_control = True
        if cp1.has_option("main", "mixinsrel") and cp1.get("main", "mixinsrel").lower() == "true":
            self.mixins_release = True

        self.product_dir = os.path.dirname(self.spec_file)
        if self.product_dir == '':
            self.product_dir = '.'
        self.spec_selections = self._split_params(cp.items("groups"))
        self.spec_mixins_tree_dir = cp.get("main", "mixinsdir")
        if self.spec_mixins_tree_dir.startswith('.'):
            self.mixins_tree_dir = os.path.join(self.product_dir, self.spec_mixins_tree_dir.replace("./", ""))
        else:
            self.mixins_tree_dir = self.spec_mixins_tree_dir

        if os.path.exists(self.internal_mixins_tree_dir):
            self.internal_selections = self.spec_selections[:]
            self.internal_mixins_tree = self.read_mixins_tree(self.internal_mixins_tree_dir)
            # After this, all default selections should be populated
            # and the selections should be sane
            self.internal_selections = self.validate_selections(self.internal_selections, self.internal_mixins_tree)

            # will use internal tree if no mixins release
            if not self.mixins_release:
                self.mixins_tree_dir = Configurer.INTERNAL_TREE
                self.mixins_tree = self.internal_mixins_tree
                self.selections = self.internal_selections
        else:
            self.selections = self.spec_selections[:]
            self.mixins_tree = self.read_mixins_tree(self.mixins_tree_dir)
            # After this, all default selections should be populated
            # and the selections should be sane
            self.selections = self.validate_selections(self.selections, self.mixins_tree)

        self.mixins_dir = os.path.dirname(self.mixins_tree_dir)
        # Set up the file map since the config files in the product directory
        # may have slightly different names than what is in Configurer.FILE_LIST
        if cp.has_section("mapping"):
            for fn, mfn in cp.items("mapping"):
                self.file_map[fn] = mfn

        self.dump()
        self.debug("Pass")


class Updater(FileHelper):

    # We're assuming '#' is valid to start a comment in all the configs we are amending
    # which so far is true.
    delimiters = {
        "header" : "# ----------------- BEGIN MIX-IN DEFINITIONS -----------------\n",
        "footer" : "# ------------------ END MIX-IN DEFINITIONS ------------------\n",
        "source" : "# Source: {}\n",
        "line"   : "##############################################################\n",
        "warn"   : "# Mix-In definitions are auto-generated by {}\n"
    }

    warn_vars = [
        "BOARD_KERNEL_CMDLINE",
        "DEVICE_PACKAGE_OVERLAYS",
        "PRODUCT_COPY_FILES",
        "PRODUCT_DEFAULT_PROPERTY_OVERRIDES",
        "PRODUCT_PACKAGES",
        "PRODUCT_PACKAGES_DEBUG",
        "PRODUCT_PACKAGES_ENG",
        "PRODUCT_PACKAGES_TESTS",
        "PRODUCT_PACKAGE_OVERLAYS",
        "PRODUCT_PROPERTY_OVERRIDES",
        ]

    def __init__(self, dry_run, abort_on_errors, log_level, tag):
        FileHelper.__init__(self, abort_on_errors, "Updater-{}".format(tag))
        self.dry_run = dry_run

        if log_level == logging.DEBUG:
            self.logger.setLevel(logging.DEBUG)
        else:
            if dry_run:
                self.logger.setLevel(logging.WARNING)
            else:
                self.logger.setLevel(logging.INFO)

    def _clear_file(self, dest):
        """Return of list of string lines based on the destination file.
        Clear out any existing mixin defintions from the specified file; after this
        is done only the header/footer will remain. If the dest file never had anything
        in it, add the header/footer"""
        output = []
        in_mixin = False
        ever_in_mixin = False

        if os.path.exists(dest):
            with open(dest) as dfile:
                dlines = dfile.readlines()
        else:
            dlines = []

        orig_dlines = dlines[:]

        for line in dlines:
            if not in_mixin:
                output.append(line)
                if line == self.delimiters["header"]:
                    in_mixin = True
                    ever_in_mixin = True
                    output.append(self.delimiters["warn"].format(os.path.basename(sys.argv[0])))
            else:
                if line == self.delimiters["footer"]:
                    output.append(line)
                    in_mixin = False

        if in_mixin:
            # header with no footer? ok whatever
            output.append(self.delimiters["footer"])

        if not ever_in_mixin:
            output.append(self.delimiters["header"])
            output.append(self.delimiters["warn"].format(os.path.basename(sys.argv[0])))
            output.append(self.delimiters["footer"])

        return orig_dlines, output


    def _amend_file(self, dlines, src, mixins_tree_dir, product_dir, params):
        """Augment the destination lines list with data from the source file provided.
        Assumes we have run clear_file() on dlines at some point beforehand"""
        with open(src) as sfile:
            src_contents = sfile.read()
            if src_contents[-1] != '\n':
                self.error("No Newline at the end of {}".format(src))

            try:
                slines = pystache.render(src_contents, params)
            except pystache.context.KeyNotFoundError as e:
                self.error("{} depends on undefined mixin parameter '{}'".format(src, e.key))

        # sanity checks
        if slines.find(mixins_tree_dir) >= 0:
            self.error("build-time references to paths inside mixin directory are not allowed; {} is invalid".format(src))

        # much faster than previous regexp (\w*)
        m = re.findall('(' + '|'.join(self.warn_vars) + ')\s*:=', slines)
        if m:
            self.warning("Non-accumulative assignment to '{}' found in {}".format(m[0], src))

        idx = dlines.index(self.delimiters["footer"])
        output = dlines[:idx]
        output.insert(idx, self.delimiters["line"])
        if product_dir in src:
            src = src.replace(product_dir, '.')
        output.insert(idx, self.delimiters["source"].format(src))
        output.insert(idx, self.delimiters["line"])
        output.extend(slines.splitlines(True))
        output.extend(dlines[idx:])
        return output


    def _update_builtin_files(self, parser):

        retval = True

        for dest in Configurer.FILE_LIST:
            src = dest
            if dest in parser.file_map.keys():
                dest = parser.file_map[dest]
            dest_fn = os.path.join(parser.product_dir, dest)
            orig_dest_lines, dest_lines = self._clear_file(dest_fn)
            if parser.mixins_control and len(dest_lines) != 3:
                self.error("Not allowed to modify {} when mixinsctl set to true!".format(os.path.join(parser.product_dir,dest)))
            # Now check all the groups to see if they have a configuration
            # fragment to insert into the destination file
            for group, option, params in parser.selections:
                cur_opt = parser.mixins_tree[group]["options"][option]
                optdir = cur_opt["optiondir"]
                # Any given file can have multiple fragments. We first
                # look for <frag>.1, <frag>.2, ... <frag>.9, <frag>
                # This is useful for when several mixin options have mostly
                # the same data except for maybe a few lines; you can
                # avoid copypasting a lot of stuff by using this feature
                # and symbolic links for the common bits.
                dsrc, fsrc = os.path.split(src)
                flist = [f for f in cur_opt["listdir"] if f.startswith(fsrc) and (dsrc in optdir)]
                fraglist = [f for f in flist if len(f) == len(fsrc) + 2 and f[-2] == '.' and f[-1].isdigit()]
                if fraglist:
                    fraglist.sort()
                    for f in fraglist:
                        dest_lines = self._amend_file(dest_lines, os.path.join(optdir, f), parser.mixins_tree_dir, parser.product_dir, params)
                if fsrc in flist:
                    dest_lines = self._amend_file(dest_lines, os.path.join(optdir, fsrc), parser.mixins_tree_dir, parser.product_dir, params)

            if not self.update_file_with_lines(dest_lines, dest_fn, self.dry_run):
                retval = False

        return retval




    def _update_extra_files(self, parser):
        return self._update_extra_or_device_files(parser, True)


    def _update_device_files(self, parser):
        return self._update_extra_or_device_files(parser, False)


    def _update_extra_or_device_files(self, parser, is_extra):

        retval = True
        key = ""
        if is_extra:
            key = "extrafiles"
        else:
            key = "devicefiles"
        for group, option, params in parser.selections:
            cur_opt = parser.mixins_tree[group]["options"][option]
            opt_dir = cur_opt["optiondir"]
            if cur_opt[key]:
                for fn in cur_opt[key]:
                    src_fn = os.path.join(opt_dir, fn)
                    if fn in cur_opt["file_map"].keys():
                        fn = cur_opt["file_map"][fn]
                    if key == "extrafiles":
                        dst_fn = os.path.join(parser.product_dir, Configurer.EXTRA_DIR, group, fn)
                    else:
                        dst_fn = os.path.join(parser.product_dir, fn)
                    if os.path.isdir(src_fn):
                        if not self.update_directory_with_params(src_fn, dst_fn, self.dry_run, params):
                            retval = False

                    else:
                        if not self.update_file_with_params(src_fn, dst_fn, self.dry_run, params):
                            retval = False

        return retval


    def _delete_folder_recursively_if_empty(self, path):

        if not os.path.isdir(path):
            return

        files = os.listdir(path)
        if len(files):
            for f in files:
                fn = os.path.join(path, f)
                if os.path.isdir(fn) and ".git" not in fn:
                    self._delete_folder_recursively_if_empty(fn)
        # if folder empty, delete it
        files = os.listdir(path)
        if len(files) == 0:
            self.info("removing empty folder {}".format(path))
            os.rmdir(path)


    def _clear_redundant_files_under_product_dir(self, parser):

        local_file_tree = set()
        for root, dirs, files in os.walk(parser.product_dir):
            # Exclude mixins folder
            if files and "mixins" not in root and ".git" not in root:
                for f in files:
                    fn = os.path.join(root, f)
                    local_file_tree.add(fn)
        delta = local_file_tree - self.update_file_set
        for f in delta:
            # exclude mixins spec file
            if not f.endswith(Configurer.SPEC_SUFFIX) and not f.find(Configurer.ANDROID_PRODUCTS_MK):
                self.info("removing unused file {}".format(f))
                os.remove(f)

        self._delete_folder_recursively_if_empty(parser.product_dir)


    def update_product_files(self, parser):

        retval = True
        self.info('processing: {}'.format(parser.spec_file))
        self.info('mixinsdir: {}'.format(parser.mixins_tree_dir))

        if not self._update_builtin_files(parser):
            retval = False
        if not self._update_extra_files(parser):
            retval = False
        if not self._update_device_files(parser):
            retval = False

        if not parser.dry_run and parser.mixins_control:
            self._clear_redundant_files_under_product_dir(parser)

        if self.dry_run and os.path.exists(parser.mixins_backup):
            self.debug('restoring original mixins:{}'.format(parser.mixins_dir))
            shutil.rmtree(parser.mixins_dir)
            shutil.copytree(parser.mixins_backup, parser.mixins_dir, symlinks=True)
            shutil.rmtree(parser.mixins_backup)
        #for group, option, params in parser.selections:
        #    unused = params.unused_keys()
        #    if unused:
        #        policy_error('Unnecessary parameters {} given for mixin option'
        #                   ' "{}: {}"'.format(unused, group, option))
        self.debug("Pass")
        return retval

def get_valid_target_list(fn, ret):
    valid_targets = ""
    cp = Parser().read_spec_file(fn)
    for platform in cp.sections():
        for target in cp.options(platform):
            valid_targets += "{} ".format(target)
            sf = os.path.join(Configurer.INTERNAL_DEVICE_DIR, platform, target, Configurer.PRODUCT_SPEC_FN)
            if os.path.exists(sf):
                ret.append(sf)
    return valid_targets

def get_valid_spec_files_and_targets():
    fn = os.path.join(os.path.dirname(sys.argv[0]), Configurer.VALID_TARGETS_FN)
    emb_fn = os.path.join(os.path.dirname(sys.argv[0]), Configurer.EMBARGOED_VALID_TARGETS_FN)
    dlines = []
    ret = []
    valid_targets = ""
    if os.path.exists(fn):
        valid_targets += get_valid_target_list(fn, ret)
        if os.path.exists(emb_fn):
            valid_targets += get_valid_target_list(emb_fn, ret)
    else:
        for root, dirs, files in os.walk("device"):
            if Configurer.PRODUCT_SPEC_FN in files:
                ret.append(os.path.join(root, Configurer.PRODUCT_SPEC_FN))

    return ret, valid_targets

def main(dry_run=False, sfiles=None, warn_only=False, debug_log=False, check_version=False, check_target=False):

    ret = 0
    abort_on_errors = not warn_only
    out_of_sync = False
    policy_errors_found = False

    # set the default level of logger
    if debug_log:
        log_level = logging.DEBUG
    else:
        log_level = logging.WARNING
    logging.basicConfig(format='%(name)s: %(message)s', level=log_level)

    if check_version:
        print ("Mixins {}".format(Configurer.VERSION))
        return ret

    if check_target:
        print (get_valid_spec_files_and_targets()[1])
        return ret

    if not sfiles:
        sfiles = get_valid_spec_files_and_targets()[0]

    for sf in sfiles:
        parser = Parser(sf, dry_run, abort_on_errors)
        tag = parser.product_name
        checker = Checker(abort_on_errors, tag)
        releaser = Releaser(dry_run, abort_on_errors, tag)
        updater = Updater(dry_run, abort_on_errors, log_level, tag)
        # parse
        parser.parse_spec_file()
        # check rules
        if parser.mixins_control:
            checker.check_rules(parser)
        # release mixins if needed
        if parser.mixins_release and os.path.exists(parser.internal_mixins_tree_dir):
            releaser.release_mixins(parser)
        # update product files
        if not updater.update_product_files(parser):
            out_of_sync = True

        if parser.errors_found or checker.errors_found or releaser.errors_found or updater.errors_found:
            policy_errors_found = True

    if out_of_sync:
        print ("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
        print ("+ WARN: Product configurations are out of sync                          +")
        print ("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
        ret = 3

    if policy_errors_found:
        print ("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
        print ("+ WARN: Some spec files have policy issues                              +")
        print ("+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
        ret = 2

    return ret



if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Update board configurations with mixin data based on spec file")
    parser.add_argument("-d", "--dry-run",
            help="Don't make any actual changes, exit nonzero if something needs to be updated",
            action="store_true")
    parser.add_argument("-s", "--spec",
            help="Read a specific spec file. Can be called multiple times. Defaults to scanning the tree under device/intel/ for spec files",
            action="append")
    parser.add_argument("-w", "--warn-only",
            help="Generate warnings instead of fatal errors for spec file policy violations",
            action="store_true")

    parser.add_argument("-g", "--debug",
            help="Verbose debug info",
            action="store_true")

    parser.add_argument("-v", "--version",
            help="Check mixins version",
            action="store_true")

    parser.add_argument("-t", "--target",
            help="Get valid lunch targets",
            action="store_true")

    args = parser.parse_args()

    ret = main(args.dry_run, args.spec, args.warn_only, args.debug, args.version, args.target)
    sys.exit(ret)