Add yamllint as tool

Closes #104

README.md

@@ -34,7 +34,8 @@ On the other hand some static code analysis does not make any sense on an image-
  * pylint (python) [pylint]
  * shellcheck (shell) [shellcheck]
  * systemdlint (systemd) [systemdlint]
- * xmlint (xml) [xmllint]
+ * xmllint (xml) [xmllint]
+ * yamllint (xml) [yamllint]
 
 ### Tools for all other recipes
 
@@ -60,7 +61,8 @@ On the other hand some static code analysis does not make any sense on an image-
  * shellcheck (shell) [shellcheck]
  * textlint (spelling) [textlint]
  * tscancode (c,c#,lua) [tscancode]
- * xmlint (xml) [xmllint]
+ * xmllint (xml) [xmllint]
+ * yamllint (xml) [yamllint]
  * zeroresourcedetector (g18n/i18n) [zrd]
 
 each tool does have it's own benefits and flaws so don't be mad if you have 10k+ findings on the initial run.
@@ -427,6 +429,15 @@ __NOTE:__ this tool does not support suppression or fatal error handling
 | SCA_XMLLINT_EXTRA_SUPPRESS | Extra error-ids to be suppressed | space-separated-list | ""
 | SCA_XMLLINT_FILE_FILTER | List of file-extensions to be checked | space-separated-list | ".xml"
 
+### Available configuration for yamllint
+
+| var | purpose | type | default |
+| ------------- |:-------------:| -----:| -----:
+| SCA_BLACKLIST_yamllint | Blacklist filter for this tool | space-separated-list | ""
+| SCA_YAMLLINT_EXTRA_FATAL | Extra error-ids leading to build termination when found | space-separated-list | ""
+| SCA_YAMLLINT_EXTRA_SUPPRESS | Extra error-ids to be suppressed | space-separated-list | ""
+| SCA_YAMLLINT_FILE_FILTER | List of file-extensions to be checked | space-separated-list | ".yaml"
+
 ### Available configuration for zrd
 
 | var | purpose | type | default |

classes/sca-blackllist.bbclass

@@ -27,6 +27,7 @@ SCA_BLACKLIST_systemdlint ?= ""
 SCA_BLACKLIST_textlint ?= ""
 SCA_BLACKLIST_tscanscode ?= "linux-.*"
 SCA_BLACKLIST_xmllint ?= ""
+SCA_BLACKLIST_yamllint ?= ""
 SCA_BLACKLIST_zrd ?= ""
 
 def sca_is_module_blacklisted(d, tool):

classes/sca-conv-checkstyle-yamllint.bbclass

@@ -0,0 +1,71 @@
+## This file contains the converter from raw
+## cppcheck-format to checkstyle
+
+inherit sca-conv-checkstyle-helper
+
+def do_sca_conv_yamllint(d):
+    import os
+    import re
+    from xml.etree.ElementTree import Element, SubElement, Comment, tostring
+    from xml.etree import ElementTree
+    from xml.dom import minidom
+
+    package_name = d.getVar("PN", True)
+    buildpath = d.getVar("SCA_SOURCES_DIR", True)
+
+    items = []
+    pattern = r"^(?P<file>.*):(?P<line>\d+):(?P<column>\d+):\s+\[(?P<severity>\w+)\]\s+(?P<msg>.*)\s\((?P<id>.*)\)"
+
+    __supress = get_suppress_entries(d)
+
+    class YamlLintItem():
+        File = ""
+        Line = ""
+        Column = ""
+        Severity = ""
+        Message = ""
+        ID = ""
+
+    severity_map = {
+        "error" : "error",
+        "warning" : "warning",
+    }
+
+    if os.path.exists(d.getVar("SCA_RAW_RESULT")):
+        f = open(d.getVar("SCA_RAW_RESULT"), "r")
+        content = f.read()
+        f.close()
+
+        for m in re.finditer(pattern, content, re.MULTILINE):
+            try:
+                g = YamlLintItem()
+                g.File = m.group("file")
+                g.Line = m.group("line")
+                g.Column = m.group("column")
+                g.Severity = severity_map[m.group("severity")]
+                g.Message = "[Package:%s Tool:yamllint] %s" % (package_name, m.group("msg"))
+                g.ID = "yamllint.yamllint.%s" % m.group("id")
+                if g.Severity in checkstyle_allowed_warning_level(d) and not g.ID in __supress:
+                    items.append(g)
+            except:
+                pass
+
+    filenames = list(set([x.File for x in items]))
+
+    top = Element("checkstyle")
+    top.set("version", "4.3")
+
+    for _file in filenames:
+        _fe = SubElement(top, "file", { "name": _file })
+        for _fileE in [x for x in items if x.File == _file ]:
+            _fee = SubElement(_fe, "error", {
+                "column": _fileE.Column,
+                "line": _fileE.Line,
+                "message": _fileE.Message,
+                "severity": _fileE.Severity,
+                "source": _fileE.ID
+            })
+    try:
+        return checkstyle_prettify(d, top).decode("utf-8")
+    except:
+        return checkstyle_prettify(d, top)

classes/sca-global.bbclass

@@ -51,6 +51,7 @@ SCA_AVAILABLE_MODULES ?= "\
                           textlint \
                           tscancode \
                           xmllint \
+                          yamllint \
                           zrd"
 
 inherit sca-conv-checkstyle-helper

classes/sca-on-image.bbclass

@@ -6,7 +6,7 @@ inherit sca-file-filter
 inherit sca-blackllist
 
 SCA_PACKAGE_LICENSE_FILTER = "CLOSED"
-SCA_ENABLED_MODULES ?= "ansible bandit bitbake eslint gixy jsonlint oelint pylint shellcheck systemdlint xmllint"
+SCA_ENABLED_MODULES ?= "ansible bandit bitbake eslint gixy jsonlint oelint pylint shellcheck systemdlint yamllint xmllint"
 SCA_SOURCES_DIR ?= "${IMAGE_ROOTFS}"
 
 SCA_MODE = "image"

classes/sca-on-recipe.bbclass

@@ -7,7 +7,7 @@ inherit sca-blackllist
 
 SCA_ENABLED_MODULES ?= "bandit bitbake cspell cve-check clang dennis eslint flint cpplint cppcheck gcc \
                         jsonlint kconfighard oelint pylint pysymcheck pytype oclint rats shellcheck \
-                        textlint tscancode xmllint zrd"
+                        textlint tscancode xmllint yamllint zrd"
 SCA_SOURCES_DIR ?= "${B}"
 
 SCA_MODE = "recipe"

classes/sca-yamllint-core.bbclass

@@ -0,0 +1,67 @@
+inherit sca-conv-checkstyle-yamllint
+inherit sca-license-filter
+inherit sca-helper
+
+SCA_YAMLLINT_EXTRA_SUPPRESS ?= ""
+SCA_YAMLLINT_EXTRA_FATAL ?= ""
+## File extension filter list (whitespace separated)
+SCA_YAMLLINT_FILE_FILTER ?= ".yaml"
+
+inherit ${@oe.utils.ifelse(d.getVar('SCA_STD_PYTHON_INTERPRETER') == 'python3', 'python3native', 'pythonnative')}
+
+DEPENDS += "${SCA_STD_PYTHON_INTERPRETER}-yamllint-native"
+
+python do_sca_yamllint_core() {
+    import os
+    import subprocess
+    import json
+
+    d.setVar("SCA_EXTRA_SUPPRESS", d.getVar("SCA_YAMLLINT_EXTRA_SUPPRESS"))
+    d.setVar("SCA_EXTRA_FATAL", d.getVar("SCA_YAMLLINT_EXTRA_FATAL"))
+    d.setVar("SCA_SUPRESS_FILE", os.path.join(d.getVar("STAGING_DATADIR_NATIVE"), "yamllint-{}-suppress".format(d.getVar("SCA_MODE"))))
+    d.setVar("SCA_FATAL_FILE", os.path.join(d.getVar("STAGING_DATADIR_NATIVE"), "yamllint-{}-fatal".format(d.getVar("SCA_MODE"))))
+
+    _supress = get_suppress_entries(d)
+    _fatal = get_fatal_entries(d)
+
+    _args = ["yamllint"]
+    _args += ["-f", "parsable"]
+
+    _files = get_files_by_extention(d, d.getVar("SCA_SOURCES_DIR"), d.getVar("SCA_YAMLLINT_FILE_FILTER"), \
+                                    sca_filter_files(d, d.getVar("SCA_SOURCES_DIR"), clean_split(d, "SCA_FILE_FILTER_EXTRA")))
+    cmd_output = ""
+    if any(_files):
+        _args += _files    
+        try:
+            cmd_output = subprocess.check_output(_args, universal_newlines=True)
+        except subprocess.CalledProcessError as e:
+            cmd_output = e.stdout or ""
+
+    result_raw_file = os.path.join(d.getVar("T"), "sca_raw_yamllint.txt")
+    d.setVar("SCA_RAW_RESULT", result_raw_file)
+    with open(result_raw_file, "w") as o:
+        o.write(cmd_output)
+
+    xml_output = do_sca_conv_yamllint(d)
+    result_file = os.path.join(d.getVar("T"), "sca_checkstyle_yamllint.xml")
+    d.setVar("SCA_RESULT_FILE", result_file)
+    with open(result_file, "w") as o:
+        o.write(xml_output)
+
+    ## Evaluate
+    _warnings = get_warnings_from_result(d)
+    _fatals = get_fatal_from_result(d, "yamllint.", _fatal)
+    _errors = get_errors_from_result(d)
+
+    warn_log = []
+    if any(_warnings):
+        warn_log.append("{} warning(s)".format(len(_warnings)))
+    if any(_errors):
+        warn_log.append("{} error(s)".format(len(_errors)))
+    if warn_log and should_emit_to_console(d):
+        bb.warn("SCA has found {}".format(",".join(warn_log)))
+
+    if any(_fatals):
+        bb.build.exec_func(d.getVar("SCA_DEPLOY_TASK"), d)
+        bb.error("SCA has following fatal errors: {}".format("\n".join(_fatals)))
+}

classes/sca-yamllint-image.bbclass

@@ -0,0 +1,30 @@
+inherit sca-helper
+inherit sca-global
+inherit sca-yamllint-core
+inherit sca-conv-checkstyle-helper
+
+SCA_DEPLOY_TASK = "do_sca_deploy_yamllint_image"
+
+python do_sca_deploy_yamllint_image() {
+    import os
+    import shutil
+
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "raw"), exist_ok=True)
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "checkstyle"), exist_ok=True)
+
+    raw_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "raw", "{}-{}.txt".format(d.getVar("PN"), d.getVar("PV")))
+    cs_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "checkstyle", "{}-{}.xml".format(d.getVar("PN"), d.getVar("PV")))
+    src_raw = os.path.join(d.getVar("T"), "sca_raw_yamllint.tt")
+    src_conv = os.path.join(d.getVar("T"), "sca_checkstyle_yamllint.xml")
+    if os.path.exists(src_raw):
+        shutil.copy(src_raw, raw_target)
+    if os.path.exists(src_conv):
+        shutil.copy(src_conv, cs_target)
+    if os.path.exists(cs_target):
+        do_sca_export_sources(d, cs_target)
+}
+
+addtask do_sca_yamllint_core before do_image_complete after do_image
+addtask do_sca_deploy_yamllint_image before do_image_complete after do_sca_yamllint_core
+
+DEPENDS += "sca-image-yamllint-rules-native"

classes/sca-yamllint-recipe.bbclass

@@ -0,0 +1,29 @@
+inherit sca-helper
+inherit sca-global
+inherit sca-yamllint-core
+
+SCA_DEPLOY_TASK = "do_sca_deploy_yamllint_recipe"
+
+python do_sca_deploy_yamllint_recipe() {
+    import os
+    import shutil
+
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "raw"), exist_ok=True)
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "checkstyle"), exist_ok=True)
+
+    raw_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "raw", "{}-{}.txt".format(d.getVar("PN"), d.getVar("PV")))
+    cs_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "yamllint", "checkstyle", "{}-{}.xml".format(d.getVar("PN"), d.getVar("PV")))
+    src_raw = os.path.join(d.getVar("T"), "sca_raw_yamllint.txt")
+    src_conv = os.path.join(d.getVar("T"), "sca_checkstyle_yamllint.xml")
+    if os.path.exists(src_raw):
+        shutil.copy(src_raw, raw_target)
+    if os.path.exists(src_conv):
+        shutil.copy(src_conv, cs_target)
+    if os.path.exists(cs_target):
+        do_sca_export_sources(d, cs_target)
+}
+
+addtask do_sca_yamllint_core before do_install after do_compile
+addtask do_sca_deploy_yamllint_recipe before do_package after do_sca_yamllint_core
+
+DEPENDS += "sca-recipe-yamllint-rules-native"

recipes-python/python-pathspec-native/python-pathspec-native.inc

@@ -0,0 +1,15 @@
+SUMMARY = "Utility library for gitignore style pattern matching of file paths. "
+DESCRIPTION = "Utility library for gitignore style pattern matching of file paths. "
+HOMEPAGE = "https://github.com/cpburnz/python-path-specification"
+LICENSE = "MPL-2.0"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=815ca599c9df247a0c7f619bab123dad"
+
+RDEPENDS_${PN} += "${PYTHON_PN}-native"
+
+PYPI_PACKAGE = "pathspec"
+
+inherit pypi
+
+FILES_${PN} += "${datadir}/pathspec"
+
+inherit native

recipes-python/python-pathspec-native/python-pathspec-native_0.5.9.bb

@@ -0,0 +1,5 @@
+inherit setuptools
+require python-pathspec-native.inc
+
+SRC_URI[md5sum] = "d76bc12293518c652e1c2b7fb028529e"
+SRC_URI[sha256sum] = "54a5eab895d89f342b52ba2bffe70930ef9f8d96e398cccf530d21fa0516a873"

recipes-python/python-pathspec-native/python3-pathspec-native_0.5.9.bb

@@ -0,0 +1,5 @@
+inherit setuptools3
+require python-pathspec-native.inc
+
+SRC_URI[md5sum] = "d76bc12293518c652e1c2b7fb028529e"
+SRC_URI[sha256sum] = "54a5eab895d89f342b52ba2bffe70930ef9f8d96e398cccf530d21fa0516a873"

recipes-sca-rules/sca-image-yamllint-rules-native/sca-image-yamllint-rules-native_1.0.bb

@@ -0,0 +1,18 @@
+SUMMARY = "Ruleset for yamllint"
+DESCRIPTION = " Rules to configure how yamllint is affecting the build"
+
+SRC_URI = "file://suppress \
+           file://fatal"
+
+LICENSE ?= "BSD-2-Clause"
+LIC_FILES_CHKSUM ?= "file://${COMMON_LICENSE_DIR}/BSD-2-Clause;md5=8bef8e6712b1be5aa76af1ebde9d6378"
+
+inherit native
+
+do_install() {
+    install -d "${D}${datadir}"
+    install "${WORKDIR}/suppress" "${D}${datadir}/yamllint-image-suppress"
+    install "${WORKDIR}/fatal" "${D}${datadir}/yamllint-image-fatal"
+}
+
+FILES_${PN} = "${datadir}/**"

recipes-sca-rules/sca-recipe-yamllint-rules-native/sca-recipe-yamllint-rules-native_1.0.bb

@@ -0,0 +1,18 @@
+SUMMARY = "Ruleset for yamllint"
+DESCRIPTION = " Rules to configure how yamllint is affecting the build"
+
+SRC_URI = "file://suppress \
+           file://fatal"
+
+LICENSE ?= "BSD-2-Clause"
+LIC_FILES_CHKSUM ?= "file://${COMMON_LICENSE_DIR}/BSD-2-Clause;md5=8bef8e6712b1be5aa76af1ebde9d6378"
+
+inherit native
+
+do_install() {
+    install -d "${D}${datadir}"
+    install "${WORKDIR}/suppress" "${D}${datadir}/yamllint-recipe-suppress"
+    install "${WORKDIR}/fatal" "${D}${datadir}/yamllint-recipe-fatal"
+}
+
+FILES_${PN} = "${datadir}/**"

recipes-sca/python-yamllint-native/files/yamllint.sca.description

@@ -0,0 +1,6 @@
+{
+    "languages": ["yaml"],
+    "buildspeed" : "fast",
+    "execspeed": "normal",
+    "quality": "good"
+}

recipes-sca/python-yamllint-native/python-yamllint-native.inc

@@ -0,0 +1,22 @@
+SUMMARY = "A linter for YAML files. "
+DESCRIPTION = "A linter for YAML files. "
+HOMEPAGE = "https://github.com/adrienverge/yamllint"
+LICENSE = "GPLv3"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=d32239bcb673463ab874e80d47fae504"
+
+PYPI_PACKAGE = "yamllint"
+
+DEPENDS += "${PYTHON_PN}-pyyaml-native \
+            ${PYTHON_PN}-pathspec-native"
+
+inherit native
+inherit pypi
+
+SRC_URI += "file://yamllint.sca.description"
+
+FILES_${PN} += "${datadir}"
+
+do_install_append() {
+    install -d ${D}${datadir}
+    install ${WORKDIR}/yamllint.sca.description ${D}${datadir}
+}

recipes-sca/python-yamllint-native/python-yamllint-native_1.15.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools
+require python-yamllint-native.inc
+
+SRC_URI[md5sum] = "9116d44eba4ae9d3bd9d0c49ce4c0b6c"
+SRC_URI[sha256sum] = "8f25759997acb42e52b96bf3af0b4b942e6516b51198bebd3402640102006af7"

recipes-sca/python-yamllint-native/python3-yamllint-native_1.15.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools3
+require python-yamllint-native.inc
+
+SRC_URI[md5sum] = "9116d44eba4ae9d3bd9d0c49ce4c0b6c"
+SRC_URI[sha256sum] = "8f25759997acb42e52b96bf3af0b4b942e6516b51198bebd3402640102006af7"