Add dennis as tool

Closes #105

README.md

@@ -45,6 +45,7 @@ On the other hand some static code analysis does not make any sense on an image-
  * cpplint (c/c++) [cpplint]
  * cspell (spelling in c/c++/html/python/txt/md) [cspell]
  * cve-check (check for unpatched cve's) [cve-check]
+ * dennis (i18n) [dennis]
  * eslint (javascript/html) [eslint]
  * flint++ (c/c++) [flint]
  * gcc (getting compiler warnings/errors) [gcc]
@@ -252,6 +253,14 @@ __NOTE:__ this tool does not support suppression or fatal error handling
 | ------------- |:-------------:| -----:| -----:
 | SCA_BLACKLIST_cve_check | Blacklist filter for this tool | space-separated-list | ""
 
+### Available configuration for dennis
+
+| var | purpose | type | default |
+| ------------- |:-------------:| -----:| -----:
+| SCA_BLACKLIST_dennis | Blacklist filter for this tool | space-separated-list | ""
+| SCA_DENNIS_EXTRA_FATAL | Extra error-ids leading to build termination when found | space-separated-list | "":
+| SCA_DENNIS_EXTRA_SUPPRESS | Extra error-ids to be suppressed | space-separated-list | ""
+
 ### Available configuration for eslint
 
 | var | purpose | type | default |

classes/sca-blackllist.bbclass

@@ -9,6 +9,7 @@ SCA_BLACKLIST_cppcheck ?= "linux-.*"
 SCA_BLACKLIST_cpplint ?= "linux-.*"
 SCA_BLACKLIST_cspell ?= "linux-.*"
 SCA_BLACKLIST_cve_check ?= ""
+SCA_BLACKLIST_dennis ?= ""
 SCA_BLACKLIST_eslint ?= "linux-.*"
 SCA_BLACKLIST_flint ?= "linux-.*"
 SCA_BLACKLIST_gcc ?= ""

classes/sca-conv-checkstyle-dennis.bbclass

@@ -0,0 +1,70 @@
+## This file contains the converter from raw
+## dennis-adv-format to checkstyle
+
+inherit sca-conv-checkstyle-helper
+
+def do_sca_conv_dennis(d):
+    import os
+    import re
+    from xml.etree.ElementTree import Element, SubElement, Comment, tostring
+    from xml.etree import ElementTree
+    from xml.dom import minidom
+
+    package_name = d.getVar("PN", True)
+    buildpath = d.getVar("SCA_SOURCES_DIR", True)
+
+    items = []
+    pattern = r"^(?P<file>.*):\s+(?P<severity>W|E)(?P<id>\d+)\:\s+(?P<msg>.*)\n.*:\s+(?P<line>\d+)"
+
+    class DennisItem():
+        File = ""
+        Line = ""
+        Column = "1"
+        Severity = ""
+        Message = ""
+        ID = ""
+
+    severity_map = {
+        "E" : "error",
+        "W" : "warning",
+    }
+
+    __supress = get_suppress_entries(d)
+
+    if os.path.exists(d.getVar("SCA_RAW_RESULT_FILE")):
+        f = open(d.getVar("SCA_RAW_RESULT_FILE"), "r")
+        content = f.read()
+        f.close()
+
+        for m in re.finditer(pattern, content, re.MULTILINE):
+            try:
+                g = DennisItem()
+                g.File = m.group("file")
+                g.Line = m.group("line")
+                g.Message = "[Package:%s Tool:dennis] %s" % (package_name, m.group("msg"))
+                g.Severity = severity_map[m.group("severity")]
+                g.ID = "dennis.dennis.%s" % m.group("id")
+                if g.Severity in checkstyle_allowed_warning_level(d) and not g.ID in __supress:
+                    items.append(g)
+            except Exception:
+                pass
+
+    filenames = list(set([x.File for x in items]))
+
+    top = Element("checkstyle")
+    top.set("version", "4.3")
+
+    for _file in filenames:
+        _fe = SubElement(top, "file", { "name": _file })
+        for _fileE in [x for x in items if x.File == _file ]:
+            _fee = SubElement(_fe, "error", {
+                "column": _fileE.Column,
+                "line": _fileE.Line,
+                "message": _fileE.Message,
+                "severity": _fileE.Severity,
+                "source": _fileE.ID
+            })
+    try:
+        return checkstyle_prettify(d, top).decode("utf-8")
+    except:
+        return checkstyle_prettify(d, top)

classes/sca-dennis.bbclass

@@ -0,0 +1,91 @@
+## Add ids to suppress on a recipe level
+SCA_DENNIS_EXTRA_SUPPRESS ?= ""
+## Add ids to lead to a fatal on a recipe level
+SCA_DENNIS_EXTRA_FATAL ?= ""
+
+inherit sca-helper
+inherit sca-conv-checkstyle-dennis
+inherit sca-global
+
+inherit ${@oe.utils.ifelse(d.getVar('SCA_STD_PYTHON_INTERPRETER') == 'python3', 'python3native', 'pythonnative')}
+
+python do_sca_dennis() {
+    import os
+    import subprocess
+    d.setVar("SCA_EXTRA_SUPPRESS", d.getVar("SCA_DENNIS_EXTRA_SUPPRESS"))
+    d.setVar("SCA_EXTRA_FATAL", d.getVar("SCA_DENNIS_EXTRA_FATAL"))
+    d.setVar("SCA_SUPRESS_FILE", os.path.join(d.getVar("STAGING_DATADIR_NATIVE", True), "dennis-{}-suppress".format(d.getVar("SCA_MODE"))))
+    d.setVar("SCA_FATAL_FILE", os.path.join(d.getVar("STAGING_DATADIR_NATIVE", True), "dennis-{}-fatal".format(d.getVar("SCA_MODE"))))
+
+    _supress = get_suppress_entries(d)
+    _fatal = get_fatal_entries(d)
+
+    tmp_result = os.path.join(d.getVar("T", True), "sca_raw_dennis.txt")
+    d.setVar("SCA_RAW_RESULT_FILE", tmp_result)
+    allrun_output = ""
+    _args = ["dennis-cmd"]
+    _args += ["lint"]
+
+    _files = get_files_by_extention(d, d.getVar("SCA_SOURCES_DIR"), ".po .pot", \
+        sca_filter_files(d, d.getVar("SCA_SOURCES_DIR"), clean_split(d, "SCA_FILE_FILTER_EXTRA")))
+    for f in _files:
+        try:
+            _targs = _args + [f]
+            cmd_output = subprocess.check_output(_targs, universal_newlines=True, stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            ## manually add syntax or similar error
+            cmd_output = "E999: Parsing Error.\n1"
+
+        if cmd_output:
+            prefix = "{}: ".format(f)
+            cmd_output = prefix + prefix.join(cmd_output.splitlines(True))
+        allrun_output += cmd_output
+
+    with open(tmp_result, "w") as o:
+        o.write(allrun_output)
+
+    result_file = os.path.join(d.getVar("T", True), "sca_checkstyle_dennis.xml")
+    d.setVar("SCA_RESULT_FILE", result_file)
+    conv_output = do_sca_conv_dennis(d)
+    with open(result_file, "w") as o:
+        o.write(conv_output)
+
+    ## Evaluate
+    _warnings = get_warnings_from_result(d)
+    _fatals = get_fatal_from_result(d, "dennis.dennis", _fatal)
+    _errors = get_errors_from_result(d)
+
+    warn_log = []
+    if any(_warnings) and should_emit_to_console(d):
+        warn_log.append("{} warning(s)".format(len(_warnings)))
+    if any(_errors) and should_emit_to_console(d):
+        warn_log.append("{} error(s)".format(len(_errors)))
+    if warn_log and should_emit_to_console(d):
+        bb.warn("SCA has found {}".format(",".join(warn_log)))
+
+    if any(_fatals):
+        bb.build.exec_func("do_sca_deploy_dennis", d)
+        bb.error("SCA has following fatal errors: {}".format("\n".join(_fatals)))
+}
+
+python do_sca_deploy_dennis() {
+    import os
+    import shutil
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "dennis", "raw"), exist_ok=True)
+    os.makedirs(os.path.join(d.getVar("SCA_EXPORT_DIR"), "dennis", "checkstyle"), exist_ok=True)
+    raw_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "dennis", "raw", "{}-{}.txt".format(d.getVar("PN"), d.getVar("PV")))
+    cs_target = os.path.join(d.getVar("SCA_EXPORT_DIR"), "dennis", "checkstyle", "{}-{}.xml".format(d.getVar("PN"), d.getVar("PV")))
+    src_raw = os.path.join(d.getVar("T"), "sca_raw_dennis.txt")
+    src_conv = os.path.join(d.getVar("T"), "sca_checkstyle_dennis.xml")
+    if os.path.exists(src_raw):
+        shutil.copy(src_raw, raw_target)
+    if os.path.exists(src_conv):
+        shutil.copy(src_conv, cs_target)
+    if os.path.exists(cs_target):
+        do_sca_export_sources(d, cs_target)
+}
+
+addtask do_sca_dennis before do_install after do_configure
+addtask do_sca_deploy_dennis after do_sca_dennis before do_package
+
+DEPENDS += "${SCA_STD_PYTHON_INTERPRETER}-dennis-native sca-recipe-dennis-rules-native"

classes/sca-global.bbclass

@@ -33,6 +33,7 @@ SCA_AVAILABLE_MODULES ?= "\
                           cpplint \
                           cspell \
                           cve-check \
+                          dennis \
                           eslint \
                           flint \
                           gcc \

classes/sca-on-recipe.bbclass

@@ -5,7 +5,7 @@ inherit sca-helper
 inherit sca-file-filter
 inherit sca-blackllist
 
-SCA_ENABLED_MODULES ?= "bandit bitbake cspell cve-check clang eslint flint cpplint cppcheck gcc \
+SCA_ENABLED_MODULES ?= "bandit bitbake cspell cve-check clang dennis eslint flint cpplint cppcheck gcc \
                         jsonlint kconfighard oelint pylint pysymcheck pytype oclint rats shellcheck \
                         textlint tscancode xmllint zrd"
 SCA_SOURCES_DIR ?= "${B}"

recipes-python/python-click-native/python-click-native.inc

@@ -0,0 +1,15 @@
+SUMMARY = "Python composable command line interface toolkit"
+DESCRIPTION = "Python composable command line interface toolkit"
+HOMEPAGE = "https://github.com/pallets/click"
+LICENSE = "BSD-3-Clause"
+LIC_FILES_CHKSUM = "file://LICENSE.rst;md5=c13ed890b210a882c1778216694c98c7"
+
+DEPENDS += "${PYTHON_PN}-native"
+
+PYPI_PACKAGE = "Click"
+
+inherit pypi
+
+FILES_${PN} += "${datadir}/Click"
+
+inherit native

recipes-python/python-click-native/python-click-native_7.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools
+require python-click-native.inc
+
+SRC_URI[md5sum] = "7f53d50f7b7373ebc7963f9ff697450a"
+SRC_URI[sha256sum] = "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"

recipes-python/python-click-native/python3-click-native_7.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools3
+require python-click-native.inc
+
+SRC_URI[md5sum] = "7f53d50f7b7373ebc7963f9ff697450a"
+SRC_URI[sha256sum] = "5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"

recipes-python/python-polib-native/python-polib-native.inc

@@ -0,0 +1,15 @@
+SUMMARY = "A library to manipulate gettext files (po and mo files)."
+DESCRIPTION = "A library to manipulate gettext files (po and mo files)."
+HOMEPAGE = "https://bitbucket.org/izi/polib/wiki/Home"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=ca9b0a3e473327024be9af47bc41fb14"
+
+DEPENDS += "${PYTHON_PN}-native"
+
+PYPI_PACKAGE = "polib"
+
+inherit pypi
+
+FILES_${PN} += "${datadir}/polib"
+
+inherit native

recipes-python/python-polib-native/python-polib-native_1.1.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools
+require python-polib-native.inc
+
+SRC_URI[md5sum] = "eeedc0f1e8f2dbfbb6192772d02795e0"
+SRC_URI[sha256sum] = "fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a"

recipes-python/python-polib-native/python3-polib-native_1.1.0.bb

@@ -0,0 +1,5 @@
+inherit setuptools3
+require python-polib-native.inc
+
+SRC_URI[md5sum] = "eeedc0f1e8f2dbfbb6192772d02795e0"
+SRC_URI[sha256sum] = "fad87d13696127ffb27ea0882d6182f1a9cf8a5e2b37a587751166c51e5a332a"

recipes-sca-rules/sca-recipe-dennis-rules-native/sca-recipe-dennis-rules-native_1.0.bb

@@ -0,0 +1,18 @@
+SUMMARY = "Ruleset for dennis"
+DESCRIPTION = " Rules to configure how dennis is affecting the build"
+
+SRC_URI = "file://suppress \
+           file://fatal"
+
+LICENSE ?= "BSD-2-Clause"
+LIC_FILES_CHKSUM ?= "file://${COMMON_LICENSE_DIR}/BSD-2-Clause;md5=8bef8e6712b1be5aa76af1ebde9d6378"
+
+inherit native
+
+do_install() {
+    install -d "${D}${datadir}"
+    install "${WORKDIR}/fatal" "${D}${datadir}/dennis-recipe-fatal"
+    install "${WORKDIR}/suppress" "${D}${datadir}/dennis-recipe-suppress"
+}
+
+FILES_${PN} = "${datadir}/**"

recipes-sca/python-dennis-native/files/dennis.sca.description

@@ -0,0 +1,6 @@
+{
+    "languages": ["i18n"],
+    "buildspeed" : "fast",
+    "execspeed": "fast",
+    "quality": "good"
+}

recipes-sca/python-dennis-native/python-dennis-native.inc

@@ -0,0 +1,21 @@
+SUMMARY = "Dennis is a set of utilities for working with PO files to ease development and improve quality. "
+DESCRIPTION = "Dennis is a set of utilities for working with PO files to ease development and improve quality. "
+HOMEPAGE = "https://github.com/willkg/dennis"
+LICENSE = "BSD-3-Clause"
+LIC_FILES_CHKSUM = "file://LICENSE;md5=5a174f6c1c019edada0f5a18bed0ff0f"
+
+DEPENDS += "${PYTHON_PN}-native ${PYTHON_PN}-click-native ${PYTHON_PN}-polib-native"
+
+PYPI_PACKAGE = "dennis"
+
+inherit native
+inherit pypi
+
+SRC_URI += "file://dennis.sca.description"
+
+FILES_${PN} += "${datadir}"
+
+do_install_append() {
+    install -d ${D}${datadir}
+    install ${WORKDIR}/dennis.sca.description ${D}${datadir}
+}

recipes-sca/python-dennis-native/python-dennis-native_0.9.bb

@@ -0,0 +1,5 @@
+inherit setuptools
+require python-dennis-native.inc
+
+SRC_URI[md5sum] = "2636710455435f612e8595c659e2c746"
+SRC_URI[sha256sum] = "8c942dd5da7d03c65daebc069c5ee5c7f1374ac9b0c8c89c627caa66fe822604"

recipes-sca/python-dennis-native/python3-dennis-native_0.9.bb

@@ -0,0 +1,5 @@
+inherit setuptools3
+require python-dennis-native.inc
+
+SRC_URI[md5sum] = "2636710455435f612e8595c659e2c746"
+SRC_URI[sha256sum] = "8c942dd5da7d03c65daebc069c5ee5c7f1374ac9b0c8c89c627caa66fe822604"