export: allow multiple export formats at once

Closes #6207 Signed-off-by: Konrad Weihmann <[email protected]>
priv-kweihmann · Jun 30, 2021 · bed8af1 · bed8af1
1 parent 311026d
commit bed8af1
Show file tree

Hide file tree

Showing 5 changed files with 80 additions and 69 deletions.
diff --git a/classes/sca-conv-to-export.bbclass b/classes/sca-conv-to-export.bbclass
@@ -13,24 +13,26 @@ def sca_allowed_warning_level(d):
 def sca_conv_to_export(d, tool):
     import bb
     from bb.parse.parse_py import BBHandler
-    try:
-        item = d.getVar("SCA_EXPORT_FORMAT")
 
-        BBHandler.inherit("sca-conv-dm-{}".format(item), "sca-export", 1, d)
-        func = "sca_conv_dm_{}".format(item)
-        if func in locals().keys():
-            return locals()[func](d, tool)
-        elif func in globals().keys():
-            return globals()[func](d, tool)
-        else:
-            raise NotImplementedError()
-    except bb.parse.ParseError as e:
-        sca_log_note(d, str(e))
+    res = []
+    for item in clean_split(d, "SCA_EXPORT_FORMAT"):
+        res_file = sca_conv_export_get_deployname(d, tool, item)
+        try:
+            BBHandler.inherit("sca-conv-dm-{}".format(item), "sca-export", 1, d)
+            func = "sca_conv_dm_{}".format(item)
+            if func in locals().keys():
+                res.append((locals()[func](d, tool), res_file))
+            elif func in globals().keys():
+                res.append((globals()[func](d, tool), res_file))
+            else:
+                raise NotImplementedError()
+        except bb.parse.ParseError as e:
+            sca_log_note(d, str(e))
+    return res
 
-def sca_conv_export_get_deployname(d, tool):
-    _exportformat = d.getVar("SCA_EXPORT_FORMAT")
-    _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(d.getVar("SCA_EXPORT_FORMAT")))
-    return "sca_{}_{}.{}".format(_exportformat, tool, _exportsuffix)
+def sca_conv_export_get_deployname(d, tool, _format):
+    _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(_format))
+    return "sca_{}_{}.{}".format(_format, tool, _exportsuffix)
 
 def sca_conv_deploy(d, tool):
     import os
@@ -39,54 +41,55 @@ def sca_conv_deploy(d, tool):
 
     _dmsuffix = "dm"
     _rawsuffix = d.getVarFlag("SCA_RAW_RESULT_FILE", tool)
-    _exportformat = d.getVar("SCA_EXPORT_FORMAT")
-    _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(d.getVar("SCA_EXPORT_FORMAT")))
 
-    os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "raw"),
-                exist_ok=True)
-    os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "datamodel"),
-                exist_ok=True)
-    os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, d.getVar("SCA_EXPORT_FORMAT")),
-                exist_ok=True)
+    for _exportformat in clean_split(d, "SCA_EXPORT_FORMAT"):
+        _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(_exportformat))
+
+        os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "raw"),
+                    exist_ok=True)
+        os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "datamodel"),
+                    exist_ok=True)
+        os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, _exportformat),
+                    exist_ok=True)
+
+        if d.getVar("SCA_CLEAN_BEFORE_EXPORT") == "1":
+            import glob
+            raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
+                                tool,
+                                "raw",
+                                "{}-*.{}".format(d.getVar("PN"), _rawsuffix))
+            dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
+                                    tool,
+                                    "datamodel",
+                                    "{}-*.{}".format(d.getVar("PN"), _dmsuffix))
+            cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
+                                    tool,
+                                    _exportformat,
+                                    "{}-*.{}".format(d.getVar("PN"), _exportsuffix))
+            for item in [raw_target, dm_target, cs_target]:
+                try:
+                    subprocess.check_call(["/bin/sh", "-c", "rm -f {}".format(item)])
+                except:
+                    pass
 
-    if d.getVar("SCA_CLEAN_BEFORE_EXPORT") == "1":
-        import glob
         raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
-                              tool,
-                              "raw",
-                              "{}-*.{}".format(d.getVar("PN"), _rawsuffix))
+                                tool,
+                                "raw",
+                                "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _rawsuffix))
         dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
                                 tool,
                                 "datamodel",
-                                "{}-*.{}".format(d.getVar("PN"), _dmsuffix))
+                                "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _dmsuffix))
         cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
                                 tool,
                                 _exportformat,
-                                "{}-*.{}".format(d.getVar("PN"), _exportsuffix))
-        for item in [raw_target, dm_target, cs_target]:
-            try:
-                subprocess.check_call(["/bin/sh", "-c", "rm -f {}".format(item)])
-            except:
-                pass
-
-    raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
-                              tool,
-                              "raw",
-                              "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _rawsuffix))
-    dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
-                              tool,
-                              "datamodel",
-                              "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _dmsuffix))
-    cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"),
-                             tool,
-                             _exportformat,
-                             "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _exportsuffix))
-    src_raw = os.path.join(d.getVar("T"), "sca_raw_{}.{}".format(tool, _rawsuffix))
-    src_dm = os.path.join(d.getVar("T"), "{}.dm".format(tool))
-    src_conv = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool))
-    if os.path.exists(src_raw):
-        shutil.copy(src_raw, raw_target)
-    if os.path.exists(src_conv):
-        shutil.copy(src_conv, cs_target)
-    if os.path.exists(src_dm):
-        shutil.copy(src_dm, dm_target)
+                                "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _exportsuffix))
+        src_raw = os.path.join(d.getVar("T"), "sca_raw_{}.{}".format(tool, _rawsuffix))
+        src_dm = os.path.join(d.getVar("T"), "{}.dm".format(tool))
+        src_conv = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool, _exportformat))
+        if os.path.exists(src_raw):
+            shutil.copy(src_raw, raw_target)
+        if os.path.exists(src_conv):
+            shutil.copy(src_conv, cs_target)
+        if os.path.exists(src_dm):
+            shutil.copy(src_dm, dm_target)
diff --git a/classes/sca-global.bbclass b/classes/sca-global.bbclass
@@ -23,7 +23,14 @@ SCA_SPARE_DIRS ?= ""
 # lead SCA to be invoked
 SCA_SPARE_IGNORE_BBAPPEND ?= "0"
 
-## Valid values are 'checkstyle' or 'console' or 'diff' or 'stat' or 'sarif' or 'plain' or 'codeclimate'
+## Valid values are 
+## 'checkstyle'
+## 'codeclimate'
+## 'console'
+## 'diff'
+## 'plain'
+## 'sarif'
+## 'stat'
 SCA_EXPORT_FORMAT ?= "checkstyle"
 SCA_EXPORT_FORMAT_SUFFIX_checkstyle = "xml"
 SCA_EXPORT_FORMAT_SUFFIX_codeclimate = "json"

diff --git a/classes/sca-helper.bbclass b/classes/sca-helper.bbclass
@@ -249,14 +249,13 @@ def get_local_includes(path):
 
 def sca_task_aftermath(d, tool, fatals=None):
     ## Write to final export
-    result_file = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool))
-    conv = sca_conv_to_export(d, tool)
-    if isinstance(conv, bytes):
-        mode = "wb"
-    else:
-        mode = "w"
-    with open(result_file, mode) as o:
-        o.write(conv)
+    for item in sca_conv_to_export(d, tool):
+        if isinstance(item[0], bytes):
+            mode = "wb"
+        else:
+            mode = "w"
+        with open(os.path.join(d.getVar("T"), item[1]), mode) as o:
+            o.write(item[0])
 
     ## Evaluate
     _warnings = get_warnings_from_result(d)

diff --git a/docs/conf/global.md b/docs/conf/global.md
@@ -20,7 +20,7 @@ The behavior of the analysis can be controlled by several __bitbake__-variables
 | SCA_EXPORT_DIR | Directory where to store the results of analysis | path | \${DEPLOY_DIR_IMAGE}/sca
 | SCA_EXPORT_FINDING_DIR | The folder where to store the original source-files of findings | path | \${DEPLOY_DIR_IMAGE}/sca/sources/\${PN}/
 | SCA_EXPORT_FINDING_SRC | Do copy the source-files of any finding to deploy-dir. This proved to helpful when integrating into Jenkins. | string | "1"
-| SCA_EXPORT_FORMAT | Specifies the output format | string: checkstyle, console, diff, stat or sarif, sonarqube or plain| "checkstyle"
+| SCA_EXPORT_FORMAT | Specifies the output format(s) | space separated list | "checkstyle" (for valid formatter see `sca-global.bbclass`)
 | SCA_FILE_FILTER | List of glob-expression of file to skip for testing | space separated list | "tests/* test/* doc/* testsuite/* \*\*/tests/* \*\*/test/* \*\*/doc/* \*\*/testsuite/*"
 | SCA_FINDINGS_DIR | Local folder where to store sca findings | path | \${WORKDIR}/sca/
 | SCA_FORCE_RUN | Force running SCA every time, no matter if recipe code was changed  | string: "0" or "1" | "0"

diff --git a/docs/conf/output_formats.md b/docs/conf/output_formats.md
@@ -1,6 +1,6 @@
 # Output formats
 
-With the help of the variable __SCA_EXPORT_FORMAT__ you can specify the format of the output.
+With the help of the variable __SCA_EXPORT_FORMAT__ you can specify the format(s) of the output.
 Currently supported are
 
 * checkstyle
@@ -12,6 +12,8 @@ Currently supported are
 * sonarqube
 * stat
 
+to enable multiple formats separate the string with spaces
+
 ## Checkstyle format
 
 Currently used in version ```4.3``` of the checkstyle schema.