From bed8af17148eef78f2070d2dc9cc811503bdb294 Mon Sep 17 00:00:00 2001 From: Konrad Weihmann Date: Wed, 30 Jun 2021 22:08:13 +0200 Subject: [PATCH] export: allow multiple export formats at once Closes #6207 Signed-off-by: Konrad Weihmann --- classes/sca-conv-to-export.bbclass | 119 +++++++++++++++-------------- classes/sca-global.bbclass | 9 ++- classes/sca-helper.bbclass | 15 ++-- docs/conf/global.md | 2 +- docs/conf/output_formats.md | 4 +- 5 files changed, 80 insertions(+), 69 deletions(-) diff --git a/classes/sca-conv-to-export.bbclass b/classes/sca-conv-to-export.bbclass index a5d9c102c1..8eccc53dd1 100644 --- a/classes/sca-conv-to-export.bbclass +++ b/classes/sca-conv-to-export.bbclass @@ -13,24 +13,26 @@ def sca_allowed_warning_level(d): def sca_conv_to_export(d, tool): import bb from bb.parse.parse_py import BBHandler - try: - item = d.getVar("SCA_EXPORT_FORMAT") - BBHandler.inherit("sca-conv-dm-{}".format(item), "sca-export", 1, d) - func = "sca_conv_dm_{}".format(item) - if func in locals().keys(): - return locals()[func](d, tool) - elif func in globals().keys(): - return globals()[func](d, tool) - else: - raise NotImplementedError() - except bb.parse.ParseError as e: - sca_log_note(d, str(e)) + res = [] + for item in clean_split(d, "SCA_EXPORT_FORMAT"): + res_file = sca_conv_export_get_deployname(d, tool, item) + try: + BBHandler.inherit("sca-conv-dm-{}".format(item), "sca-export", 1, d) + func = "sca_conv_dm_{}".format(item) + if func in locals().keys(): + res.append((locals()[func](d, tool), res_file)) + elif func in globals().keys(): + res.append((globals()[func](d, tool), res_file)) + else: + raise NotImplementedError() + except bb.parse.ParseError as e: + sca_log_note(d, str(e)) + return res -def sca_conv_export_get_deployname(d, tool): - _exportformat = d.getVar("SCA_EXPORT_FORMAT") - _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(d.getVar("SCA_EXPORT_FORMAT"))) - return "sca_{}_{}.{}".format(_exportformat, tool, _exportsuffix) +def sca_conv_export_get_deployname(d, tool, _format): + _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(_format)) + return "sca_{}_{}.{}".format(_format, tool, _exportsuffix) def sca_conv_deploy(d, tool): import os @@ -39,54 +41,55 @@ def sca_conv_deploy(d, tool): _dmsuffix = "dm" _rawsuffix = d.getVarFlag("SCA_RAW_RESULT_FILE", tool) - _exportformat = d.getVar("SCA_EXPORT_FORMAT") - _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(d.getVar("SCA_EXPORT_FORMAT"))) - os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "raw"), - exist_ok=True) - os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "datamodel"), - exist_ok=True) - os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, d.getVar("SCA_EXPORT_FORMAT")), - exist_ok=True) + for _exportformat in clean_split(d, "SCA_EXPORT_FORMAT"): + _exportsuffix = d.getVar("SCA_EXPORT_FORMAT_SUFFIX_{}".format(_exportformat)) + + os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "raw"), + exist_ok=True) + os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "datamodel"), + exist_ok=True) + os.makedirs(os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, _exportformat), + exist_ok=True) + + if d.getVar("SCA_CLEAN_BEFORE_EXPORT") == "1": + import glob + raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), + tool, + "raw", + "{}-*.{}".format(d.getVar("PN"), _rawsuffix)) + dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), + tool, + "datamodel", + "{}-*.{}".format(d.getVar("PN"), _dmsuffix)) + cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), + tool, + _exportformat, + "{}-*.{}".format(d.getVar("PN"), _exportsuffix)) + for item in [raw_target, dm_target, cs_target]: + try: + subprocess.check_call(["/bin/sh", "-c", "rm -f {}".format(item)]) + except: + pass - if d.getVar("SCA_CLEAN_BEFORE_EXPORT") == "1": - import glob raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), - tool, - "raw", - "{}-*.{}".format(d.getVar("PN"), _rawsuffix)) + tool, + "raw", + "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _rawsuffix)) dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, "datamodel", - "{}-*.{}".format(d.getVar("PN"), _dmsuffix)) + "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _dmsuffix)) cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), tool, _exportformat, - "{}-*.{}".format(d.getVar("PN"), _exportsuffix)) - for item in [raw_target, dm_target, cs_target]: - try: - subprocess.check_call(["/bin/sh", "-c", "rm -f {}".format(item)]) - except: - pass - - raw_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), - tool, - "raw", - "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _rawsuffix)) - dm_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), - tool, - "datamodel", - "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _dmsuffix)) - cs_target = os.path.join(d.getVar("SCA_FINDINGS_DIR"), - tool, - _exportformat, - "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _exportsuffix)) - src_raw = os.path.join(d.getVar("T"), "sca_raw_{}.{}".format(tool, _rawsuffix)) - src_dm = os.path.join(d.getVar("T"), "{}.dm".format(tool)) - src_conv = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool)) - if os.path.exists(src_raw): - shutil.copy(src_raw, raw_target) - if os.path.exists(src_conv): - shutil.copy(src_conv, cs_target) - if os.path.exists(src_dm): - shutil.copy(src_dm, dm_target) + "{}-{}.{}".format(d.getVar("PN"), d.getVar("PV"), _exportsuffix)) + src_raw = os.path.join(d.getVar("T"), "sca_raw_{}.{}".format(tool, _rawsuffix)) + src_dm = os.path.join(d.getVar("T"), "{}.dm".format(tool)) + src_conv = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool, _exportformat)) + if os.path.exists(src_raw): + shutil.copy(src_raw, raw_target) + if os.path.exists(src_conv): + shutil.copy(src_conv, cs_target) + if os.path.exists(src_dm): + shutil.copy(src_dm, dm_target) diff --git a/classes/sca-global.bbclass b/classes/sca-global.bbclass index 90452f42bd..29fdea4ff9 100644 --- a/classes/sca-global.bbclass +++ b/classes/sca-global.bbclass @@ -23,7 +23,14 @@ SCA_SPARE_DIRS ?= "" # lead SCA to be invoked SCA_SPARE_IGNORE_BBAPPEND ?= "0" -## Valid values are 'checkstyle' or 'console' or 'diff' or 'stat' or 'sarif' or 'plain' or 'codeclimate' +## Valid values are +## 'checkstyle' +## 'codeclimate' +## 'console' +## 'diff' +## 'plain' +## 'sarif' +## 'stat' SCA_EXPORT_FORMAT ?= "checkstyle" SCA_EXPORT_FORMAT_SUFFIX_checkstyle = "xml" SCA_EXPORT_FORMAT_SUFFIX_codeclimate = "json" diff --git a/classes/sca-helper.bbclass b/classes/sca-helper.bbclass index fe5cb5137a..0e65223cbd 100755 --- a/classes/sca-helper.bbclass +++ b/classes/sca-helper.bbclass @@ -249,14 +249,13 @@ def get_local_includes(path): def sca_task_aftermath(d, tool, fatals=None): ## Write to final export - result_file = os.path.join(d.getVar("T"), sca_conv_export_get_deployname(d, tool)) - conv = sca_conv_to_export(d, tool) - if isinstance(conv, bytes): - mode = "wb" - else: - mode = "w" - with open(result_file, mode) as o: - o.write(conv) + for item in sca_conv_to_export(d, tool): + if isinstance(item[0], bytes): + mode = "wb" + else: + mode = "w" + with open(os.path.join(d.getVar("T"), item[1]), mode) as o: + o.write(item[0]) ## Evaluate _warnings = get_warnings_from_result(d) diff --git a/docs/conf/global.md b/docs/conf/global.md index 1d97d12d3d..66fe587193 100644 --- a/docs/conf/global.md +++ b/docs/conf/global.md @@ -20,7 +20,7 @@ The behavior of the analysis can be controlled by several __bitbake__-variables | SCA_EXPORT_DIR | Directory where to store the results of analysis | path | \${DEPLOY_DIR_IMAGE}/sca | SCA_EXPORT_FINDING_DIR | The folder where to store the original source-files of findings | path | \${DEPLOY_DIR_IMAGE}/sca/sources/\${PN}/ | SCA_EXPORT_FINDING_SRC | Do copy the source-files of any finding to deploy-dir. This proved to helpful when integrating into Jenkins. | string | "1" -| SCA_EXPORT_FORMAT | Specifies the output format | string: checkstyle, console, diff, stat or sarif, sonarqube or plain| "checkstyle" +| SCA_EXPORT_FORMAT | Specifies the output format(s) | space separated list | "checkstyle" (for valid formatter see `sca-global.bbclass`) | SCA_FILE_FILTER | List of glob-expression of file to skip for testing | space separated list | "tests/* test/* doc/* testsuite/* \*\*/tests/* \*\*/test/* \*\*/doc/* \*\*/testsuite/*" | SCA_FINDINGS_DIR | Local folder where to store sca findings | path | \${WORKDIR}/sca/ | SCA_FORCE_RUN | Force running SCA every time, no matter if recipe code was changed | string: "0" or "1" | "0" diff --git a/docs/conf/output_formats.md b/docs/conf/output_formats.md index 8fb30ada1d..876cbae64d 100644 --- a/docs/conf/output_formats.md +++ b/docs/conf/output_formats.md @@ -1,6 +1,6 @@ # Output formats -With the help of the variable __SCA_EXPORT_FORMAT__ you can specify the format of the output. +With the help of the variable __SCA_EXPORT_FORMAT__ you can specify the format(s) of the output. Currently supported are * checkstyle @@ -12,6 +12,8 @@ Currently supported are * sonarqube * stat +to enable multiple formats separate the string with spaces + ## Checkstyle format Currently used in version ```4.3``` of the checkstyle schema.