diff --git a/README.md b/README.md index 97440647fc..5c22d04414 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,6 @@ The layer can check on a recipe-level or on an image-level. | safety | Check for known vulnerabilities in python-packages | https://github.com/pyupio/safety | | | | x | | | x | | | | | | | | | | x | | x | | | | scancode | Scan code for license information | https://github.com/nexB/scancode-toolkit | | | | x | | | | | | | | | | | | | | x | x | | | | secretlint | Tool to prevent committing credential | https://github.com/secretlint/secretlint | meta-oe, manual enable | | x | x | | | | | | | | | | | | | | x | x | | | -| semgrep | Pattern based linting for many formats and languages | https://github.com/returntocorp/semgrep | only avaiable for x86-64 hosts | | | x | x | x | x | | x | x | x | | | | | | | x | | x | | | setuptoolslint | Lint python-setup.py | https://github.com/johnnoone/setuptools-pylint | | | | x | | | x | | | | | | | | | | | | | x | | | shellcheck | Shell script linter | https://github.com/koalaman/shellcheck | | | x | x | | | | x | | | | | | | | | | | | x | x | | slick | Shell script linter | https://github.com/mcandre/slick | | | x | x | | | | x | | | | | | | | | | | | x | | @@ -359,7 +358,6 @@ each tool does have it's own benefits and flaws so don't be mad if you have 10k+ - [safety](docs/conf/module/safety.md) - [scancode](docs/conf/module/scancode.md) - [secretlint](docs/conf/module/secretlint.md) - - [semgrep](docs/conf/module/semgrep.md) - [setuptoolslint](docs/conf/module/setuptoolslint.md) - [shellcheck](docs/conf/module/shellcheck.md) - [slick](docs/conf/module/slick.md) diff --git a/classes/sca-blacklist.bbclass b/classes/sca-blacklist.bbclass index 6e3b148be9..52652218b5 100644 --- a/classes/sca-blacklist.bbclass +++ b/classes/sca-blacklist.bbclass @@ -73,7 +73,6 @@ SCA_BLACKLIST_rubycritic ?= "" SCA_BLACKLIST_safety ?= "" SCA_BLACKLIST_scancode ?= "" SCA_BLACKLIST_secretlint ?= "" -SCA_BLACKLIST_semgrep ?= "busybox linux-.* openssh" SCA_BLACKLIST_setuptoolslint ?= "" SCA_BLACKLIST_shellcheck ?= "" SCA_BLACKLIST_slick ?= "openssh" diff --git a/classes/sca-global.bbclass b/classes/sca-global.bbclass index 1073f4f181..9cf6a00b11 100644 --- a/classes/sca-global.bbclass +++ b/classes/sca-global.bbclass @@ -134,7 +134,6 @@ SCA_AVAILABLE_MODULES ?= "\ reuse \ revive \ safety \ - semgrep \ setuptoolslint \ shellcheck \ slick \ diff --git a/classes/sca-on-recipe.bbclass b/classes/sca-on-recipe.bbclass index c82d9ff733..2bd0df4e72 100755 --- a/classes/sca-on-recipe.bbclass +++ b/classes/sca-on-recipe.bbclass @@ -74,7 +74,6 @@ SCA_ENABLED_MODULES_RECIPE ?= "\ safety \ scancode \ secretlint \ - semgrep \ setuptoolslint \ shellcheck \ slick \ diff --git a/classes/sca-semgrep.bbclass b/classes/sca-semgrep.bbclass deleted file mode 100755 index 82328524f0..0000000000 --- a/classes/sca-semgrep.bbclass +++ /dev/null @@ -1,116 +0,0 @@ -## SPDX-License-Identifier: BSD-2-Clause -## Copyright (c) 2021, Konrad Weihmann - -## Add ids to suppress on a recipe level -SCA_SEMGREP_EXTRA_SUPPRESS ?= "" -## Add ids to lead to a fatal on a recipe level -SCA_SEMGREP_EXTRA_FATAL ?= "" - -SCA_SEMGREP_RULESETS ?= "\ - ${STAGING_DATADIR_NATIVE}/semgrep/semgrep-go \ -" - -SCA_SEMGREP_USER_RULES_RECIPES ?= "" - -SCA_RAW_RESULT_FILE[semgrep] = "json" - -inherit sca-conv-to-export -inherit sca-datamodel -inherit sca-global -inherit sca-helper -inherit sca-suppress -inherit sca-image-backtrack -inherit sca-tracefiles - -def do_sca_conv_semgrep(d): - import os - import json - - package_name = d.getVar("PN") - buildpath = d.getVar("SCA_SOURCES_DIR") - rule_prefixes = [os.path.relpath(x, d.getVar("TOPDIR")).lstrip("/").replace("/", ".") for x in clean_split(d, "SCA_SEMGREP_RULESETS")] - - items = [] - _suppress = sca_suppress_init(d, "SCA_SEMGREP_EXTRA_SUPPRESS", - d.expand("${STAGING_DATADIR_NATIVE}/semgrep-${SCA_MODE}-suppress")) - _findings = [] - - _severity_map = { - "WARNING": "warning", - "ERROR": "error" - } - - if os.path.exists(sca_raw_result_file(d, "semgrep")): - content = { "results": [] } - with open(sca_raw_result_file(d, "semgrep"), "r") as f: - try: - content = json.load(f) - except json.JSONDecodeError: - pass - for item in content["results"]: - try: - _id = item["check_id"] - for x in rule_prefixes: - _id = _id.replace(x, "", 1).lstrip(".") - g = sca_get_model_class(d, - PackageName=package_name, - Tool="semgrep", - BuildPath=buildpath, - File=item["path"], - Line=str(item["start"]["line"]), - Column=str(item["start"]["col"]), - Message=item["extra"]["message"], - ID=_id, - Severity=_severity_map.get(item["extra"]["severity"], "warning")) - if _suppress.Suppressed(g): - continue - if g.Scope not in clean_split(d, "SCA_SCOPE_FILTER"): - continue - if g.Severity in sca_allowed_warning_level(d): - _findings += sca_backtrack_findings(d, g) - except Exception as e: - sca_log_note(d, str(e)) - sca_add_model_class_list(d, _findings) - return sca_save_model_to_string(d) - -do_sca_conv_semgrep[vardepsexclude] += "TOPDIR" - -python do_sca_semgrep() { - import os - import subprocess - - _args = ["semgrep", "--json", "--quiet"] - cmd_output = "" - for ruleset in clean_split(d, "SCA_SEMGREP_RULESETS"): - _tmp = exec_wrap_check_output(d, _args + ["-c", ruleset], [d.getVar("SCA_SOURCES_DIR")], - default_val={"results": []}, - combine=exec_wrap_combine_json_subarray, key="results") - cmd_output = exec_wrap_combine_json_subarray(cmd_output, _tmp, key="results") - with open(sca_raw_result_file(d, "semgrep"), "w") as o: - o.write(cmd_output) -} - -python do_sca_semgrep_report() { - import os - ## Create data model - d.setVar("SCA_DATAMODEL_STORAGE", "{}/semgrep.dm".format(d.getVar("T"))) - dm_output = do_sca_conv_semgrep(d) - with open(d.getVar("SCA_DATAMODEL_STORAGE"), "w") as o: - o.write(dm_output) - - sca_task_aftermath(d, "semgrep", get_fatal_entries(d, "SCA_SEMGREP_EXTRA_FATAL", - d.expand("${STAGING_DATADIR_NATIVE}/semgrep-${SCA_MODE}-fatal"))) -} - -do_sca_semgrep[doc] = "Lint many files with semgrep" -do_sca_semgrep_report[doc] = "Report findings of do_sca_semgrep" -addtask do_sca_semgrep after do_configure before do_sca_tracefiles -addtask do_sca_semgrep_report after do_sca_tracefiles before do_sca_deploy - -DEPENDS += "\ - ${SCA_SEMGREP_USER_RULES_RECIPES} \ - python3-semgrep-native \ - sca-recipe-semgrep-rules-native \ - semgrep-go-native \ - semgrep-core-native \ -" diff --git a/docs/conf/examples/integrators_favorites.md b/docs/conf/examples/integrators_favorites.md index fc4fa51cd5..f88bd46d72 100644 --- a/docs/conf/examples/integrators_favorites.md +++ b/docs/conf/examples/integrators_favorites.md @@ -34,7 +34,6 @@ SCA_ENABLED_MODULES_IMAGE ?= "\ pkgqaenc \ scancode \ secretlint \ - semgrep \ shellcheck \ systemdlint \ " diff --git a/docs/conf/module/semgrep.md b/docs/conf/module/semgrep.md deleted file mode 100644 index 878151ca31..0000000000 --- a/docs/conf/module/semgrep.md +++ /dev/null @@ -1,79 +0,0 @@ -# Configuration for semgrep - -## Supported environments/languages - -* c -* c++ -* go -* html -* javascript -* json -* php -* python -* ruby -* yaml" - -## Configuration - -| var | purpose | type | default | -| ------------- |:-------------:| -----:| -----: -| SCA_BLACKLIST_semgrep | Blacklist filter for this tool | space-separated-list | "" -| SCA_SEMGREP_EXTRA_FATAL | Extra error-IDs leading to build termination when found | space-separated-list | "": -| SCA_SEMGREP_EXTRA_SUPPRESS | Extra error-IDs to be suppressed | space-separated-list | "" -| SCA_SEMGREP_RULESETS | Rule directories to use | comma separated regex list | see `sca-semgrep.bbclass` -| SCA_SEMGREP_USER_RULES_RECIPES | Additional recipes providing user rules | comma separated regex list | "" - -## Add your own rules - -You can add your own rules to semgrep. For the format description please see the [rule documentation](https://semgrep.dev/docs/writing-rules/rule-syntax/). -Also check out [the interactive playyground](https://semgrep.dev/learn). - -Resulting `yml`-files or complete folder have to be installed from `native` recipe into the sysroot. -Just add the recipe name to `SCA_SEMGREP_USER_RULES_RECIPES` and add the path in the sysroot where to find file/files/folder to `SCA_SEMGREP_RULESETS` (e.g. `SCA_SEMGREP_RULESETS_append = "${STAGING_DATADIR_NATIVE}/my-personal-semgrep/rules`) - -## Supports - -* [x] suppression of IDs -* [x] terminate build on fatal -* [x] run on recipe -* [ ] run on image -* [x] run with SCA-layer default settings (see SCA_AVAILABLE_MODULES) - -## Requires - -* [ ] requires online access - -## Known error-IDs - -__tbd__ - -## Checking scope - -* [ ] security -* [x] functional defects -* [ ] compliance -* [ ] style issues - -## Statistics - -* ⬛⬛⬜⬜⬜⬜⬜⬜⬜⬜ 02/10 Build Speed -* ⬛⬛⬛⬛⬛⬛⬛⬜⬜⬜ 07/10 Execution Speed -* ⬛⬛⬛⬛⬛⬛⬛⬛⬜⬜ 08/10 Quality - -## Score mapping - -### Error considered as security relevant - -* n.a. - -### Error considered as functional defect - -* semgrep.semgrep.* - -### Error considered as compliance issue - -* n.a. - -### Error considered as style issue - -* n.a. diff --git a/files/module_list.csv b/files/module_list.csv index b0b841366e..5c19f39528 100644 --- a/files/module_list.csv +++ b/files/module_list.csv @@ -70,7 +70,6 @@ ropgadget,Determine ROP exploitability in binaries,https://github.com/JonathanSa rubycritic,Ruby linter,https://github.com/whitesmith/rubycritic/,"meta-rubygems, manual enable",,,x,,,,,,,,,,,,,,x,,,x safety,Check for known vulnerabilities in python-packages,https://github.com/pyupio/safety,,,,x,,,x,,,,,,,,,,x,,x,, scancode,Scan code for license information,https://github.com/nexB/scancode-toolkit,,,,x,,,,,,,,,,,,,,x,x,, -semgrep,Pattern based linting for many formats and languages,https://github.com/returntocorp/semgrep,only avaiable for x86-64 hosts,,,x,x,x,x,,x,x,x,,,,,,,x,,x, setuptoolslint,Lint python-setup.py,https://github.com/johnnoone/setuptools-pylint,,,,x,,,x,,,,,,,,,,,,,x, secretlint,Tool to prevent committing credential,https://github.com/secretlint/secretlint,"meta-oe, manual enable",,x,x,,,,,,,,,,,,,,x,x,, shellcheck,Shell script linter,https://github.com/koalaman/shellcheck,,,x,x,,,,x,,,,,,,,,,,,x,x diff --git a/recipes-sca-rules/sca-recipe-semgrep-rules-native/files/fatal b/recipes-sca-rules/sca-recipe-semgrep-rules-native/files/fatal deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/recipes-sca-rules/sca-recipe-semgrep-rules-native/files/suppress b/recipes-sca-rules/sca-recipe-semgrep-rules-native/files/suppress deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/recipes-sca-rules/sca-recipe-semgrep-rules-native/sca-recipe-semgrep-rules-native_1.0.bb b/recipes-sca-rules/sca-recipe-semgrep-rules-native/sca-recipe-semgrep-rules-native_1.0.bb deleted file mode 100755 index 69137343a8..0000000000 --- a/recipes-sca-rules/sca-recipe-semgrep-rules-native/sca-recipe-semgrep-rules-native_1.0.bb +++ /dev/null @@ -1,19 +0,0 @@ -SUMMARY = "SCA ruleset for semgrep at recipes" -DESCRIPTION = "Rules to configure how semgrep is affecting the build" - -DEFAULT_PREFERENCE = "${SCA_DEFAULT_PREFERENCE}" -LICENSE = "BSD-2-Clause" -LIC_FILES_CHKSUM = "file://${SCA_LAYERDIR}/LICENSE;md5=a4a2bbea1db029f21b3a328c7a059172" - -SRC_URI = "file://fatal \ - file://suppress" - -inherit native - -do_install() { - install -d "${D}${datadir}" - install "${WORKDIR}/suppress" "${D}${datadir}/semgrep-recipe-suppress" - install "${WORKDIR}/fatal" "${D}${datadir}/semgrep-recipe-fatal" -} - -FILES:${PN} = "${datadir}" diff --git a/recipes-sca/python3-semgrep-native/files/semgrep.sca.description b/recipes-sca/python3-semgrep-native/files/semgrep.sca.description deleted file mode 100644 index 07f4fa7a94..0000000000 --- a/recipes-sca/python3-semgrep-native/files/semgrep.sca.description +++ /dev/null @@ -1,33 +0,0 @@ -{ - "buildspeed": 2, - "execspeed": 7, - "languages": [ - "c", - "c++", - "go", - "html", - "javascript", - "json", - "php", - "python", - "ruby", - "yaml" - ], - "uses": [ - "@python" - ], - "quality": 8, - "scope": [ - "functional" - ], - "score": { - "functional": [ - "semgrep.semgrep..*" - ] - }, - "test": { - "integration": [ - {"cmd": ["semgrep", "--help"], "returncode": 0} - ] - } -} \ No newline at end of file diff --git a/recipes-sca/python3-semgrep-native/python3-semgrep-native_0.85.0.bb b/recipes-sca/python3-semgrep-native/python3-semgrep-native_0.85.0.bb deleted file mode 100644 index 9d407d65d4..0000000000 --- a/recipes-sca/python3-semgrep-native/python3-semgrep-native_0.85.0.bb +++ /dev/null @@ -1,48 +0,0 @@ -SUMMARY = "Lightweight static analysis for many languages" -HOMEPAGE = "https://github.com/returntocorp/semgrep" - -DEFAULT_PREFERENCE = "${SCA_DEFAULT_PREFERENCE}" -LICENSE = "LGPL-2.1-only" -LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/LGPL-2.1-only;md5=1a6d268fd218675ffea8be556788b780" - -DEPENDS += "\ - python3-attrs-native \ - python3-click-native \ - python3-click-option-group-native \ - python3-colorama-native \ - python3-defusedxml-native \ - python3-jsonschema-native \ - python3-packaging-native \ - python3-peewee-native \ - python3-requests-native \ - python3-ruamelyaml-native \ - python3-ruamelyamlcli-native \ - python3-tqdm-native \ - python3-wcmatch-native \ - python3-wheel-native \ - semgrep-core-native \ - " - -SRC_URI[md5sum] = "3706dcc58d50c9632f7e571a79c6e777" -SRC_URI[sha256sum] = "2452f039d1885b3b32729999bef8959d84eb82512fabe7321b7fa7735e173771" - -PYPI_PACKAGE = "semgrep" - -inherit pypi -inherit sca-description -inherit setuptools3 -inherit native - -SCA_TOOL_DESCRIPTION = "semgrep" - -export SEMGREP_SKIP_BIN = "1" - -do_configure:prepend() { - # Remove the version pinning - sed -i "s#~=#>=#g" "${S}/setup.py" -} - -do_install:append() { - # remove test files - rm -rf ${D}${PYTHON_SITEPACKAGES_DIR}/tests -} diff --git a/recipes-sca/python3-semgrep-native/semgrep-core-native_0.85.0.bb b/recipes-sca/python3-semgrep-native/semgrep-core-native_0.85.0.bb deleted file mode 100755 index 4ecccec279..0000000000 --- a/recipes-sca/python3-semgrep-native/semgrep-core-native_0.85.0.bb +++ /dev/null @@ -1,31 +0,0 @@ -SUMMARY = "Lightweight static analysis for many languages (core)" -HOMEPAGE = "https://github.com/returntocorp/semgrep" - -COMPATIBLE_HOST = "^x86_64-linux" - -DEFAULT_PREFERENCE = "${SCA_DEFAULT_PREFERENCE}" -LICENSE = "LGPL-2.1-or-later" -LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/${LICENSE};md5=2a4f4fd2128ea2f65047ee63fbca9f68" - -SRC_URI:append = " https://github.com/returntocorp/semgrep/releases/download/v${PV}/semgrep-v${PV}-ubuntu-16.04.tgz;name=app" - -UPSTREAM_CHECK_URI = "https://github.com/returntocorp/semgrep/tags" -UPSTREAM_CHECK_REGEX = "releases/tag/v(?P\d+\.\d+\.\d+)" - -SRC_URI[app.sha256sum] = "b37c9b32d75d41eb88bab3f0b03258691fc022f70897ddf2278d4b55b27745b3" -SRC_URI[license.sha256sum] = "7c34d28e784b202aa4998f477fd0aa9773146952d7f6fa5971369fcdda59cf48" - -inherit native -INHIBIT_PACKAGE_DEBUG_SPLIT = "1" -INHIBIT_PACKAGE_STRIP = "1" -INHIBIT_SYSROOT_STRIP = "1" - -do_configure[noexec] = "1" -do_compile[noexec] = "1" - -do_install() { - install -d ${D}${bindir} - install -m 0755 ${WORKDIR}/semgrep-files/semgrep-core ${D}${bindir}/semgrep-core -} - -FILES:${PN} += "${bindir}" diff --git a/recipes-support/semgrep-rules/semgrep-go-native_git.bb b/recipes-support/semgrep-rules/semgrep-go-native_git.bb deleted file mode 100755 index 829257e4ec..0000000000 --- a/recipes-support/semgrep-rules/semgrep-go-native_git.bb +++ /dev/null @@ -1,24 +0,0 @@ -SUMMARY = "This repo holds patterns for finding odd Go code" -HOMEPAGE = "https://github.com/dgryski/semgrep-go.git" - -DEFAULT_PREFERENCE = "${SCA_DEFAULT_PREFERENCE}" -LICENSE = "MIT" -LIC_FILES_CHKSUM = "file://LICENSE;md5=cceb70c3f326568ccff13faee2f67113" - -SRC_URI = "git://github.com/dgryski/semgrep-go.git;branch=master;protocol=https" -SRCREV = "cf22ff421d48b255b6bbe9770226d23e253cbd9e" -UPSTREAM_CHECK_COMMITS = "1" - -S = "${WORKDIR}/git" - -inherit native - -do_configure[noexec] = "1" -do_compile[noexec] = "1" - -do_install() { - install -d ${D}${datadir}/semgrep/semgrep-go - install -m 0644 ${S}/*.yml ${D}${datadir}/semgrep/semgrep-go/ -} - -FILES:${PN} = "${datadir}" diff --git a/test/lang_standard_01.txt b/test/lang_standard_01.txt index 2d0e9ff274..2fbbacfe4e 100644 --- a/test/lang_standard_01.txt +++ b/test/lang_standard_01.txt @@ -24,7 +24,6 @@ rats reuse revive scancode -semgrep shellcheck slick sparse