From e6bdcc410c12dc6c022d4b02fac6bca5f3293f2e Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Wed, 11 Oct 2023 08:26:53 -0400 Subject: [PATCH] feat: convert NVD CVE configuration options to the new NVD API Configuration options --- .../dependencycheck/taskdefs/Update.java | 290 +++++++++--------- .../java/org/owasp/dependencycheck/App.java | 31 +- .../org/owasp/dependencycheck/CliParser.java | 86 +++--- .../completion-for-dependency-check.sh | 13 +- .../org/owasp/dependencycheck/AppTest.java | 87 ------ .../agent/DependencyCheckScanAgent.java | 70 ++--- .../data/update/EngineVersionCheck.java | 10 +- .../data/update/KnownExploitedDataSource.java | 2 +- .../data/update/NvdApiDataSource.java | 14 +- .../update/nvd/{ => api}/DownloadTask.java | 2 +- .../maven/BaseDependencyCheckMojo.java | 141 +++++---- .../maven/BaseDependencyCheckMojoTest.java | 122 -------- .../owasp/dependencycheck/utils/Settings.java | 90 +----- .../dependencycheck/utils/DownloaderIT.java | 6 +- .../dependencycheck/utils/SettingsTest.java | 2 +- 15 files changed, 337 insertions(+), 629 deletions(-) rename core/src/main/java/org/owasp/dependencycheck/data/update/nvd/{ => api}/DownloadTask.java (98%) diff --git a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java index 3cf35c381a8..ed128d7c45c 100644 --- a/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java +++ b/ant/src/main/java/org/owasp/dependencycheck/taskdefs/Update.java @@ -17,14 +17,11 @@ */ package org.owasp.dependencycheck.taskdefs; -import java.util.Optional; - import org.apache.tools.ant.BuildException; import org.apache.tools.ant.Project; import org.owasp.dependencycheck.Engine; import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.utils.CveUrlParser; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.impl.StaticLoggerBinder; @@ -40,6 +37,31 @@ @SuppressWarnings("common-java:DuplicatedBlocks") public class Update extends Purge { + /** + * The NVD API Key. + */ + private String nvdApiKey; + /** + * The number of hours to wait before checking for new updates from the NVD. + */ + private Integer nvdValidForHours; + /** + * The NVD API Data Feed URL. + */ + private String nvdDatafeedUrl; + /** + * The username for basic auth to the NVD Data Feed. + */ + private String nvdUser; + /** + * The password for basic auth to the NVD Data Feed. + */ + private String nvdPassword; + /** + * The time in milliseconds to wait between downloading NVD API data. + */ + private int nvdApiDelay = 0; + /** * The Proxy Server. */ @@ -89,47 +111,136 @@ public class Update extends Purge { */ private String databasePassword; /** - * The URL for the modified NVD CVE JSON file. + * The number of hours to wait before re-checking hosted suppressions file + * for updates. */ - private String cveUrlModified; + private Integer hostedSuppressionsValidForHours; /** - * Base Data Mirror URL for CVE JSON files. + * Whether the hosted suppressions file will be updated regardless of the + * `autoupdate` settings. Defaults to false. */ - private String cveUrlBase; + private Boolean hostedSuppressionsForceUpdate; /** - * The wait time in milliseconds between downloads from the NVD. + * Whether the hosted suppressions file will be used. Defaults to true. */ - private String cveWaitTime; + private Boolean hostedSuppressionsEnabled; + /** - * The number of hours to wait before re-checking for updates. + * Construct a new UpdateTask. */ - private Integer cveValidForHours; + public Update() { + super(); + // Call this before Dependency Check Core starts logging anything - this way, all SLF4J messages from + // core end up coming through this tasks logger + StaticLoggerBinder.getSingleton().setTask(this); + } + /** - * The number of hours to wait before re-checking hosted suppressions file for updates. + * Get the value of nvdApiKey. + * + * @return the value of nvdApiKey */ - private Integer hostedSuppressionsValidForHours; + public String getNvdApiKey() { + return nvdApiKey; + } + /** - * Whether the hosted suppressions file will be updated regardless of the `autoupdate` settings. Defaults to false. + * Set the value of nvdApiKey. + * + * @param nvdApiKey new value of nvdApiKey */ - private Boolean hostedSuppressionsForceUpdate; + public void setNvdApiKey(String nvdApiKey) { + this.nvdApiKey = nvdApiKey; + } + /** - * Whether the hosted suppressions file will be used. Defaults to true. + * Get the value of nvdValidForHours. + * + * @return the value of nvdValidForHours */ - private Boolean hostedSuppressionsEnabled; + public int getNvdValidForHours() { + return nvdValidForHours; + } /** - * Specify the first year of NVD CVE data to download; default is 2002. + * Set the value of nvdValidForHours. + * + * @param nvdValidForHours new value of nvdValidForHours */ - private Integer cveStartYear; + public void setNvdValidForHours(int nvdValidForHours) { + this.nvdValidForHours = nvdValidForHours; + } /** - * Construct a new UpdateTask. + * Get the value of nvdDatafeedUrl. + * + * @return the value of nvdDatafeedUrl */ - public Update() { - super(); - // Call this before Dependency Check Core starts logging anything - this way, all SLF4J messages from - // core end up coming through this tasks logger - StaticLoggerBinder.getSingleton().setTask(this); + public String getNvdDatafeedUrl() { + return nvdDatafeedUrl; + } + + /** + * Set the value of nvdDatafeedUrl. + * + * @param nvdDatafeedUrl new value of nvdDatafeedUrl + */ + public void setNvdDatafeedUrl(String nvdDatafeedUrl) { + this.nvdDatafeedUrl = nvdDatafeedUrl; + } + + /** + * Get the value of nvdUser. + * + * @return the value of nvdUser + */ + public String getNvdUser() { + return nvdUser; + } + + /** + * Set the value of nvdUser. + * + * @param nvdUser new value of nvdUser + */ + public void setNvdUser(String nvdUser) { + this.nvdUser = nvdUser; + } + + /** + * Get the value of nvdPassword. + * + * @return the value of nvdPassword + */ + public String getNvdPassword() { + return nvdPassword; + } + + /** + * Set the value of nvdPassword. + * + * @param nvdPassword new value of nvdPassword + */ + public void setNvdPassword(String nvdPassword) { + this.nvdPassword = nvdPassword; + } + + /** + * Get the value of nvdApiDelay. + * + * @return the value of nvdApiDelay + */ + public int getNvdApiDelay() { + return nvdApiDelay; + } + + /** + * Set the value of nvdApiDelay. + * + * @param nvdApiDelay new value of nvdApiDelay + */ + public void setNvdApiDelay(int nvdApiDelay) { + this.nvdApiDelay = nvdApiDelay; } /** @@ -348,101 +459,6 @@ public void setDatabasePassword(String databasePassword) { this.databasePassword = databasePassword; } - /** - * Set the value of cveUrlModified. - * - * @param cveUrlModified new value of cveUrlModified - */ - public void setCveUrlModified(String cveUrlModified) { - this.cveUrlModified = cveUrlModified; - } - - /** - * Get the value of cveUrlModified. - * - * @return the value of cveUrlModified - */ - public String getCveUrlModified() { - return cveUrlModified; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveUrlBase() { - return cveUrlBase; - } - - /** - * Set the value of cveUrlBase. - * - * @param cveUrlBase new value of cveUrlBase - */ - public void setCveUrlBase(String cveUrlBase) { - this.cveUrlBase = cveUrlBase; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveWaitTime() { - return cveWaitTime; - } - - /** - * Set the value of cveWaitTime. - * - * @param cveWaitTime new value of cveWaitTime - */ - public void setCveWaitTime(String cveWaitTime) { - this.cveWaitTime = cveWaitTime; - } - - /** - * Get the value of cveValidForHours. - * - * @return the value of cveValidForHours - */ - public Integer getCveValidForHours() { - return cveValidForHours; - } - - /** - * Set the value of cveValidForHours. - * - * @param cveValidForHours new value of cveValidForHours - */ - public void setCveValidForHours(Integer cveValidForHours) { - this.cveValidForHours = cveValidForHours; - } - - /** - * Get the value of cveStartYear. - * - * @return the value of cveStartYear - */ - public Integer getCveStartYear() { - return cveStartYear; - } - - /** - * Set the value of cveStartYear. - * - * @param cveStartYear new value of cveStartYear - */ - public void setCveStartYear(Integer cveStartYear) { - if (cveStartYear != null && cveStartYear < 2002) { - log("Invalid Configuration: cveStartYear must be 2002 or greater", Project.MSG_ERR); - this.cveStartYear = 2002; - } else { - this.cveStartYear = cveStartYear; - } - } - /** * Get the value of hostedSuppressionsValidForHours. * @@ -455,7 +471,8 @@ public Integer getHostedSuppressionsValidForHours() { /** * Set the value of hostedSuppressionsValidForHours. * - * @param hostedSuppressionsValidForHours new value of hostedSuppressionsValidForHours + * @param hostedSuppressionsValidForHours new value of + * hostedSuppressionsValidForHours */ public void setHostedSuppressionsValidForHours(final Integer hostedSuppressionsValidForHours) { this.hostedSuppressionsValidForHours = hostedSuppressionsValidForHours; @@ -473,7 +490,8 @@ public Boolean isHostedSuppressionsForceUpdate() { /** * Set the value of hostedSuppressionsForceUpdate. * - * @param hostedSuppressionsForceUpdate new value of hostedSuppressionsForceUpdate + * @param hostedSuppressionsForceUpdate new value of + * hostedSuppressionsForceUpdate */ public void setHostedSuppressionsForceUpdate(final Boolean hostedSuppressionsForceUpdate) { this.hostedSuppressionsForceUpdate = hostedSuppressionsForceUpdate; @@ -487,6 +505,7 @@ public void setHostedSuppressionsForceUpdate(final Boolean hostedSuppressionsFor public Boolean isHostedSuppressionsEnabled() { return hostedSuppressionsEnabled; } + /** * Set the value of hostedSuppressionsEnabled. * @@ -550,28 +569,21 @@ protected void populateSettings() throws BuildException { getSettings().setStringIfNotEmpty(Settings.KEYS.DB_CONNECTION_STRING, connectionString); getSettings().setStringIfNotEmpty(Settings.KEYS.DB_USER, databaseUser); getSettings().setStringIfNotEmpty(Settings.KEYS.DB_PASSWORD, databasePassword); - - final String cveModifiedJson = Optional.ofNullable(cveUrlModified) - .filter(url -> !url.isEmpty()) - .orElseGet(this::getDefaultCveUrlModified); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); - getSettings().setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime); - getSettings().setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, cveStartYear); getSettings().setIntIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS, hostedSuppressionsValidForHours); getSettings().setBooleanIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_FORCEUPDATE, hostedSuppressionsForceUpdate); getSettings().setBooleanIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_ENABLED, hostedSuppressionsEnabled); - if (cveValidForHours != null) { - if (cveValidForHours >= 0) { - getSettings().setInt(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours); + + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey); + getSettings().setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, nvdApiDelay); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, nvdDatafeedUrl); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, nvdUser); + getSettings().setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, nvdPassword); + if (nvdValidForHours != null) { + if (nvdValidForHours >= 0) { + getSettings().setInt(Settings.KEYS.NVD_API_VALID_FOR_HOURS, nvdValidForHours); } else { - throw new BuildException("Invalid setting: `cpeValidForHours` must be 0 or greater"); + throw new BuildException("Invalid setting: `nvdValidForHours` must be 0 or greater"); } } } - - private String getDefaultCveUrlModified() { - return CveUrlParser.newInstance(getSettings()) - .getDefaultCveUrlModified(cveUrlBase); - } } diff --git a/cli/src/main/java/org/owasp/dependencycheck/App.java b/cli/src/main/java/org/owasp/dependencycheck/App.java index 56ff5cfd493..ab8ff38cef7 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/App.java +++ b/cli/src/main/java/org/owasp/dependencycheck/App.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Set; import org.apache.commons.cli.ParseException; @@ -34,7 +33,6 @@ import org.owasp.dependencycheck.data.update.exception.UpdateException; import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.ReportException; -import org.owasp.dependencycheck.utils.CveUrlParser; import org.owasp.dependencycheck.utils.InvalidSettingException; import org.owasp.dependencycheck.utils.Settings; import org.slf4j.Logger; @@ -477,10 +475,6 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getStringArgument(CliParser.ARGUMENT.CONNECTION_READ_TIMEOUT)); settings.setStringIfNotEmpty(Settings.KEYS.HINTS_FILE, cli.getStringArgument(CliParser.ARGUMENT.HINTS_FILE)); - settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, - cli.getIntegerValue(CliParser.ARGUMENT.CVE_VALID_FOR_HOURS)); - settings.setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, - cli.getIntegerValue(CliParser.ARGUMENT.CVE_START_YEAR)); settings.setArrayIfNotEmpty(Settings.KEYS.SUPPRESSION_FILE, cli.getStringArguments(CliParser.ARGUMENT.SUPPRESSION_FILES)); //File Type Analyzer Settings @@ -649,21 +643,13 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getStringArgument(CliParser.ARGUMENT.ADDITIONAL_ZIP_EXTENSIONS)); settings.setStringIfNotEmpty(Settings.KEYS.ANALYZER_ASSEMBLY_DOTNET_PATH, cli.getStringArgument(CliParser.ARGUMENT.PATH_TO_CORE)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, - cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, - cli.getStringArgument(CliParser.ARGUMENT.CVE_DOWNLOAD_WAIT_TIME)); - final String cveModifiedJson = Optional.ofNullable(cli.getStringArgument(CliParser.ARGUMENT.CVE_MODIFIED_URL)) - .filter(arg -> !arg.isEmpty()) - .orElseGet(() -> getDefaultCveUrlModified(cli)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, - cveModifiedJson); - - settings.setStringIfNotEmpty(Settings.KEYS.CVE_USER, - cli.getStringArgument(CliParser.ARGUMENT.CVE_USER)); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_PASSWORD, - cli.getStringArgument(CliParser.ARGUMENT.CVE_PASSWORD, Settings.KEYS.CVE_PASSWORD)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_KEY)); + settings.setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_DELAY)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_URL)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_USER)); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, cli.getStringArgument(CliParser.ARGUMENT.NVD_API_DATAFEED_PASSWORD)); + settings.setIntIfNotNull(Settings.KEYS.NVD_API_VALID_FOR_HOURS, cli.getIntegerValue(CliParser.ARGUMENT.NVD_API_VALID_FOR_HOURS)); settings.setStringIfNotNull(Settings.KEYS.HOSTED_SUPPRESSIONS_URL, cli.getStringArgument(CliParser.ARGUMENT.HOSTED_SUPPRESSIONS_URL)); @@ -675,11 +661,6 @@ protected void populateSettings(CliParser cli) throws InvalidSettingException { cli.getIntegerValue(CliParser.ARGUMENT.HOSTED_SUPPRESSIONS_VALID_FOR_HOURS)); } - private String getDefaultCveUrlModified(CliParser cli) { - return CveUrlParser.newInstance(settings) - .getDefaultCveUrlModified(cli.getStringArgument(CliParser.ARGUMENT.CVE_BASE_URL)); - } - //CSON: MethodLength /** * Creates a file appender and adds it to logback. diff --git a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java index 2a88f067330..eef5b28f068 100644 --- a/cli/src/main/java/org/owasp/dependencycheck/CliParser.java +++ b/cli/src/main/java/org/owasp/dependencycheck/CliParser.java @@ -114,26 +114,26 @@ private CommandLine parseArgs(String[] args) throws ParseException { */ private void validateArgs() throws FileNotFoundException, ParseException { if (isUpdateOnly() || isRunScan()) { - String value = line.getOptionValue(ARGUMENT.CVE_VALID_FOR_HOURS); + String value = line.getOptionValue(ARGUMENT.NVD_API_VALID_FOR_HOURS); if (value != null) { try { final int i = Integer.parseInt(value); if (i < 0) { - throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0."); + throw new ParseException("Invalid Setting: nvdValidForHours must be a number greater than or equal to 0."); } } catch (NumberFormatException ex) { - throw new ParseException("Invalid Setting: cveValidForHours must be a number greater than or equal to 0."); + throw new ParseException("Invalid Setting: nvdValidForHours must be a number greater than or equal to 0."); } } - value = line.getOptionValue(ARGUMENT.CVE_START_YEAR); + value = line.getOptionValue(ARGUMENT.NVD_API_DELAY); if (value != null) { try { final int i = Integer.parseInt(value); - if (i < 2002) { - throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + if (i < 0) { + throw new ParseException("Invalid Setting: nvdApiDelay must be a number greater than or equal to 0."); } } catch (NumberFormatException ex) { - throw new ParseException("Invalid Setting: cveStartYear must be a number greater than or equal to 2002."); + throw new ParseException("Invalid Setting: nvdApiDelay must be a number greater than or equal to 0."); } } } @@ -155,12 +155,6 @@ private void validateArgs() throws FileNotFoundException, ParseException { } } } - final String base = getStringArgument(ARGUMENT.CVE_BASE_URL); - final String modified = getStringArgument(ARGUMENT.CVE_MODIFIED_URL); - if ((base != null && modified == null) || (base == null && modified != null)) { - final String msg = "If one of the CVE URLs is specified they must all be specified; please add the missing CVE URL."; - throw new ParseException(msg); - } if (line.hasOption(ARGUMENT.SYM_LINK_DEPTH)) { try { final int i = Integer.parseInt(line.getOptionValue(ARGUMENT.SYM_LINK_DEPTH)); @@ -325,6 +319,7 @@ private void addStandardOptions(final Options options) { .addOptionGroup(newOptionGroup(newOptionWithArg(ARGUMENT.SUPPRESSION_FILES, "file", "The file path to the suppression XML file. This can be specified more then once to utilize multiple suppression files"))) .addOption(newOption(ARGUMENT.EXPERIMENTAL, "Enables the experimental analyzers.")) + .addOption(newOption(ARGUMENT.NVD_API_KEY, "The API Key to access the NVD API.")) .addOption(newOptionWithArg(ARGUMENT.FAIL_ON_CVSS, "score", "Specifies if the build should be failed if a CVSS score above a specified level is identified. The default is 11; " + "since the CVSS scores are 0-10, by default the build will never fail.")) @@ -344,16 +339,18 @@ private void addAdvancedOptions(final Options options) { options .addOption(newOption(ARGUMENT.UPDATE_ONLY, "Only update the local NVD data cache; no scan will be executed.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_BASE_URL, "url", - "Base URL for each year’s CVE files (json.gz), the %d will be replaced with the year.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_MODIFIED_URL, "url", - "URL for the modified CVE (json.gz).")) - .addOption(newOptionWithArg(ARGUMENT.CVE_DOWNLOAD_WAIT_TIME, "milliseconds", + + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DELAY, "milliseconds", "Time in milliseconds to wait between downloading from the NVD.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_USER, "user", - "Credentials for basic authentication to the CVE data.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_PASSWORD, "password", - "Credentials for basic authentication to the CVE data.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_URL, "url", + "The URL to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_USER, "user", + "Credentials for basic authentication to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_DATAFEED_PASSWORD, "password", + "Credentials for basic authentication to the NVD API Datafeed.")) + .addOption(newOptionWithArg(ARGUMENT.NVD_API_VALID_FOR_HOURS, "hours", + "The number of hours to wait before checking for new updates from the NVD.")) + .addOption(newOptionWithArg(ARGUMENT.PROXY_PORT, "port", "The proxy port to use when downloading resources.")) .addOption(newOptionWithArg(ARGUMENT.PROXY_SERVER, "server", @@ -429,10 +426,6 @@ private void addAdvancedOptions(final Options options) { "The path to the `yarn` executable.")) .addOption(newOptionWithArg(ARGUMENT.PATH_TO_PNPM, "path", "The path to the `pnpm` executable.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_VALID_FOR_HOURS, "hours", - "The number of hours to wait before checking for new updates from the NVD.")) - .addOption(newOptionWithArg(ARGUMENT.CVE_START_YEAR, "year", - "The first year to retrieve NVD CVE data for; default is 2002.")) .addOption(newOptionWithArg(ARGUMENT.RETIREJS_FILTERS, "pattern", "Specify Retire JS content filter used to exclude files from analysis based on their content; " + "most commonly used to exclude based on your applications own copyright line. This " @@ -1125,15 +1118,28 @@ public static class ARGUMENT { /** * The CLI argument name for setting the URL for the CVE Data Files. */ - public static final String CVE_MODIFIED_URL = "cveUrlModified"; + public static final String NVD_API_KEY = "nvdApiKey"; /** - * The CLI argument name for setting the URL for the CVE Data Files. + * The CLI argument name for setting the number of hours to wait before + * checking for new updates from the NVD. */ - public static final String CVE_BASE_URL = "cveUrlBase"; + public static final String NVD_API_VALID_FOR_HOURS = "nvdValidForHours"; /** - * The time in milliseconds to wait between downloading NVD CVE data. + * The CLI argument name for the NVD API Data Feed URL. */ - public static final String CVE_DOWNLOAD_WAIT_TIME = "cveDownloadWait"; + public static final String NVD_API_DATAFEED_URL = "nvdDatafeed"; + /** + * The username for basic auth to the CVE data. + */ + public static final String NVD_API_DATAFEED_USER = "nvdUser"; + /** + * The password for basic auth to the CVE data. + */ + public static final String NVD_API_DATAFEED_PASSWORD = "nvdPassword"; + /** + * The time in milliseconds to wait between downloading NVD API data. + */ + public static final String NVD_API_DELAY = "nvdApiDelay"; /** * The short CLI argument name for setting the location of the data * directory. @@ -1162,24 +1168,6 @@ public static class ARGUMENT { * The CLI argument name for setting the location of the hint file. */ public static final String HINTS_FILE = "hints"; - /** - * The CLI argument name for setting the number of hours to wait before - * checking for new updates from the NVD. - */ - public static final String CVE_VALID_FOR_HOURS = "cveValidForHours"; - /** - * The CLI argument name for setting the first year to retrieve NVD - * data. - */ - public static final String CVE_START_YEAR = "cveStartYear"; - /** - * The username for basic auth to the CVE data. - */ - public static final String CVE_USER = "cveUser"; - /** - * The password for basic auth to the CVE data. - */ - public static final String CVE_PASSWORD = "cvePassword"; /** * Disables the Jar Analyzer. */ diff --git a/cli/src/main/resources/completion-for-dependency-check.sh b/cli/src/main/resources/completion-for-dependency-check.sh index 76af9d41d9a..f2a4c0d903a 100755 --- a/cli/src/main/resources/completion-for-dependency-check.sh +++ b/cli/src/main/resources/completion-for-dependency-check.sh @@ -19,13 +19,6 @@ _odc_completions() --bundleAuditWorkingDirectory -c --connectiontimeout --connectionString - --cveUrlBase - --cveUrlModified - --cveValidForHours - --cveStartYear - --cveUser - --cvePassword - --cveDownloadWait -d --data --dbDriverName --dbDriverPath @@ -94,6 +87,12 @@ _odc_completions() --nodeAuditSkipDevDependencies --nodePackageSkipDevDependencies --nonProxyHosts + --nvdApiKey + --nvdDatafeed + --nvdUser + --nvdPassword + --nvdApiDelay + --nvdValidForHours -o --out --ossIndexPassword --ossIndexUsername diff --git a/cli/src/test/java/org/owasp/dependencycheck/AppTest.java b/cli/src/test/java/org/owasp/dependencycheck/AppTest.java index 4e8ca751be4..ea41abb957e 100644 --- a/cli/src/test/java/org/owasp/dependencycheck/AppTest.java +++ b/cli/src/test/java/org/owasp/dependencycheck/AppTest.java @@ -171,93 +171,6 @@ public void testPopulatingSuppressionSettingsWithMultipleFiles() throws Exceptio assertThat("Expected the suppression files to be set in the Settings with a separator", getSettings().getString(KEYS.SUPPRESSION_FILE), is("[\"first-file.xml\",\"another-file.xml\"]")); } - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModified() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModifiedWhenCveUrlModifiedIsEmpty() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz", "--cveUrlModified", ""}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model when arg is empty", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenValueIsExplicitelySet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz", "--cveUrlModified", "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to the specified value", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenUnknownValueIsSet() throws Exception { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - final Settings settings = getSettings(); - final App app = new App(settings); - - String[] args = {"--cveUrlBase", "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"}; - final CliParser parser = new CliParser(settings); - parser.parse(args); - - // When - app.populateSettings(parser); - - // Then - String output = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must not be set when name is not the same as from the nvd datasource", output, is(expectedOutput)); - } private boolean testBooleanProperties(String[] args, Map expected) throws URISyntaxException, FileNotFoundException, ParseException, InvalidSettingException { this.reloadSettings(); diff --git a/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java b/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java index c45637b6b4d..25ff8e08f20 100644 --- a/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java +++ b/core/src/main/java/org/owasp/dependencycheck/agent/DependencyCheckScanAgent.java @@ -107,6 +107,11 @@ public class DependencyCheckScanAgent { * recommended that this be turned to false. Default is true. */ private boolean autoUpdate = true; + /** + * The NVD API key. + */ + private String nvdApiKey; + /** * Sets whether the data directory should be updated without performing a * scan. Default is false. @@ -208,14 +213,6 @@ public class DependencyCheckScanAgent { * comma-separated list of file extensions to treat like ZIP files. */ private String zipExtensions; - /** - * The URL for the modified NVD CVE JSON. - */ - private String cveUrlModified; - /** - * The base URL for the NVD CVE JSON data feeds. - */ - private String cveUrlBase; /** * The path to dotnet core for .NET assembly analysis. */ @@ -250,6 +247,23 @@ public String getApplicationName() { public void setApplicationName(String applicationName) { this.applicationName = applicationName; } + /** + * Get the value of nvdApiKey + * + * @return the value of nvdApiKey + */ + public String getNvdApiKey() { + return nvdApiKey; + } + + /** + * Set the value of nvdApiKey + * + * @param nvdApiKey new value of nvdApiKey + */ + public void setNvdApiKey(String nvdApiKey) { + this.nvdApiKey = nvdApiKey; + } /** * Returns a list of pre-determined dependencies. @@ -801,42 +815,6 @@ public void setZipExtensions(String zipExtensions) { this.zipExtensions = zipExtensions; } - /** - * Get the value of cveUrlModified. - * - * @return the value of cveUrlModified - */ - public String getCveUrlModified() { - return cveUrlModified; - } - - /** - * Set the value of cveUrlModified. - * - * @param cveUrlModified new value of cveUrlModified - */ - public void setCveUrlModified(String cveUrlModified) { - this.cveUrlModified = cveUrlModified; - } - - /** - * Get the value of cveUrlBase. - * - * @return the value of cveUrlBase - */ - public String getCveUrlBase() { - return cveUrlBase; - } - - /** - * Set the value of cveUrlBase. - * - * @param cveUrlBase new value of cveUrlBase - */ - public void setCveUrlBase(String cveUrlBase) { - this.cveUrlBase = cveUrlBase; - } - /** * Get the value of pathToCore. * @@ -970,8 +948,8 @@ private void populateSettings() { settings.setStringIfNotEmpty(Settings.KEYS.DB_USER, databaseUser); settings.setStringIfNotEmpty(Settings.KEYS.DB_PASSWORD, databasePassword); settings.setStringIfNotEmpty(Settings.KEYS.ADDITIONAL_ZIP_EXTENSIONS, zipExtensions); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveUrlModified); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); + + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey); settings.setStringIfNotEmpty(Settings.KEYS.ANALYZER_ASSEMBLY_DOTNET_PATH, pathToCore); } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java index c81d8c7a740..4fffe1a5373 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/EngineVersionCheck.java @@ -123,14 +123,14 @@ public boolean update(Engine engine) throws UpdateException { final CveDB db = engine.getDatabase(); final boolean autoupdate = settings.getBoolean(Settings.KEYS.AUTO_UPDATE, true); final boolean enabled = settings.getBoolean(Settings.KEYS.UPDATE_VERSION_CHECK_ENABLED, true); - final String original = settings.getString(Settings.KEYS.CVE_ORIGINAL_JSON); - final String current = settings.getString(Settings.KEYS.CVE_MODIFIED_JSON); + final String datafeed = settings.getString(Settings.KEYS.NVD_API_DATAFEED_URL); /* * Only update if auto-update is enabled, the engine check is - * enabled, and the NVD CVE URLs have not been modified (i.e. the - * user has not configured them to point to an internal source). + * enabled, and the NVD DataFeed is being used (i.e. the user + * is likely on a private network). This check is not really needed + * so we are okay skipping it. */ - if (enabled && autoupdate && original != null && original.equals(current)) { + if (enabled && autoupdate && datafeed != null) { LOGGER.debug("Begin Engine Version Check"); final DatabaseProperties properties = db.getDatabaseProperties(); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java index 459af715d94..dea02961c39 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java @@ -98,7 +98,7 @@ public boolean update(Engine engine) throws UpdateException { @Override public boolean purge(Engine engine) { - //do nothing - covered by the NvdCveUpdater data source. + //do nothing - covered by the NvdApiDataSource. return true; } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java index a525c3f690f..c70dc22140a 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/NvdApiDataSource.java @@ -47,7 +47,7 @@ import org.owasp.dependencycheck.data.nvdcve.DatabaseException; import org.owasp.dependencycheck.data.nvdcve.DatabaseProperties; import org.owasp.dependencycheck.data.update.exception.UpdateException; -import org.owasp.dependencycheck.data.update.nvd.DownloadTask; +import org.owasp.dependencycheck.data.update.nvd.api.DownloadTask; import org.owasp.dependencycheck.data.update.nvd.api.NvdApiProcessor; import org.owasp.dependencycheck.utils.DateUtil; import org.owasp.dependencycheck.utils.DownloadFailedException; @@ -266,6 +266,18 @@ private boolean processApi() throws UpdateException { .withDelay(3000) .withThreadCount(4); } + long delay = 0; + try { + delay = settings.getLong(Settings.KEYS.NVD_API_DELAY); + } catch (InvalidSettingException ex) { + LOGGER.debug("Invalid setting `NVD_API_DELAY`?"); + } + if (delay > 0) { + builder.withDelay(delay); + } + + //TODO consider using CVE_CPE_STARTS_WITH_FILTER + ExecutorService processingExecutorService = null; try { processingExecutorService = Executors.newFixedThreadPool(PROCESSING_THREAD_POOL_SIZE); diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java similarity index 98% rename from core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java rename to core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java index 7bcc8fb1664..1eb00fb95d7 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/DownloadTask.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/nvd/api/DownloadTask.java @@ -15,7 +15,7 @@ * * Copyright (c) 2013 Jeremy Long. All Rights Reserved. */ -package org.owasp.dependencycheck.data.update.nvd; +package org.owasp.dependencycheck.data.update.nvd.api; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java index 4e8816bf05b..d43d876901d 100644 --- a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java +++ b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java @@ -66,7 +66,6 @@ import org.owasp.dependencycheck.exception.ExceptionCollection; import org.owasp.dependencycheck.exception.ReportException; import org.owasp.dependencycheck.utils.Checksum; -import org.owasp.dependencycheck.utils.CveUrlParser; import org.owasp.dependencycheck.utils.Filter; import org.owasp.dependencycheck.utils.Settings; import org.sonatype.plexus.components.sec.dispatcher.DefaultSecDispatcher; @@ -236,7 +235,7 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma @Deprecated private boolean failBuildOnAnyVulnerability = false; /** - * Sets whether auto-updating of the NVD CVE/CPE data is enabled. It is not + * Sets whether auto-updating of the NVD CVE data is enabled. It is not * recommended that this be turned to false. Default is true. */ @SuppressWarnings("CanBeFinal") @@ -816,13 +815,6 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma @SuppressWarnings("CanBeFinal") @Parameter(property = "databaseDriverPath") private String databaseDriverPath; - /** - * The server id in the settings.xml; used to retrieve encrypted passwords - * from the settings.xml. - */ - @SuppressWarnings("CanBeFinal") - @Parameter(property = "serverId") - private String serverId; /** * A reference to the settings.xml settings. */ @@ -912,56 +904,69 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma @SuppressWarnings("CanBeFinal") @Parameter(property = "dbFilename") private String dbFilename; - /** - * Data Mirror URL for CVE 1.2. + * The server id in the settings.xml; used to retrieve encrypted passwords + * from the settings.xml. This is used for the database username and + * password. */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveUrlModified") - private String cveUrlModified; + @Parameter(property = "nvdApiKey") + private String serverId; /** - * Base Data Mirror URL for CVE 1.2. + * The NVD API Key. */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveUrlBase") - private String cveUrlBase; + @Parameter(property = "nvdApiKey") + private String nvdApiKey; + /** - * The wait timeout between downloading from the NVD. + * The server id in the settings.xml; used to retrieve encrypted API Key + * from the settings.xml for the NVD API Key. Note that the password is used + * as the API Key */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveWaitTime") - private String cveWaitTime; + @Parameter(property = "nvdApiServerId") + private String nvdApiServerId; + /** - * The username to use when connecting to the CVE-URL. + * The number of hours to wait before checking for new updates from the NVD. */ - @Parameter(property = "cveUser") - private String cveUser; + @SuppressWarnings("CanBeFinal") + @Parameter(property = "nvdValidForHours") + private Integer nvdValidForHours; + /** - * The password to authenticate to the CVE-URL. + * The NVD API Data Feed URL. */ - @Parameter(property = "cvePassword") - private String cvePassword; + @SuppressWarnings("CanBeFinal") + @Parameter(property = "nvdDatafeedUrl") + private String nvdDatafeedUrl; + /** * The server id in the settings.xml; used to retrieve encrypted passwords - * from the settings.xml for cve-URLs. + * from the settings.xml for the NVD Data Feed. */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveServerId") - private String cveServerId; + @Parameter(property = "nvdDatafeedServerId") + private String nvdDatafeedServerId; /** - * Optionally skip excessive CVE update checks for a designated duration in - * hours. + * The username for basic auth to the NVD Data Feed. */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveValidForHours") - private Integer cveValidForHours; - + @Parameter(property = "nvdUser") + private String nvdUser; + /** + * The password for basic auth to the NVD Data Feed. + */ + @SuppressWarnings("CanBeFinal") + @Parameter(property = "nvdPassword") + private String nvdPassword; /** - * Specify the first year of NVD CVE data to download; default is 2002. + * The time in milliseconds to wait between downloading NVD API data. */ @SuppressWarnings("CanBeFinal") - @Parameter(property = "cveStartYear") - private Integer cveStartYear; + @Parameter(property = "nvdApiDelay") + private Integer nvdApiDelay; /** * The path to dotnet core. @@ -2313,27 +2318,25 @@ protected void populateSettings() { settings.setStringIfNotEmpty(Settings.KEYS.DATA_DIRECTORY, dataDirectory); settings.setStringIfNotEmpty(Settings.KEYS.DB_FILE_NAME, dbFilename); - final String cveModifiedJson = Optional.ofNullable(cveUrlModified) - .filter(arg -> !arg.isEmpty()) - .orElseGet(this::getDefaultCveUrlModified); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_MODIFIED_JSON, cveModifiedJson); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_BASE_JSON, cveUrlBase); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_DOWNLOAD_WAIT_TIME, cveWaitTime); - settings.setIntIfNotNull(Settings.KEYS.CVE_CHECK_VALID_FOR_HOURS, cveValidForHours); - if (cveStartYear != null && cveStartYear < 2002) { - getLog().warn("Invalid configuration: cveStartYear must be 2002 or greater"); - cveStartYear = 2002; + settings.setIntIfNotNull(Settings.KEYS.NVD_API_DELAY, nvdApiDelay); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_URL, nvdDatafeedUrl); + settings.setIntIfNotNull(Settings.KEYS.NVD_API_VALID_FOR_HOURS, nvdValidForHours); + + if (nvdApiKey == null && nvdApiServerId != null) { + configureServerCredentialsApiKey(nvdApiServerId, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + } else { + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_KEY, nvdApiKey); + } + if (nvdUser == null && nvdPassword == null && nvdDatafeedServerId != null) { + configureServerCredentials(nvdDatafeedServerId, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_PASSWORD); + } else { + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_USER, nvdUser); + settings.setStringIfNotEmpty(Settings.KEYS.NVD_API_DATAFEED_PASSWORD, nvdPassword); } - settings.setIntIfNotNull(Settings.KEYS.CVE_START_YEAR, cveStartYear); + settings.setBooleanIfNotNull(Settings.KEYS.PRETTY_PRINT, prettyPrint); artifactScopeExcluded = new ArtifactScopeExcluded(skipTestScope, skipProvidedScope, skipSystemScope, skipRuntimeScope); artifactTypeExcluded = new ArtifactTypeExcluded(skipArtifactType); - if (cveUser == null && cvePassword == null && cveServerId != null) { - configureServerCredentials(cveServerId, Settings.KEYS.CVE_USER, Settings.KEYS.CVE_PASSWORD); - } else { - settings.setStringIfNotEmpty(Settings.KEYS.CVE_USER, cveUser); - settings.setStringIfNotEmpty(Settings.KEYS.CVE_PASSWORD, cvePassword); - } if (suppressionFileUser == null && suppressionFilePassword == null && suppressionFileServerId != null) { configureServerCredentials(suppressionFileServerId, Settings.KEYS.SUPPRESSION_FILE_USER, Settings.KEYS.SUPPRESSION_FILE_PASSWORD); } else { @@ -2374,6 +2377,32 @@ private void configureServerCredentials(String serverId, String userSettingKey, } } + /** + * Retrieves the server credentials from the settings.xml, decrypts the + * password, and places the values into the settings under the given key + * names. This is used to retrieve an encrypted password as an API key. + * + * @param serverId the server id + * @param userSettingKey the property name for the username + * @param passwordSettingKey the property name for the password + */ + private void configureServerCredentialsApiKey(String serverId, String apiKeySetting) { + if (serverId != null) { + final Server server = settingsXml.getServer(serverId); + if (server != null) { + String password = null; + try { + password = decryptPasswordFromSettings(server.getPassword()); + } catch (SecDispatcherException ex) { + password = handleSecDispatcherException("server", serverId, server.getPassword(), ex); + } + settings.setStringIfNotEmpty(apiKeySetting, password); + } else { + getLog().error(String.format("Server '%s' not found in the settings.xml file", serverId)); + } + } + } + /** * Decrypts a password from the Maven settings if it needs to be decrypted. * If it's not encrypted the input password will be returned unchanged. @@ -2661,11 +2690,6 @@ protected void showSummary(MavenProject mp, Dependency[] dependencies) { } } - private String getDefaultCveUrlModified() { - return CveUrlParser.newInstance(getSettings()) - .getDefaultCveUrlModified(cveUrlBase); - } - // //CSOFF: ParameterNumber private ExceptionCollection scanDependencyNode(DependencyNode dependencyNode, DependencyNode root, @@ -2931,5 +2955,6 @@ private ExceptionCollection processPomArtifact(File artifactFile, DependencyNode } return exCol; } + } //CSON: FileLength diff --git a/maven/src/test/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojoTest.java b/maven/src/test/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojoTest.java index ca13bcb0126..99826ca13db 100644 --- a/maven/src/test/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojoTest.java +++ b/maven/src/test/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojoTest.java @@ -18,7 +18,6 @@ package org.owasp.dependencycheck.maven; import java.io.File; -import java.lang.reflect.Field; import java.net.URISyntaxException; import java.util.HashSet; import java.util.Locale; @@ -35,8 +34,6 @@ import org.apache.maven.plugin.testing.stubs.ArtifactStub; import org.apache.maven.project.MavenProject; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -237,123 +234,4 @@ protected ExceptionCollection scanPlugins(Engine engine, ExceptionCollection exC } } - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModified() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - BaseDependencyCheckMojo instance = new BaseDependencyCheckMojoImpl(); - - Field cveUrlModified = instance.getClass().getSuperclass().getDeclaredField("cveUrlModified"); - cveUrlModified.setAccessible(true); - cveUrlModified.set(instance, null); - - Field cveUrlBase = instance.getClass().getSuperclass().getDeclaredField("cveUrlBase"); - cveUrlBase.setAccessible(true); - cveUrlBase.set(instance, "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"); - - org.apache.maven.settings.Settings mavenSettings = new org.apache.maven.settings.Settings(); - Field mavenSettingsField = instance.getClass().getSuperclass().getDeclaredField("mavenSettings"); - mavenSettingsField.setAccessible(true); - mavenSettingsField.set(instance, mavenSettings); - - // When - instance.populateSettings(); - - // Then - String output = instance.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldSetDefaultValueToCveUrlModifiedWhenCveUrlModifiedIsEmpty() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - BaseDependencyCheckMojo instance = new BaseDependencyCheckMojoImpl(); - - Field cveUrlModified = instance.getClass().getSuperclass().getDeclaredField("cveUrlModified"); - cveUrlModified.setAccessible(true); - cveUrlModified.set(instance, ""); - - Field cveUrlBase = instance.getClass().getSuperclass().getDeclaredField("cveUrlBase"); - cveUrlBase.setAccessible(true); - cveUrlBase.set(instance, "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz"); - - org.apache.maven.settings.Settings mavenSettings = new org.apache.maven.settings.Settings(); - Field mavenSettingsField = instance.getClass().getSuperclass().getDeclaredField("mavenSettings"); - mavenSettingsField.setAccessible(true); - mavenSettingsField.set(instance, mavenSettings); - - // When - instance.populateSettings(); - - // Then - String output = instance.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to a default of the same model when arg is empty", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenValueIsExplicitelySet() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - BaseDependencyCheckMojo instance = new BaseDependencyCheckMojoImpl(); - - Field cveUrlModified = instance.getClass().getSuperclass().getDeclaredField("cveUrlModified"); - cveUrlModified.setAccessible(true); - cveUrlModified.set(instance, "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"); - - Field cveUrlBase = instance.getClass().getSuperclass().getDeclaredField("cveUrlBase"); - cveUrlBase.setAccessible(true); - cveUrlBase.set(instance, "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"); - - org.apache.maven.settings.Settings mavenSettings = new org.apache.maven.settings.Settings(); - Field mavenSettingsField = instance.getClass().getSuperclass().getDeclaredField("mavenSettings"); - mavenSettingsField.setAccessible(true); - mavenSettingsField.set(instance, mavenSettings); - - // When - instance.populateSettings(); - - // Then - String output = instance.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://another-custom-mirror-of-nvd/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must be set to the specified value", output, is(expectedOutput)); - } - - @Test - public void testPopulateSettingsShouldNotSetDefaultValueToCveUrlModifiedWhenUnknownValueIsSet() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException { - // Given - System.clearProperty(Settings.KEYS.CVE_MODIFIED_JSON); - System.clearProperty(Settings.KEYS.CVE_BASE_JSON); - - BaseDependencyCheckMojo instance = new BaseDependencyCheckMojoImpl(); - - Field cveUrlModified = instance.getClass().getSuperclass().getDeclaredField("cveUrlModified"); - cveUrlModified.setAccessible(true); - cveUrlModified.set(instance, null); - - Field cveUrlBase = instance.getClass().getSuperclass().getDeclaredField("cveUrlBase"); - cveUrlBase.setAccessible(true); - cveUrlBase.set(instance, "https://my-custom-mirror-of-nvd/feeds/json/cve/1.1/some-unusual-file-name-%d.json.gz"); - - org.apache.maven.settings.Settings mavenSettings = new org.apache.maven.settings.Settings(); - Field mavenSettingsField = instance.getClass().getSuperclass().getDeclaredField("mavenSettings"); - mavenSettingsField.setAccessible(true); - mavenSettingsField.set(instance, mavenSettings); - - // When - instance.populateSettings(); - - // Then - String output = instance.getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON); - String expectedOutput = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-modified.json.gz"; - assertThat("cveUrlModified must not be set when name is not the same as from the nvd datasource", output, is(expectedOutput)); - } } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java index c85c7170ed6..891d813e809 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Settings.java @@ -162,10 +162,15 @@ public static final class KEYS { */ public static final String CVE_CPE_STARTS_WITH_FILTER = "cve.cpe.startswith.filter"; + //TODO NVD Begin New /** * API Key for the NVD API. */ public static final String NVD_API_KEY = "nvd.api.key"; + /** + * The delay between requests for the NVD API. + */ + public static final String NVD_API_DELAY = "nvd.api.key"; /** * The properties key to control the skipping of the check for NVD * updates. @@ -192,94 +197,11 @@ public static final class KEYS { * The starting year for the NVD CVE Data feed cache. */ public static final String NVD_API_DATAFEED_START_YEAR = "nvd.api.datafeed.startyear"; - - //TODO DELETE BEGIN - /** - * The properties key for the URL to retrieve the recently modified and - * added CVE entries (last 8 days) using the JSON data feeds. - */ - public static final String CVE_MODIFIED_JSON = "cve.url.modified"; - /** - * The properties key for the default filename of the CVE modified URL. - */ - static final String CVE_MODIFIED_DEFAULT_FILENAME = "cve.url.modified.defaultFilename"; - /** - * The properties key for the original/modified URL to retrieve the - * recently modified and added CVE entries (last 8 days). Note, this is - * only used to compare against CVE_MODIFIED_JSON. - */ - public static final String CVE_ORIGINAL_JSON = "cve.url.original"; - /** - * The properties key for the URL to retrieve the recently modified and - * added CVE entries (last 8 days) using the JSON data feeds. - */ - public static final String CVE_BASE_JSON = "cve.url.base"; - /** - * The properties key for the default filename of the CVE base URL. - */ - static final String CVE_BASE_DEFAULT_FILENAME = "cve.url.base.defaultFilename"; - /** - * The username to use when connecting to the CVE-URL. - */ - public static final String CVE_USER = "cve.user"; - /** - * The password to authenticate to the CVE-URL. - */ - public static final String CVE_PASSWORD = "cve.password"; - /** - * The properties key for the URL to retrieve the recently modified and - * added CVE entries (last 8 days). - */ - public static final String CVE_MODIFIED_VALID_FOR_DAYS = "cve.url.modified.validfordays"; - /** - * The properties key to control the skipping of the check for CVE - * updates. - */ - public static final String CVE_CHECK_VALID_FOR_HOURS = "cve.check.validforhours"; - /** - * The properties key for the telling us how many cve.url.* URLs exists. - * This is used in combination with CVE_BASE_URL to be able to retrieve - * the URLs for all of the files that make up the NVD CVE listing. - */ - public static final String CVE_START_YEAR = "cve.startyear"; - /** - * A configurable sleep time between downloading the NVD CVE data. The - * time is in milliseconds. - */ - public static final String CVE_DOWNLOAD_WAIT_TIME = "cve.download.waittime"; - - /** - * The properties key for the telling us how many nvd.url.* URLs exists. - * This is used in combination with NVD_CACHE_URL to be able to retrieve - * the URLs for all of the files that make up the NVD CVE listing. - */ - public static final String NVD_CACHE_START_YEAR = "nvd.cache.startyear"; - /** - * The username to use when connecting to the NVD Cache URL. - */ - public static final String NVD_CACHE_USER = "nvd.user"; - /** - * The password to authenticate to the NVD Cache URL. - */ - public static final String NVD_CACHE_PASSWORD = "nvd.password"; - /** - * The properties key for the URL to retrieve the recently cached NVD - * JSON data feeds. - */ - public static final String NVD_CACHE_URL = "nvd.cache.url"; + //END NEW /** * The key to determine if the NVD CVE analyzer is enabled. */ public static final String ANALYZER_NVD_CVE_ENABLED = "analyzer.nvdcve.enabled"; - /** - * The properties key setting indicating how many days past the new year - * that ODC will "skip" updating that years data feed if not present. - */ - public static final String NVD_NEW_YEAR_GRACE_PERIOD = "nvd.newyear.grace.period"; - //---------------------------------------------------------------------- - //DELETE END - //---------------------------------------------------------------------- - /** * The properties key that indicates how often the CPE data needs to be * updated. diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java index 637caf863db..7c905809cbb 100644 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java +++ b/utils/src/test/java/org/owasp/dependencycheck/utils/DownloaderIT.java @@ -46,9 +46,9 @@ public void setUp() { */ @Test public void testFetchFile() throws Exception { - - URL url = new URL(getSettings().getString(Settings.KEYS.CVE_MODIFIED_JSON)); - File outputPath = new File("target/downloaded_cve.xml"); + final String str = getSettings().getString(Settings.KEYS.ENGINE_VERSION_CHECK_URL, "https://jeremylong.github.io/DependencyCheck/current.txt"); + URL url = new URL(str); + File outputPath = new File("target/current.txt"); Downloader downloader = new Downloader(getSettings()); downloader.fetchFile(url, outputPath); assertTrue(outputPath.isFile()); diff --git a/utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java b/utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java index ceb6e49fe3c..81f938607d2 100644 --- a/utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java +++ b/utils/src/test/java/org/owasp/dependencycheck/utils/SettingsTest.java @@ -43,7 +43,7 @@ public class SettingsTest extends BaseTest { */ @Test public void testGetString() { - String key = Settings.KEYS.CVE_MODIFIED_VALID_FOR_DAYS; + String key = Settings.KEYS.NVD_API_DATAFEED_VALID_FOR_DAYS; String expResult = "7"; String result = getSettings().getString(key); Assert.assertTrue(result.endsWith(expResult));