diff --git a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java index 023bc065..7c7a112e 100644 --- a/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java +++ b/core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java @@ -99,7 +99,11 @@ public enum LogMessages { + "skipped due to unkown SPDX: {}, mapped using type SCANCODE: {}, mapped using type OSS-SPDX: {}, mapped to IGNORE: {}"), // NOT_A_VALID_NPM_PACKAGE_NAME(67, "{} is not a valid name for an NPM package"), // SCANCODE_ISSUE_DETECTION_REGEX(68, - "The list of regular expressions for detecting licenses from scancode having issues is set to '{}'"); + "The list of regular expressions for detecting licenses from scancode having issues is set to '{}'"), // + MODERN_YARN_VIRTUAL_PACKAGE(69, + "When reading yarn license info from file '{}' there was at least one virtual package encountered. Check if package resolution is correct"), // + MODERN_YARN_PATCHED_PACKAGE(70, + "When reading yarn license info from file '{}' there was at least one patched package encountered. Processing only the base package, not the patched version."); private final String message; diff --git a/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java new file mode 100644 index 00000000..1ca903b4 --- /dev/null +++ b/core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java @@ -0,0 +1,181 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.devonfw.tools.solicitor.reader.yarnmodern; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.devonfw.tools.solicitor.common.LogMessages; +import com.devonfw.tools.solicitor.common.PackageURLHelper; +import com.devonfw.tools.solicitor.common.SolicitorRuntimeException; +import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent; +import com.devonfw.tools.solicitor.model.masterdata.Application; +import com.devonfw.tools.solicitor.model.masterdata.UsagePattern; +import com.devonfw.tools.solicitor.reader.AbstractReader; +import com.devonfw.tools.solicitor.reader.Reader; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * A {@link Reader} which reads data generated by + * yarn-plugin-licenses. + */ + +@Component +public class YarnModernReader extends AbstractReader implements Reader { + + private static final Logger LOG = LoggerFactory.getLogger(YarnModernReader.class); + + /** + * The supported type of this {@link Reader}. + */ + public static final String SUPPORTED_TYPE = "yarn-modern"; + + /** {@inheritDoc} */ + @Override + public Set getSupportedTypes() { + + return Collections.singleton(SUPPORTED_TYPE); + } + + /** {@inheritDoc} */ + @SuppressWarnings("rawtypes") + @Override + public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern, + String repoType, Map configuration) { + + String content = readAndPreprocessJson(sourceUrl); + + int componentCount = 0; + int licenseCount = 0; + + ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + List body; + try { + body = mapper.readValue(content, List.class); + } catch (IOException e) { + throw new SolicitorRuntimeException("Could not read yarn modern inventory source '" + sourceUrl + "'", e); + } + for (int i = 0; i < body.size(); i++) { + Map licenseBlock = (Map) body.get(i); + + String license = licenseBlock.get("value"); + List locators = new ArrayList<>(); + + for (Map.Entry entry : licenseBlock.entrySet()) { + if (entry.getKey().endsWith(".value.locator")) { + locators.add(entry.getValue()); + } + } + + for (String locator : locators) { + int lastAtPosition = locator.lastIndexOf("@"); + String name = locator.substring(0, lastAtPosition); + String version = locator.substring(lastAtPosition + 1); + String repo = licenseBlock.get("children." + locator + ".children.url"); + String licenseUrl = defaultGithubLicenseURL(repo); + String vendorUrl = licenseBlock.get("children." + locator + ".children.vendorUrl"); + String homePage = ""; + if (vendorUrl != null) { + homePage = vendorUrl; + } + + ApplicationComponent appComponent = getModelFactory().newApplicationComponent(); + appComponent.setApplication(application); + componentCount++; + appComponent.setArtifactId(name); + appComponent.setVersion(version); + appComponent.setUsagePattern(usagePattern); + appComponent.setGroupId(""); + appComponent.setOssHomepage(homePage); + appComponent.setSourceRepoUrl(repo); + appComponent.setRepoType(repoType); + appComponent.setPackageUrl(PackageURLHelper.fromNpmPackageNameAndVersion(name, version).toString()); + + addRawLicense(appComponent, license, licenseUrl, sourceUrl); + licenseCount++; + } + } + doLogging(sourceUrl, application, componentCount, licenseCount); + + } + + // helper method that defaults github-links (html) to a default license path + private String defaultGithubLicenseURL(String repo) { + + if (repo == null) { + return null; + } + + if (repo.contains("github.com") && !repo.contains("/raw/")) { + repo = repo.replace("git://", "https://"); + repo = repo.replace("github.com", "raw.githubusercontent.com"); + repo = repo.concat("/master/LICENSE"); + } + return repo; + } + + // helper method that extracts information from the .json created by yarn + // licenses into a correct form + private String readAndPreprocessJson(String sourceURL) { + + String filePath = sourceURL.replaceAll("file:", ""); + File input = new File(filePath); + String content = ""; + + try { + BufferedReader reader = new BufferedReader(new FileReader(input)); + String line = null; + StringBuilder sb = new StringBuilder(); + + // read in the complete file + while ((line = reader.readLine()) != null) { + sb.append(line).append("\n"); + } + + content = sb.toString(); + + // patch data of virtual and patched packages to conform to "normal" packages + String oldContent = content; + content = content.replaceAll("virtual:.*?#", ""); + if (!content.equals(oldContent)) { + LOG.warn(LogMessages.MODERN_YARN_VIRTUAL_PACKAGE.msg(), sourceURL); + } + oldContent = content; + content = content.replaceAll("@patch:.*?@npm%3A(.*?)#.*?(.value.|.children.|\")", "@npm:$1$2"); + if (!content.equals(oldContent)) { + LOG.warn(LogMessages.MODERN_YARN_PATCHED_PACKAGE.msg(), sourceURL); + } + content = content.replaceAll("@workspace:\\.", "@none"); + content = content.replace("@npm:", "@"); + // fixes URL issues + content = content.replace("git+", ""); + content = content.replace("www.github", "github"); + content = content.replace(".git", ""); + content = content.replace("git://", "https://"); + content = content.replace("git@github.com:", "https://github.com/"); + content = content.replace("ssh://git@", "https://"); + content = content.replace("Unknown", ""); + + reader.close(); + + } catch (IOException e) { + throw new SolicitorRuntimeException("Could not read yarn inventory source '" + sourceURL + "'", e); + } + return content; + } + +} diff --git a/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java new file mode 100644 index 00000000..dc02828e --- /dev/null +++ b/core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java @@ -0,0 +1,121 @@ +/** + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.devonfw.tools.solicitor.reader.yarnmodern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.devonfw.tools.solicitor.common.FileInputStreamFactory; +import com.devonfw.tools.solicitor.model.ModelFactory; +import com.devonfw.tools.solicitor.model.impl.ModelFactoryImpl; +import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent; +import com.devonfw.tools.solicitor.model.masterdata.Application; +import com.devonfw.tools.solicitor.model.masterdata.UsagePattern; + +/** + * Tests for {@link YarnModernReader}. + */ +public class YarnModernReaderTests { + private static final Logger LOG = LoggerFactory.getLogger(YarnModernReaderTests.class); + + Application application; + + /** + * The constructor. + */ + public YarnModernReaderTests() { + + ModelFactory modelFactory = new ModelFactoryImpl(); + + this.application = modelFactory.newApplication("testApp", "0.0.0.TEST", "1.1.2111", "http://bla.com", "Angular"); + YarnModernReader yr = new YarnModernReader(); + yr.setModelFactory(modelFactory); + yr.setInputStreamFactory(new FileInputStreamFactory()); + yr.readInventory("yarn-modern", "src/test/resources/yarnModernReport.json", this.application, + UsagePattern.STATIC_LINKING, "npm", null); + + } + + /** + * Check the read in data. + */ + @Test + public void findArtifact() { + + assertArtifactExists("@babel/runtime", // + "6.15.7", // + "pkg:npm/%40babel/runtime@6.15.7", // + "MIT", "https://babel.dev/docs/en/next/babel-runtime", // + "https://github.com/babel/babel", // + "https://raw.githubusercontent.com/babel/babel/master/LICENSE"); + assertArtifactExists("@floating-ui/react-dom", // + "2.2.0", // + "pkg:npm/%40floating-ui/react-dom@2.2.0", // + "MIT", "https://floating-ui.com/docs/react-dom", // + "https://github.com/floating-ui/floating-ui", // + "https://raw.githubusercontent.com/floating-ui/floating-ui/master/LICENSE"); + assertArtifactExists("@humanwhocodes/config-array", // + "0.9.17", // + "pkg:npm/%40humanwhocodes/config-array@0.9.17", // + "Apache-2.0", // + "https://github.com/humanwhocodes/config-array#readme", // + "https://github.com/humanwhocodes/config-array", // + "https://raw.githubusercontent.com/humanwhocodes/config-array/master/LICENSE"); + assertArtifactExists("typescript", // + "4.3.7", // + "pkg:npm/typescript@4.3.7", // + "Apache-2.0", // + "https://www.typescriptlang.org/", // + "https://github.com/Microsoft/TypeScript", // + "https://raw.githubusercontent.com/Microsoft/TypeScript/master/LICENSE"); + assertArtifactExists("my-space", // + "none", // + "pkg:npm/my-space@none", // + "UNKNOWN", // + "", // + null, // + null); + } + + private void assertArtifactExists(String artifactId, String version, String packageUrl, String license, + String ossHomepage, String sourceRepoUrl, String licenseUrl) { + + List lapc = this.application.getApplicationComponents(); + boolean found = false; + for (ApplicationComponent ap : lapc) { + if (ap.getArtifactId().equals(artifactId) && // + ap.getVersion().equals(version)) { + found = true; + assertEquals(packageUrl, ap.getPackageUrl()); + assertEquals(license, ap.getRawLicenses().get(0).getDeclaredLicense()); + assertEquals(ossHomepage, ap.getOssHomepage()); + assertEquals(sourceRepoUrl, ap.getSourceRepoUrl()); + assertEquals(licenseUrl, ap.getRawLicenses().get(0).getLicenseUrl()); + assertEquals("npm", ap.getRepoType()); + assertEquals(UsagePattern.STATIC_LINKING, ap.getUsagePattern()); + + break; + } + } + assertTrue(found); + } + + /** + * Test if the number of read components is correct. + */ + @Test + public void readFileAndCheckSize() { + + LOG.info(this.application.toString()); + assertEquals(5, this.application.getApplicationComponents().size()); + } + +} diff --git a/core/src/test/resources/yarnModernReport.json b/core/src/test/resources/yarnModernReport.json new file mode 100644 index 00000000..753f4ff1 --- /dev/null +++ b/core/src/test/resources/yarnModernReport.json @@ -0,0 +1,33 @@ +[ + { + "value": "MIT", + "children.@babel/runtime@npm:6.15.7.value.locator": "@babel/runtime@npm:6.15.7", + "children.@babel/runtime@npm:6.15.7.value.descriptor": "@babel/runtime@npm:^6.17.7", + "children.@babel/runtime@npm:6.15.7.children.url": "https://github.com/babel/babel.git", + "children.@babel/runtime@npm:6.15.7.children.vendorName": "The Babel Team", + "children.@babel/runtime@npm:6.15.7.children.vendorUrl": "https://babel.dev/docs/en/next/babel-runtime", + "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.value.locator": "@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0", + "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.value.descriptor": "@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:^2.0.0", + "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.url": "https://github.com/floating-ui/floating-ui.git", + "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.vendorName": "atomiks", + "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.vendorUrl": "https://floating-ui.com/docs/react-dom" + }, + { + "value": "Apache-2.0", + "children.@humanwhocodes/config-array@npm:0.9.17.value.locator": "@humanwhocodes/config-array@npm:0.9.17", + "children.@humanwhocodes/config-array@npm:0.9.17.value.descriptor": "@humanwhocodes/config-array@npm:^0.9.8", + "children.@humanwhocodes/config-array@npm:0.9.17.children.url": "git+https://github.com/humanwhocodes/config-array.git", + "children.@humanwhocodes/config-array@npm:0.9.17.children.vendorName": "Nicholas C. Zakas", + "children.@humanwhocodes/config-array@npm:0.9.17.children.vendorUrl": "https://github.com/humanwhocodes/config-array#readme", + "children.typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e.value.locator": "typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e", + "children.typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e.value.descriptor": "typescript@patch:typescript@4.3.7#~builtin", + "children.typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e.children.url": "https://github.com/Microsoft/TypeScript.git", + "children.typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e.children.vendorName": "Microsoft Corp.", + "children.typescript@patch:typescript@npm%3A4.3.7#~builtin::version=4.3.7&hash=c7623e.children.vendorUrl": "https://www.typescriptlang.org/" + }, + { + "value": "UNKNOWN", + "children.my-space@workspace:..value.locator": "my-space@workspace:.", + "children.my-space@workspace:..value.descriptor": "my-space@workspace:." + } +] diff --git a/documentation/files/yarnmodernlicenses.json b/documentation/files/yarnmodernlicenses.json new file mode 100644 index 00000000..3b1ac5dd --- /dev/null +++ b/documentation/files/yarnmodernlicenses.json @@ -0,0 +1,10 @@ +[ + { + "value": "MIT", + "children.@babel/runtime@npm:6.15.7.value.locator": "@babel/runtime@npm:6.15.7", + "children.@babel/runtime@npm:6.15.7.value.descriptor": "@babel/runtime@npm:^6.17.7", + "children.@babel/runtime@npm:6.15.7.children.url": "https://github.com/babel/babel.git", + "children.@babel/runtime@npm:6.15.7.children.vendorName": "The Babel Team", + "children.@babel/runtime@npm:6.15.7.children.vendorUrl": "https://babel.dev/docs/en/next/babel-runtime", + } +] diff --git a/documentation/master-solicitor.asciidoc b/documentation/master-solicitor.asciidoc index 9d8f74ab..63563a0e 100644 --- a/documentation/master-solicitor.asciidoc +++ b/documentation/master-solicitor.asciidoc @@ -686,7 +686,7 @@ In _Solicitor_ the data is read with the following part of the config } ] ---- -=== Yarn +=== Yarn Classic (Yarn 1) To generate the input file required for Solicitor, yarn needs to be executed with the following command within the directory that contains the project's package.json (we require JSON output here): @@ -714,6 +714,37 @@ In _Solicitor_ the data is read with the following part of the config } ] ---- +=== Yarn Modern (Yarn 2 and above) + +In Yarn Modern the functionality to create a licenses report can be achieved with a separate component: https://github.com/mhassan1/yarn-plugin-licenses + +To generate the input file required for Solicitor, the plugin needs to be executed with the following command within the directory that contains the project's package.json (we require JSON output here): + +---- +yarn licenses list --production --recursive --json > /path/to/yarnmodernlicenses.json +---- + +The export should look like the following + +[source] +---- +include::files/yarnmodernlicenses.json[] + +---- + +Source: https://github.com/mhassan1/yarn-plugin-licenses + +In _Solicitor_ the data is read with the following part of the config + +---- +"readers" : [ { + "type" : "yarn-modern", + "source" : "file:path/to/yarnmodernlicenses.json", + "usagePattern" : "STATIC_LINKING" +} ] +---- + + === Pip To generate the input file required for Solicitor, one has to follow two steps: @@ -1671,6 +1702,7 @@ Changes in 1.20.0:: * https://github.com/devonfw/solicitor/issues/232: Set a standard for ordering LicenseNameMapping rules. Rules with an 'or-later' suffix are put before '-only' rules. * https://github.com/devonfw/solicitor/issues/234: Correct handling of new data model fields in ModelImporterExporter `dataStatus`,`traceabilityNotes` etc. * https://github.com/devonfw/solicitor/pull/235: Improvements in Curation Data Handling. When the curationDataSelector parameter is set to "none," no curations will be applied. +* https://github.com/devonfw/solicitor/issues/237: New Reader for license info of systems built with Yarn Modern. See <>. Changes in 1.19.0:: * https://github.com/devonfw/solicitor/issues/227: Fixed a bug where the `dataStatus` field in the aggregated OSS-Inventory was not filled.