Implementation of YarnModernReader to support Yarn 2 and above (#238)

core/src/main/java/com/devonfw/tools/solicitor/common/LogMessages.java

@@ -99,7 +99,11 @@ public enum LogMessages {
           + "skipped due to unkown SPDX: {}, mapped using type SCANCODE: {}, mapped using type OSS-SPDX: {}, mapped to IGNORE: {}"), //
   NOT_A_VALID_NPM_PACKAGE_NAME(67, "{} is not a valid name for an NPM package"), //
   SCANCODE_ISSUE_DETECTION_REGEX(68,
-      "The list of regular expressions for detecting licenses from scancode having issues is set to '{}'");
+      "The list of regular expressions for detecting licenses from scancode having issues is set to '{}'"), //
+  MODERN_YARN_VIRTUAL_PACKAGE(69,
+      "When reading yarn license info from file '{}' there was at least one virtual package encountered. Check if package resolution is correct"), //
+  MODERN_YARN_PATCHED_PACKAGE(70,
+      "When reading yarn license info from file '{}' there was at least one patched package encountered. Processing only the base package, not the patched version.");
 
   private final String message;
 

core/src/main/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReader.java

@@ -0,0 +1,181 @@
+/**
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.devonfw.tools.solicitor.reader.yarnmodern;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import com.devonfw.tools.solicitor.common.LogMessages;
+import com.devonfw.tools.solicitor.common.PackageURLHelper;
+import com.devonfw.tools.solicitor.common.SolicitorRuntimeException;
+import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent;
+import com.devonfw.tools.solicitor.model.masterdata.Application;
+import com.devonfw.tools.solicitor.model.masterdata.UsagePattern;
+import com.devonfw.tools.solicitor.reader.AbstractReader;
+import com.devonfw.tools.solicitor.reader.Reader;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+
+/**
+ * A {@link Reader} which reads data generated by
+ * <a href="https://github.com/mhassan1/yarn-plugin-licenses">yarn-plugin-licenses</a>.
+ */
+
+@Component
+public class YarnModernReader extends AbstractReader implements Reader {
+
+  private static final Logger LOG = LoggerFactory.getLogger(YarnModernReader.class);
+
+  /**
+   * The supported type of this {@link Reader}.
+   */
+  public static final String SUPPORTED_TYPE = "yarn-modern";
+
+  /** {@inheritDoc} */
+  @Override
+  public Set<String> getSupportedTypes() {
+
+    return Collections.singleton(SUPPORTED_TYPE);
+  }
+
+  /** {@inheritDoc} */
+  @SuppressWarnings("rawtypes")
+  @Override
+  public void readInventory(String type, String sourceUrl, Application application, UsagePattern usagePattern,
+      String repoType, Map<String, String> configuration) {
+
+    String content = readAndPreprocessJson(sourceUrl);
+
+    int componentCount = 0;
+    int licenseCount = 0;
+
+    ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+    List body;
+    try {
+      body = mapper.readValue(content, List.class);
+    } catch (IOException e) {
+      throw new SolicitorRuntimeException("Could not read yarn modern inventory source '" + sourceUrl + "'", e);
+    }
+    for (int i = 0; i < body.size(); i++) {
+      Map<String, String> licenseBlock = (Map<String, String>) body.get(i);
+
+      String license = licenseBlock.get("value");
+      List<String> locators = new ArrayList<>();
+
+      for (Map.Entry<String, String> entry : licenseBlock.entrySet()) {
+        if (entry.getKey().endsWith(".value.locator")) {
+          locators.add(entry.getValue());
+        }
+      }
+
+      for (String locator : locators) {
+        int lastAtPosition = locator.lastIndexOf("@");
+        String name = locator.substring(0, lastAtPosition);
+        String version = locator.substring(lastAtPosition + 1);
+        String repo = licenseBlock.get("children." + locator + ".children.url");
+        String licenseUrl = defaultGithubLicenseURL(repo);
+        String vendorUrl = licenseBlock.get("children." + locator + ".children.vendorUrl");
+        String homePage = "";
+        if (vendorUrl != null) {
+          homePage = vendorUrl;
+        }
+
+        ApplicationComponent appComponent = getModelFactory().newApplicationComponent();
+        appComponent.setApplication(application);
+        componentCount++;
+        appComponent.setArtifactId(name);
+        appComponent.setVersion(version);
+        appComponent.setUsagePattern(usagePattern);
+        appComponent.setGroupId("");
+        appComponent.setOssHomepage(homePage);
+        appComponent.setSourceRepoUrl(repo);
+        appComponent.setRepoType(repoType);
+        appComponent.setPackageUrl(PackageURLHelper.fromNpmPackageNameAndVersion(name, version).toString());
+
+        addRawLicense(appComponent, license, licenseUrl, sourceUrl);
+        licenseCount++;
+      }
+    }
+    doLogging(sourceUrl, application, componentCount, licenseCount);
+
+  }
+
+  // helper method that defaults github-links (html) to a default license path
+  private String defaultGithubLicenseURL(String repo) {
+
+    if (repo == null) {
+      return null;
+    }
+
+    if (repo.contains("github.com") && !repo.contains("/raw/")) {
+      repo = repo.replace("git://", "https://");
+      repo = repo.replace("github.com", "raw.githubusercontent.com");
+      repo = repo.concat("/master/LICENSE");
+    }
+    return repo;
+  }
+
+  // helper method that extracts information from the .json created by yarn
+  // licenses into a correct form
+  private String readAndPreprocessJson(String sourceURL) {
+
+    String filePath = sourceURL.replaceAll("file:", "");
+    File input = new File(filePath);
+    String content = "";
+
+    try {
+      BufferedReader reader = new BufferedReader(new FileReader(input));
+      String line = null;
+      StringBuilder sb = new StringBuilder();
+
+      // read in the complete file
+      while ((line = reader.readLine()) != null) {
+        sb.append(line).append("\n");
+      }
+
+      content = sb.toString();
+
+      // patch data of virtual and patched packages to conform to "normal" packages
+      String oldContent = content;
+      content = content.replaceAll("virtual:.*?#", "");
+      if (!content.equals(oldContent)) {
+        LOG.warn(LogMessages.MODERN_YARN_VIRTUAL_PACKAGE.msg(), sourceURL);
+      }
+      oldContent = content;
+      content = content.replaceAll("@patch:.*?@npm%3A(.*?)#.*?(.value.|.children.|\")", "@npm:$1$2");
+      if (!content.equals(oldContent)) {
+        LOG.warn(LogMessages.MODERN_YARN_PATCHED_PACKAGE.msg(), sourceURL);
+      }
+      content = content.replaceAll("@workspace:\\.", "@none");
+      content = content.replace("@npm:", "@");
+      // fixes URL issues
+      content = content.replace("git+", "");
+      content = content.replace("www.github", "github");
+      content = content.replace(".git", "");
+      content = content.replace("git://", "https://");
+      content = content.replace("[email protected]:", "https://github.com/");
+      content = content.replace("ssh://git@", "https://");
+      content = content.replace("Unknown", "");
+
+      reader.close();
+
+    } catch (IOException e) {
+      throw new SolicitorRuntimeException("Could not read yarn inventory source '" + sourceURL + "'", e);
+    }
+    return content;
+  }
+
+}

core/src/test/java/com/devonfw/tools/solicitor/reader/yarnmodern/YarnModernReaderTests.java

@@ -0,0 +1,121 @@
+/**
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.devonfw.tools.solicitor.reader.yarnmodern;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.devonfw.tools.solicitor.common.FileInputStreamFactory;
+import com.devonfw.tools.solicitor.model.ModelFactory;
+import com.devonfw.tools.solicitor.model.impl.ModelFactoryImpl;
+import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent;
+import com.devonfw.tools.solicitor.model.masterdata.Application;
+import com.devonfw.tools.solicitor.model.masterdata.UsagePattern;
+
+/**
+ * Tests for {@link YarnModernReader}.
+ */
+public class YarnModernReaderTests {
+  private static final Logger LOG = LoggerFactory.getLogger(YarnModernReaderTests.class);
+
+  Application application;
+
+  /**
+   * The constructor.
+   */
+  public YarnModernReaderTests() {
+
+    ModelFactory modelFactory = new ModelFactoryImpl();
+
+    this.application = modelFactory.newApplication("testApp", "0.0.0.TEST", "1.1.2111", "http://bla.com", "Angular");
+    YarnModernReader yr = new YarnModernReader();
+    yr.setModelFactory(modelFactory);
+    yr.setInputStreamFactory(new FileInputStreamFactory());
+    yr.readInventory("yarn-modern", "src/test/resources/yarnModernReport.json", this.application,
+        UsagePattern.STATIC_LINKING, "npm", null);
+
+  }
+
+  /**
+   * Check the read in data.
+   */
+  @Test
+  public void findArtifact() {
+
+    assertArtifactExists("@babel/runtime", //
+        "6.15.7", //
+        "pkg:npm/%40babel/[email protected]", //
+        "MIT", "https://babel.dev/docs/en/next/babel-runtime", //
+        "https://github.com/babel/babel", //
+        "https://raw.githubusercontent.com/babel/babel/master/LICENSE");
+    assertArtifactExists("@floating-ui/react-dom", //
+        "2.2.0", //
+        "pkg:npm/%40floating-ui/[email protected]", //
+        "MIT", "https://floating-ui.com/docs/react-dom", //
+        "https://github.com/floating-ui/floating-ui", //
+        "https://raw.githubusercontent.com/floating-ui/floating-ui/master/LICENSE");
+    assertArtifactExists("@humanwhocodes/config-array", //
+        "0.9.17", //
+        "pkg:npm/%40humanwhocodes/[email protected]", //
+        "Apache-2.0", //
+        "https://github.com/humanwhocodes/config-array#readme", //
+        "https://github.com/humanwhocodes/config-array", //
+        "https://raw.githubusercontent.com/humanwhocodes/config-array/master/LICENSE");
+    assertArtifactExists("typescript", //
+        "4.3.7", //
+        "pkg:npm/[email protected]", //
+        "Apache-2.0", //
+        "https://www.typescriptlang.org/", //
+        "https://github.com/Microsoft/TypeScript", //
+        "https://raw.githubusercontent.com/Microsoft/TypeScript/master/LICENSE");
+    assertArtifactExists("my-space", //
+        "none", //
+        "pkg:npm/my-space@none", //
+        "UNKNOWN", //
+        "", //
+        null, //
+        null);
+  }
+
+  private void assertArtifactExists(String artifactId, String version, String packageUrl, String license,
+      String ossHomepage, String sourceRepoUrl, String licenseUrl) {
+
+    List<ApplicationComponent> lapc = this.application.getApplicationComponents();
+    boolean found = false;
+    for (ApplicationComponent ap : lapc) {
+      if (ap.getArtifactId().equals(artifactId) && //
+          ap.getVersion().equals(version)) {
+        found = true;
+        assertEquals(packageUrl, ap.getPackageUrl());
+        assertEquals(license, ap.getRawLicenses().get(0).getDeclaredLicense());
+        assertEquals(ossHomepage, ap.getOssHomepage());
+        assertEquals(sourceRepoUrl, ap.getSourceRepoUrl());
+        assertEquals(licenseUrl, ap.getRawLicenses().get(0).getLicenseUrl());
+        assertEquals("npm", ap.getRepoType());
+        assertEquals(UsagePattern.STATIC_LINKING, ap.getUsagePattern());
+
+        break;
+      }
+    }
+    assertTrue(found);
+  }
+
+  /**
+   * Test if the number of read components is correct.
+   */
+  @Test
+  public void readFileAndCheckSize() {
+
+    LOG.info(this.application.toString());
+    assertEquals(5, this.application.getApplicationComponents().size());
+  }
+
+}

core/src/test/resources/yarnModernReport.json

@@ -0,0 +1,33 @@
+[
+  {
+    "value": "MIT",
+    "children.@babel/runtime@npm:6.15.7.value.locator": "@babel/runtime@npm:6.15.7",
+    "children.@babel/runtime@npm:6.15.7.value.descriptor": "@babel/runtime@npm:^6.17.7",
+    "children.@babel/runtime@npm:6.15.7.children.url": "https://github.com/babel/babel.git",
+    "children.@babel/runtime@npm:6.15.7.children.vendorName": "The Babel Team",
+    "children.@babel/runtime@npm:6.15.7.children.vendorUrl": "https://babel.dev/docs/en/next/babel-runtime",
+    "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.value.locator": "@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0",
+    "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.value.descriptor": "@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:^2.0.0",
+    "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.url": "https://github.com/floating-ui/floating-ui.git",
+    "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.vendorName": "atomiks",
+    "children.@floating-ui/react-dom@virtual:3144d465a5f41fdd2d427886c8d853d1294ea068e6d3ea0018fc9787793599db665d173a4e52d32010c4b3e07a91aa0eeac00cbb00a677214be83ab38f3b1afc#npm:2.2.0.children.vendorUrl": "https://floating-ui.com/docs/react-dom"
+  },
+  {
+    "value": "Apache-2.0",
+    "children.@humanwhocodes/config-array@npm:0.9.17.value.locator": "@humanwhocodes/config-array@npm:0.9.17",
+    "children.@humanwhocodes/config-array@npm:0.9.17.value.descriptor": "@humanwhocodes/config-array@npm:^0.9.8",
+    "children.@humanwhocodes/config-array@npm:0.9.17.children.url": "git+https://github.com/humanwhocodes/config-array.git",
+    "children.@humanwhocodes/config-array@npm:0.9.17.children.vendorName": "Nicholas C. Zakas",
+    "children.@humanwhocodes/config-array@npm:0.9.17.children.vendorUrl": "https://github.com/humanwhocodes/config-array#readme",
+    "children.typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e.value.locator": "typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e",
+    "children.typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e.value.descriptor": "typescript@patch:[email protected]#~builtin<compat/typescript>",
+    "children.typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e.children.url": "https://github.com/Microsoft/TypeScript.git",
+    "children.typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e.children.vendorName": "Microsoft Corp.",
+    "children.typescript@patch:typescript@npm%3A4.3.7#~builtin<compat/typescript>::version=4.3.7&hash=c7623e.children.vendorUrl": "https://www.typescriptlang.org/"
+  },
+  {
+    "value": "UNKNOWN",
+    "children.my-space@workspace:..value.locator": "my-space@workspace:.",
+    "children.my-space@workspace:..value.descriptor": "my-space@workspace:."
+  }
+]

documentation/files/yarnmodernlicenses.json

@@ -0,0 +1,10 @@
+[
+  {
+    "value": "MIT",
+    "children.@babel/runtime@npm:6.15.7.value.locator": "@babel/runtime@npm:6.15.7",
+    "children.@babel/runtime@npm:6.15.7.value.descriptor": "@babel/runtime@npm:^6.17.7",
+    "children.@babel/runtime@npm:6.15.7.children.url": "https://github.com/babel/babel.git",
+    "children.@babel/runtime@npm:6.15.7.children.vendorName": "The Babel Team",
+    "children.@babel/runtime@npm:6.15.7.children.vendorUrl": "https://babel.dev/docs/en/next/babel-runtime",
+  }
+]