Change bundling to use relatedArtifacts

tooling/src/main/java/org/opencds/cqf/tooling/library/LibraryProcessor.java

@@ -2,12 +2,7 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.regex.Pattern;
@@ -117,52 +112,47 @@ public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encodin
      * Bundles library dependencies for a given FHIR library file and populates the provided resource map.
      * This method executes asynchronously by invoking the associated task queue.
      *
-     * @param path        The path to the FHIR library file.
+     * @param library     The Library resource.
      * @param fhirContext The FHIR context to use for processing resources.
      * @param resources   The map to populate with library resources.
      * @param encoding    The encoding to use for reading and processing resources.
      * @param versioned   A boolean indicating whether to consider versioned resources.
      */
-    public void bundleLibraryDependencies(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
+    public void bundleLibraryDependencies(IBaseResource library, FhirContext fhirContext, Map<String, IBaseResource> resources,
                                           Encoding encoding, boolean versioned) throws Exception {
-        Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(path, fhirContext, resources, encoding, versioned);
+        Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(library, fhirContext, resources, encoding, versioned);
         ThreadUtils.executeTasks(bundleLibraryDependenciesTasks);
     }
 
     /**
      * Recursively bundles library dependencies for a given FHIR library file and populates the provided resource map.
      * Each dependency is added as a Callable task to be executed asynchronously.
      *
-     * @param path        The path to the FHIR library file.
+     * @param library     The Library resource
      * @param fhirContext The FHIR context to use for processing resources.
      * @param resources   The map to populate with library resources.
      * @param encoding    The encoding to use for reading and processing resources.
      * @param versioned   A boolean indicating whether to consider versioned resources.
      * @return A queue of Callable tasks, each representing the bundling of a library dependency.
      * The Callable returns null (Void) and is meant for asynchronous execution.
      */
-    public Queue<Callable<Void>> bundleLibraryDependenciesTasks(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
+    public Queue<Callable<Void>> bundleLibraryDependenciesTasks(IBaseResource library, FhirContext fhirContext, Map<String, IBaseResource> resources,
                                                                 Encoding encoding, boolean versioned) throws Exception {
 
         Queue<Callable<Void>> returnTasks = new ConcurrentLinkedQueue<>();
 
-        String fileName = FilenameUtils.getName(path);
-        boolean prefixed = fileName.toLowerCase().startsWith("library-");
-        Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(path, fhirContext, encoding, versioned, logger);
-        // String currentResourceID = IOUtils.getTypeQualifiedResourceId(path, fhirContext);
-        for (IBaseResource resource : dependencies.values()) {
-            returnTasks.add(() -> {
-                resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);
-
-                // NOTE: Assuming dependency library will be in directory of dependent.
-                String dependencyPath = IOUtils.getResourceFileName(IOUtils.getResourceDirectory(path), resource, encoding, fhirContext, versioned, prefixed);
+        returnTasks.add(() -> {
+            Set<String> missingDependencies = new HashSet<>();
+            Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(library, fhirContext, true, versioned, missingDependencies);
+            for (IBaseResource resource : dependencies.values()) {
+                resources.putIfAbsent(resource.fhirType() + '/' + resource.getIdElement().getIdPart(), resource);
+            }
 
-                returnTasks.addAll(bundleLibraryDependenciesTasks(dependencyPath, fhirContext, resources, encoding, versioned));
+            // TODO: Return missing dependencies as translator warnings...
 
-                //return statement needed for Callable<Void>
-                return null;
-            });
-        }
+            //return statement needed for Callable<Void>
+            return null;
+        });
         return returnTasks;
     }
 

tooling/src/main/java/org/opencds/cqf/tooling/processor/AbstractBundler.java

@@ -203,6 +203,8 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
                         shouldPersist = shouldPersist
                                 & ResourceUtils.safeAddResource(primaryLibrarySourcePath, resources, fhirContext);
 
+                        /*
+                        // Should not use CQL processing to determine value set dependencies
                         String cqlFileName = IOUtils.formatFileName(primaryLibraryName, IOUtils.Encoding.CQL, fhirContext);
 
                         String cqlLibrarySourcePath = IOUtils.getCqlLibrarySourcePath(primaryLibraryName, cqlFileName, binaryPaths);
@@ -212,11 +214,13 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
                             //exit from task:
                             return null;
                         }
+                        */
 
                         if (includeTerminology) {
                             //throws CQLTranslatorException if failed with severe errors, which will be logged and reported it in the final summary
                             try {
-                                ValueSetsProcessor.bundleValueSets(cqlLibrarySourcePath, igPath, fhirContext, resources, encoding, includeDependencies, includeVersion);
+                                //ValueSetsProcessor.bundleValueSets(cqlLibrarySourcePath, igPath, fhirContext, resources, encoding, includeDependencies, includeVersion);
+                                ValueSetsProcessor.bundleValueSets(primaryLibrary, fhirContext, resources, encoding, includeDependencies, includeVersion);
                             } catch (CqlTranslatorException cqlTranslatorException) {
                                 cqlTranslatorErrorMessages.put(primaryLibraryName, cqlTranslatorException.getErrors());
                             }
@@ -225,7 +229,7 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
                         if (includeDependencies) {
                             if (libraryProcessor == null) libraryProcessor = new LibraryProcessor();
                             try {
-                                libraryProcessor.bundleLibraryDependencies(primaryLibrarySourcePath, fhirContext, resources, encoding, includeVersion);
+                                libraryProcessor.bundleLibraryDependencies(primaryLibrary, fhirContext, resources, encoding, includeVersion);
                             } catch (Exception bre) {
                                 failedExceptionMessages.put(resourceSourcePath, getResourceBundlerType() + " will not be bundled because Library Dependency bundling failed: " + bre.getMessage());
                                 //exit from task:
@@ -248,9 +252,11 @@ public void bundleResources(ArrayList<String> refreshedLibraryNames, String igPa
 
                             persistBundle(bundleDestPath, resourceName, encoding, fhirContext, new ArrayList<IBaseResource>(resources.values()), fhirUri, addBundleTimestamp);
 
-                            bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath,
-                                    primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios,
-                                    includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages);
+                            // It's not clear at all why this is happening... we've already persisted the bundle? Why write out all the bundle files??
+                            // And if we _do_ need to write out the bundle files, why go through the whole assembling process again? Just write out the resources in the bundle we already have, right?
+                            //bundleFiles(igPath, bundleDestPath, resourceName, binaryPaths, resourceSourcePath,
+                            //        primaryLibrarySourcePath, fhirContext, encoding, includeTerminology, includeDependencies, includePatientScenarios,
+                            //        includeVersion, addBundleTimestamp, cqlTranslatorErrorMessages);
 
                             //If user supplied a fhir server url, inform them of total # of files to be persisted to the server:
                             if (fhirUri != null && !fhirUri.isEmpty()) {

tooling/src/main/java/org/opencds/cqf/tooling/processor/ValueSetsProcessor.java

@@ -1,15 +1,14 @@
 package org.opencds.cqf.tooling.processor;
 
 import ca.uhn.fhir.context.FhirContext;
+import org.cqframework.cql.cql2elm.CqlCompilerException;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.opencds.cqf.tooling.cql.exception.CqlTranslatorException;
 import org.opencds.cqf.tooling.utilities.IOUtils;
 import org.opencds.cqf.tooling.utilities.IOUtils.Encoding;
 import org.opencds.cqf.tooling.utilities.ResourceUtils;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 public class ValueSetsProcessor {
@@ -80,7 +79,19 @@ public static void bundleValueSets(String cqlContentPath, String igPath, FhirCon
             Map<String, IBaseResource> resources, Encoding encoding, Boolean includeDependencies, Boolean includeVersion) throws CqlTranslatorException {
             Map<String, IBaseResource> dependencies = ResourceUtils.getDepValueSetResources(cqlContentPath, igPath, fhirContext, includeDependencies, includeVersion);
             for (IBaseResource resource : dependencies.values()) {
-                resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);
+                resources.putIfAbsent(resource.fhirType() + '/' + resource.getIdElement().getIdPart(), resource);
             }
     }
+
+    public static void bundleValueSets(IBaseResource resource, FhirContext fhirContext,
+           Map<String, IBaseResource> resources, Encoding encoding, Boolean includeDependencies, Boolean includeVersion) throws CqlTranslatorException {
+        Set<String> missingDependencies = new HashSet<>();
+        Map<String, IBaseResource> dependencies = ResourceUtils.getDepValueSetResources(resource, fhirContext, includeDependencies, includeVersion, missingDependencies);
+        for (IBaseResource dependency : dependencies.values()) {
+            resources.putIfAbsent(resource.fhirType() + '/' + dependency.getIdElement().getIdPart(), resource);
+        }
+        if (missingDependencies.size() > 0) {
+            throw new CqlTranslatorException(missingDependencies.stream().collect(Collectors.toList()), CqlCompilerException.ErrorSeverity.Warning);
+        }
+    }
 }

tooling/src/main/java/org/opencds/cqf/tooling/utilities/IOUtils.java

@@ -799,7 +799,10 @@ private static void setupLibraryPaths(FhirContext fhirContext) {
                         libraryPaths.add(entry.getKey());
                         libraries.put(entry.getValue().getIdElement().getIdPart(), entry.getValue());
                         libraryPathMap.put(entry.getValue().getIdElement().getIdPart(), entry.getKey());
-                        libraryUrlMap.put(ResourceUtils.getUrl(entry.getValue(), fhirContext), entry.getValue());
+                        String url = ResourceUtils.getUrl(entry.getValue(), fhirContext);
+                        if (url != null) {
+                            libraryUrlMap.put(ResourceUtils.getUrl(entry.getValue(), fhirContext), entry.getValue());
+                        }
                     });
         }
     }

tooling/src/main/java/org/opencds/cqf/tooling/utilities/ResourceUtils.java

@@ -16,6 +16,7 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.Validate;
@@ -234,50 +235,49 @@ private static Map<String, IBaseResource> getR4DepLibraryResources(String path,
       return dependencyLibraries;
    }
 
-   public static List<String> getStu3TerminologyDependencies(List<org.hl7.fhir.dstu3.model.RelatedArtifact> relatedArtifacts) {
+   public static List<String> getStu3Dependencies(List<org.hl7.fhir.dstu3.model.RelatedArtifact> relatedArtifacts) {
       List<String> urls = new ArrayList<>();
       for (org.hl7.fhir.dstu3.model.RelatedArtifact relatedArtifact : relatedArtifacts) {
          if (relatedArtifact.hasType() && relatedArtifact.getType() == org.hl7.fhir.dstu3.model.RelatedArtifact.RelatedArtifactType.DEPENDSON) {
-            if (relatedArtifact.hasResource() && relatedArtifact.getResource().hasReference()
-                    && (relatedArtifact.getResource().getReference().contains("CodeSystem/") || relatedArtifact.getResource().getReference().contains("ValueSet/"))) {
+            if (relatedArtifact.hasResource() && relatedArtifact.getResource().hasReference()) {
                urls.add(relatedArtifact.getResource().getReference());
             }
          }
       }
       return urls;
    }
 
-   public static List<String> getR4TerminologyDependencies(List<org.hl7.fhir.r4.model.RelatedArtifact> relatedArtifacts) {
+   public static List<String> getR4Dependencies(List<org.hl7.fhir.r4.model.RelatedArtifact> relatedArtifacts) {
       List<String> urls = new ArrayList<>();
       for (org.hl7.fhir.r4.model.RelatedArtifact relatedArtifact : relatedArtifacts) {
          if (relatedArtifact.hasType() && relatedArtifact.getType() == org.hl7.fhir.r4.model.RelatedArtifact.RelatedArtifactType.DEPENDSON) {
-            if (relatedArtifact.hasResource() && (relatedArtifact.getResource().contains("CodeSystem/") || relatedArtifact.getResource().contains("ValueSet/"))) {
+            if (relatedArtifact.hasResource()) {
                urls.add(relatedArtifact.getResource());
             }
          }
       }
       return urls;
    }
 
-   public static List<String> getTerminologyDependencies(IBaseResource resource, FhirContext fhirContext) {
+   public static List<String> getDependencies(IBaseResource resource, FhirContext fhirContext) {
       switch (fhirContext.getVersion().getVersion()) {
          case DSTU3:
             switch (resource.fhirType()) {
                case "Library": {
-                  return getStu3TerminologyDependencies(((org.hl7.fhir.dstu3.model.Library)resource).getRelatedArtifact());
+                  return getStu3Dependencies(((org.hl7.fhir.dstu3.model.Library)resource).getRelatedArtifact());
                }
                case "Measure": {
-                  return getStu3TerminologyDependencies(((org.hl7.fhir.dstu3.model.Measure)resource).getRelatedArtifact());
+                  return getStu3Dependencies(((org.hl7.fhir.dstu3.model.Measure)resource).getRelatedArtifact());
                }
                default: throw new IllegalArgumentException(String.format("Could not retrieve relatedArtifacts from %s", resource.fhirType()));
             }
          case R4:
             switch (resource.fhirType()) {
                case "Library": {
-                  return getR4TerminologyDependencies(((org.hl7.fhir.r4.model.Library)resource).getRelatedArtifact());
+                  return getR4Dependencies(((org.hl7.fhir.r4.model.Library)resource).getRelatedArtifact());
                }
                case "Measure": {
-                  return getR4TerminologyDependencies(((org.hl7.fhir.r4.model.Measure)resource).getRelatedArtifact());
+                  return getR4Dependencies(((org.hl7.fhir.r4.model.Measure)resource).getRelatedArtifact());
                }
                default: throw new IllegalArgumentException(String.format("Could not retrieve relatedArtifacts from %s", resource.fhirType()));
             }
@@ -286,6 +286,83 @@ public static List<String> getTerminologyDependencies(IBaseResource resource, Fh
       }
    }
 
+   public static List<String> getTerminologyDependencies(IBaseResource resource, FhirContext fhirContext) {
+      return
+         getDependencies(resource, fhirContext).stream()
+                 .filter(url -> url.contains("/CodeSystem") || url.contains("/ValueSet"))
+                 .collect(Collectors.toList());
+   }
+
+   public static List<String> getLibraryDependencies(IBaseResource resource, FhirContext fhirContext) {
+      return
+        getDependencies(resource, fhirContext).stream()
+                .filter(url -> url.contains("/Library"))
+                .collect(Collectors.toList());
+   }
+
+   public static Map<String, IBaseResource> getDepLibraryResources(IBaseResource resource, FhirContext fhirContext, Boolean includeDependencies, Boolean includeVersion, Set<String> missingDependencies) {
+      Map<String, IBaseResource> libraryResources = new HashMap<>();
+
+      List<String> libraryUrls = getLibraryDependencies(resource, fhirContext);
+
+      for (String libraryUrl : libraryUrls) {
+         IBaseResource library = IOUtils.getLibraryUrlMap(fhirContext).get(libraryUrl);
+         if (library != null) {
+            libraryResources.putIfAbsent(libraryUrl, library);
+
+            if (includeDependencies) {
+               Map<String, IBaseResource> libraryDependencies = getDepLibraryResources(library, fhirContext, includeDependencies, includeVersion, missingDependencies);
+               for (Entry<String, IBaseResource> entry : libraryDependencies.entrySet()) {
+                  libraryResources.putIfAbsent(entry.getKey(), entry.getValue());
+               }
+            }
+         }
+         else {
+            missingDependencies.add(libraryUrl);
+         }
+      }
+
+      return libraryResources;
+   }
+
+   public static Map<String, IBaseResource> getDepValueSetResources(IBaseResource resource, FhirContext fhirContext, Boolean includeDependencies, Boolean includeVersion, Set<String> missingDependencies) {
+      Map<String, IBaseResource> valueSetResources = new HashMap<>();
+
+      List<String> valueSetUrls = getTerminologyDependencies(resource, fhirContext);
+
+      for (String valueSetUrl : valueSetUrls) {
+         ValueSetsProcessor.getCachedValueSets(fhirContext).entrySet().stream()
+                 .filter(entry -> entry.getKey().equals(valueSetUrl))
+                 .forEach(entry -> valueSetResources.put(entry.getKey(), entry.getValue()));
+      }
+      Set<String> dependencies = new HashSet<>(valueSetUrls);
+
+      if (includeDependencies) {
+         List<String> libraryUrls = getLibraryDependencies(resource, fhirContext);
+         for (String url : libraryUrls) {
+            IBaseResource library = IOUtils.getLibraryUrlMap(fhirContext).get(url);
+            if (library != null) {
+               Map<String, IBaseResource> dependencyValueSets = getDepValueSetResources(library, fhirContext, includeDependencies, includeVersion, missingDependencies);
+               dependencies.addAll(dependencyValueSets.keySet());
+               for (Entry<String, IBaseResource> entry : dependencyValueSets.entrySet()) {
+                  valueSetResources.putIfAbsent(entry.getKey(), entry.getValue());
+               }
+            }
+            else {
+               missingDependencies.add(url);
+            }
+         }
+      }
+
+      if (dependencies.size() != valueSetResources.size()) {
+         dependencies.removeAll(valueSetResources.keySet());
+         for (String valueSetUrl : dependencies) {
+            missingDependencies.add(valueSetUrl);
+         }
+      }
+      return valueSetResources;
+   }
+
    public static Map<String, IBaseResource> getDepValueSetResources(String cqlContentPath, String igPath, FhirContext fhirContext, boolean includeDependencies, Boolean includeVersion) throws CqlTranslatorException {
       Map<String, IBaseResource> valueSetResources = new HashMap<>();