Skip to content

Commit

Permalink
Cqf-Tooling : NewRefreshIg update (#528)
Browse files Browse the repository at this point in the history
* Added missing tests

* _Added CqlRefresh in order to update cql file before conversion.
_Made sure that we have the cql data inside the json library and that it matches the udpated cql
_Update tests to verify everything works as expected.

* Formatting

* Test update

* Test path update

* Update test files

* Update test files 2

* Clean up of test file

* Clean up of test file 2

* This one is not a file path

* Fix output directory

* Move files to same directory

* TEST: don't swallow exceptions

* Add logging of missing files

* Revert previous experiments

* Fix casing of folder name

* Add path to CQL source information

* Add more logging

* Fix prefix, fix some level 1.3 warnings

* Change build to use info log level for debug purposes

* Swap up dependency loading code

* Revert to normal logging

---------

Co-authored-by: Jonathan Percival <[email protected]>
  • Loading branch information
ddieppois and JPercival authored May 17, 2024
1 parent 074e341 commit 0949d9b
Show file tree
Hide file tree
Showing 26 changed files with 300 additions and 44 deletions.
2 changes: 1 addition & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,5 @@
"testng"
],
"java.compile.nullAnalysis.mode": "automatic",
"java.jdt.ls.vmargs": "-XX:+UseParallelGC -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -Dsun.zip.disableMemoryMapping=true -Xmx2G -Xms100m -Xlog:disable"
"java.jdt.ls.vmargs": "-XX:+UseParallelGC -XX:GCTimeRatio=4 -XX:AdaptiveSizePolicyWeight=90 -Dsun.zip.disableMemoryMapping=true -Xmx4G -Xms100m -Xlog:disable"
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@

public class LibraryProcessor extends BaseProcessor {
private static final Logger logger = LoggerFactory.getLogger(LibraryProcessor.class);
public static final String ResourcePrefix = "library-";
public static final String ResourcePrefix = "Library-";

public static String getId(String baseId) {
return ResourcePrefix + baseId;
Expand Down Expand Up @@ -280,14 +280,14 @@ protected List<Library> refreshGeneratedContent(List<Library> sourceLibraries) {
}

public List<Library> refreshGeneratedContent(String cqlDirectoryPath, String fhirVersion) {
List<String> result = new ArrayList<String>();
var result = new ArrayList<String>();
File input = new File(cqlDirectoryPath);
if (input.exists() && input.isDirectory()) {
result.add(input.getAbsolutePath());
}
setBinaryPaths(result);

List<Library> libraries = new ArrayList<Library>();
var libraries = new ArrayList<Library>();
return internalRefreshGeneratedContent(libraries);
}

Expand Down Expand Up @@ -330,6 +330,11 @@ private List<Library> internalRefreshGeneratedContent(List<Library> sourceLibrar
sourceLibraries.add(newLibrary);
}
}
else
{
logger.warn("No identifier found for CQL file {}", fileInfo.getPath());
}

}

List<Library> resources = new ArrayList<Library>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,10 @@ private void loadCorePackage() {
}

private void loadIg(ImplementationGuide.ImplementationGuideDependsOnComponent dep, int index) throws IOException {
logger.info("Loading IG Dependency {}#{}", dep.getUri(), dep.getVersion());
String name = dep.getId();
if (!dep.hasId()) {
logger.info("Dependency '{}' has no id, so can't be referred to in markdown in the IG", idForDep(dep));
logger.warn("Dependency '{}' has no id, so can't be referred to in markdown in the IG", idForDep(dep));
name = "u" + Utilities.makeUuidLC().replace("-", "");
}
if (!isValidIGToken(name)) {
Expand All @@ -155,10 +156,9 @@ private void loadIg(ImplementationGuide.ImplementationGuideDependsOnComponent de
throw new IllegalArgumentException(
"You must specify a version for the IG " + packageId + " (" + canonical + ")");

NpmPackage pi = packageId == null ? null : pcm.loadPackageFromCacheOnly(packageId, igver);
if (pi != null)
npmList.add(pi);
NpmPackage pi = pcm.loadPackage(packageId, igver);
if (pi == null) {
logger.warn("Dependency " + name + " (" + canonical + ") not found by FilesystemPackageCacheManager");
pi = resolveDependency(canonical, packageId, igver);
if (pi == null) {
if (Utilities.noString(packageId))
Expand All @@ -169,6 +169,8 @@ private void loadIg(ImplementationGuide.ImplementationGuideDependsOnComponent de
}
}

npmList.add(pi);

logger.debug(
"Load " + name + " (" + canonical + ") from " + packageId + "#" + igver);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package org.opencds.cqf.tooling.operation.ig;

import org.hl7.fhir.instance.model.api.IBaseResource;
import org.opencds.cqf.tooling.parameter.RefreshIGParameters;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

public class CqlRefresh extends Refresh {

private static final Logger logger = LoggerFactory.getLogger(CqlRefresh.class);
private final Pattern VERSION_PATTERN = Pattern.compile("^(library\\s+(\\S+)\\s+version\\s+)'[0-9]+\\.[0-9]+\\.[0-9]+'");
private final Pattern INCLUDE_PATTERN = Pattern.compile("^(include\\s+(\\S+)\\s+version\\s+)'([0-9]+\\.[0-9]+\\.[0-9]+)'(\\s+called\\s+(\\S+))?");

public CqlRefresh(IGInfo igInfo) {
super(igInfo);
}

@Override
public List<IBaseResource> refresh() {
return List.of();
}

public void refreshCql(IGInfo igInfo, RefreshIGParameters params) {
Map<String, String> updatedLibraries = refreshCqlFile(igInfo.getCqlBinaryPath(), params.updatedVersion);
updateCqlReferences(igInfo.getCqlBinaryPath(), updatedLibraries);
}

private Map<String, String> refreshCqlFile(String cqlBinaryPath, String updatedVersion) {
Map<String, String> updatedLibraries = new HashMap<>();
try (Stream<Path> paths = Files.walk(Paths.get(cqlBinaryPath))) {
List<Path> files = paths
.filter(Files::isRegularFile)
.filter(path -> path.toString().endsWith(".cql"))
.collect(Collectors.toList());

for (Path file : files) {
List<String> lines = Files.readAllLines(file);
for (int i = 0; i < lines.size(); i++) {
Matcher matcher = VERSION_PATTERN.matcher(lines.get(i));
if (matcher.matches()) {
String libraryName = matcher.group(2);
String updatedLine = matcher.replaceFirst("$1'" + updatedVersion + "'");
lines.set(i, updatedLine);
Files.write(file, lines);
updatedLibraries.put(libraryName, updatedVersion);
break;
}
}
}
} catch (IOException e) {
logger.error("Error updating cql files: {}", e.getMessage());
}
return updatedLibraries;
}

private void updateCqlReferences(String cqlBinaryPath, Map<String, String> updatedLibraries) {
try (Stream<Path> paths = Files.walk(Paths.get(cqlBinaryPath))) {
List<Path> files = paths
.filter(Files::isRegularFile)
.filter(path -> path.toString().endsWith(".cql"))
.collect(Collectors.toList());

for (Path file : files) {
List<String> lines = Files.readAllLines(file);
boolean fileUpdated = false;
for (int i = 0; i < lines.size(); i++) {
Matcher matcher = INCLUDE_PATTERN.matcher(lines.get(i));
while (matcher.find()) {
String libraryName = matcher.group(2);
String newVersion = updatedLibraries.get(libraryName);
if (newVersion != null && !matcher.group(3).equals(newVersion)) {
String calledPart = matcher.group(4) != null ? matcher.group(4) : "";
String newLine = matcher.replaceFirst("$1'" + newVersion + "'" + calledPart);
lines.set(i, newLine);
fileUpdated = true;
}
}
}
if (fileUpdated) {
Files.write(file, lines);
}
}
} catch (IOException e) {
logger.error("Error updating CQL references: {}", e.getMessage());
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ public List<IBaseResource> refresh(RefreshIGParameters params) {
logger.info("Refreshing {}", library.getIdElement());

for (CqlProcessor.CqlSourceFileInformation info : cqlProcessor.getAllFileInformation()) {
if (info.getIdentifier() == null) {
logger.warn("No identifier found for CQL file {}", info.getPath());
}

if (info.getIdentifier().getId().endsWith(name)) {
// TODO: should likely verify or resolve/refresh the following elements:
// cpg-knowledgeCapability, cpg-knowledgeRepresentationLevel, url, identifier, status,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ public void execute(String[] args) {
try {
this.params = new RefreshIGArgumentProcessor().parseAndConvert(args);
IGInfo info = new IGInfo(null, params);
CqlRefresh cqlRefresh = new CqlRefresh(info);
cqlRefresh.refreshCql(info, params);
LibraryRefresh libraryRefresh = new LibraryRefresh(info);
publishLibraries(info, libraryRefresh.refresh(this.params));
PlanDefinitionRefresh planDefinitionRefresh = new PlanDefinitionRefresh(info, libraryRefresh.getCqlProcessor(), libraryRefresh.getLibraryPackages());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.io.File;
import java.io.FilenameFilter;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -34,13 +35,17 @@
import org.opencds.cqf.tooling.npm.NpmLibrarySourceProvider;
import org.opencds.cqf.tooling.npm.NpmModelInfoProvider;
import org.opencds.cqf.tooling.utilities.ResourceUtils;
import org.slf4j.Logger;

public class CqlProcessor {

private static final Logger log = org.slf4j.LoggerFactory.getLogger(CqlProcessor.class);

/**
* information about a cql file
*/
public class CqlSourceFileInformation {
private final String path;
private CqlTranslatorOptions options;
private VersionedIdentifier identifier;
private byte[] cql;
Expand All @@ -50,6 +55,15 @@ public class CqlSourceFileInformation {
private List<RelatedArtifact> relatedArtifacts = new ArrayList<>();
private List<DataRequirement> dataRequirements = new ArrayList<>();
private List<ParameterDefinition> parameters = new ArrayList<>();

public CqlSourceFileInformation(String path) {
this.path = path;
}

public String getPath() {
return path;
}

public CqlTranslatorOptions getOptions() {
return options;
}
Expand Down Expand Up @@ -361,7 +375,7 @@ public static ValidationMessage exceptionToValidationMessage(File file, CqlCompi

private void translateFile(LibraryManager libraryManager, File file, CqlCompilerOptions options) {
// logger.logMessage(String.format("Translating CQL source in file %s", file.toString()));
CqlSourceFileInformation result = new CqlSourceFileInformation();
CqlSourceFileInformation result = new CqlSourceFileInformation(file.getAbsolutePath());
fileMap.put(file.getAbsoluteFile().toString(), result);

if (options.getValidateUnits()) {
Expand All @@ -382,6 +396,8 @@ private void translateFile(LibraryManager libraryManager, File file, CqlCompiler


if (!severeErrorList.isEmpty()) {
var messages = severeErrorList.stream().map(x -> x.getMessage()).reduce("", (x, y) -> x + "\n" + y);
log.warn("CQL Processing failed with errors count: {}, messages: {}", severeErrorList.size(), messages);
result.getErrors().add(new ValidationMessage(ValidationMessage.Source.Publisher, IssueType.EXCEPTION, file.getName(),
String.format("CQL Processing failed with (%d) errors.", translator.getErrors().size()), IssueSeverity.ERROR));
}
Expand All @@ -390,6 +406,7 @@ private void translateFile(LibraryManager libraryManager, File file, CqlCompiler
result.setOptions(new CqlTranslatorOptions().withCqlCompilerOptions(options));
// convert to base64 bytes
// NOTE: Publication tooling requires XML content
result.setCql(Files.readAllBytes(file.toPath()));
result.setElm(translator.toXml().getBytes());
result.setIdentifier(translator.toELM().getIdentifier());
result.setJsonElm(translator.toJson().getBytes());
Expand Down
Loading

0 comments on commit 0949d9b

Please sign in to comment.