diff --git a/JeMPI_Apps/JeMPI_Linker/checkstyle/suppression.xml b/JeMPI_Apps/JeMPI_Linker/checkstyle/suppression.xml
index 72dcc0be1..314de0e1c 100644
--- a/JeMPI_Apps/JeMPI_Linker/checkstyle/suppression.xml
+++ b/JeMPI_Apps/JeMPI_Linker/checkstyle/suppression.xml
@@ -32,7 +32,7 @@
/>
diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java
index 1a3976ca4..6d30d9206 100644
--- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java
+++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/AppConfig.java
@@ -44,6 +44,8 @@ public final class AppConfig {
public static final String API_IP = CONFIG.getString("API_IP");
public static final String API_HTTP_PORT = CONFIG.getString("API_HTTP_PORT");
public static final Float LINKER_MATCH_THRESHOLD = (float) CONFIG.getDouble("LINKER_MATCH_THRESHOLD");
+ public static final Float LINKER_MIN_THRESHOLD = (float) CONFIG.getDouble("LINKER_MIN_THRESHOLD");
+ public static final Float LINKER_MAX_THRESHOLD = (float) CONFIG.getDouble("LINKER_MAX_THRESHOLD");
public static final Float LINKER_MATCH_THRESHOLD_MARGIN = (float) CONFIG.getDouble("LINKER_MATCH_THRESHOLD_MARGIN");
public static final Level GET_LOG_LEVEL = Level.toLevel(CONFIG.getString("LOG4J2_LEVEL"));
diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/BackEnd.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/BackEnd.java
index f8d69bf72..ebbb59406 100644
--- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/BackEnd.java
+++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/BackEnd.java
@@ -187,6 +187,9 @@ private Behavior syncLinkInteractionHandler(final SyncLinkInteractionRe
request.link.matchThreshold() == null
? AppConfig.LINKER_MATCH_THRESHOLD
: request.link.matchThreshold(),
+ request.link.externalLinkRange().low(),
+ request.link.externalLinkRange().high(),
+ AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
request.link.stan());
request.replyTo.tell(new SyncLinkInteractionResponse(request.link.stan(),
listLinkInfo.isRight()
@@ -209,14 +212,25 @@ private Behavior asyncLinkInteractionHandler(final AsyncLinkInteraction
return Behaviors.same();
});
}
+ final var uploadConfig = req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig();
final var linkInfo =
LinkerDWH.linkInteraction(libMPI,
- req.batchInteraction.interaction(),
- null,
- req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig() != null
- ? req.batchInteraction.sessionMetadata().commonMetaData().uploadConfig().linkThreshold().floatValue()
- : AppConfig.LINKER_MATCH_THRESHOLD,
- req.batchInteraction.stan());
+ req.batchInteraction.interaction(),
+ null,
+ uploadConfig != null
+ ? uploadConfig.linkThreshold().floatValue()
+ : AppConfig.LINKER_MATCH_THRESHOLD,
+ uploadConfig != null
+ ? uploadConfig.minThreshold().floatValue()
+ : AppConfig.LINKER_MIN_THRESHOLD,
+ uploadConfig != null
+ ? uploadConfig.maxThreshold().floatValue()
+ : AppConfig.LINKER_MAX_THRESHOLD,
+ uploadConfig != null
+ ? uploadConfig.marginWindowSize().floatValue()
+ : AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
+ req.batchInteraction.stan());
+ req.batchInteraction.stan();
if (linkInfo.isRight()) {
req.replyTo.tell(new AsyncLinkInteractionResponse(linkInfo.get()));
} else {
@@ -235,46 +249,6 @@ private Behavior asyncLinkInteractionHandler(final AsyncLinkInteraction
});
}
-/*
- private Behavior syncLinkInteractionToGidHandler(final SyncLinkInteractionToGidRequest request) {
- final LinkInfo linkInfo;
- final var interaction = request.link.interaction();
- final var gid = request.link.gid();
- try {
- // Check if we have new M&U values
- LinkerProbabilistic.checkUpdatedMU();
-
- libMPI.startTransaction();
- if (StringUtils.isBlank(gid)) {
- linkInfo = libMPI.createInteractionAndLinkToClonedGoldenRecord(interaction, 1.0F);
- } else {
- final var goldenRecord = libMPI.findGoldenRecord(gid);
- if (goldenRecord == null) {
- LOGGER.error("Golden Record for GID {} is null", gid);
- linkInfo = null;
- } else {
- final var validated1 = CustomLinkerDeterministic.validateDeterministicMatch(goldenRecord.demographicData(),
- interaction.demographicData());
- final var validated2 = CustomLinkerProbabilistic.validateProbabilisticScore(goldenRecord.demographicData(),
- interaction.demographicData());
-
- linkInfo = libMPI.createInteractionAndLinkToExistingGoldenRecord(interaction,
- new LibMPIClientInterface.GoldenIdScore(gid,
- 3.0F),
- validated1, validated2);
- if (Boolean.TRUE.equals(goldenRecord.customUniqueGoldenRecordData().auxAutoUpdateEnabled())) {
- CustomLinkerBackEnd.updateGoldenRecordFields(libMPI, 0.0F, linkInfo.interactionUID(), gid);
- }
- }
- }
- } finally {
- libMPI.closeTransaction();
- }
- request.replyTo.tell(new SyncLinkInteractionToGidResponse(request.link.stan(), linkInfo));
- return Behaviors.same();
- }
-*/
-
private Behavior workTimeHandler(final WorkTimeRequest request) {
LOGGER.info("WORK TIME");
return Behaviors.same();
diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerCR.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerCR.java
index 73a1e8ad0..a18da2ed2 100644
--- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerCR.java
+++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerCR.java
@@ -177,6 +177,9 @@ private static Either crLinkBySourceIdUpdate(
demographicData),
null,
AppConfig.LINKER_MATCH_THRESHOLD,
+ AppConfig.LINKER_MIN_THRESHOLD,
+ AppConfig.LINKER_MAX_THRESHOLD,
+ AppConfig.LINKER_MATCH_THRESHOLD_MARGIN,
"STAN");
return Either.right(linkInfo.get());
diff --git a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
index 2a96dd1e7..4c2ff4957 100644
--- a/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
+++ b/JeMPI_Apps/JeMPI_Linker/src/main/java/org/jembi/jempi/linker/backend/LinkerDWH.java
@@ -263,6 +263,9 @@ static Either, LinkInfo> linkInteraction(
final Interaction interaction,
final ExternalLinkRange externalLinkRange,
final float matchThreshold_,
+ final float minThreshold_,
+ final float maxThreshold_,
+ final float marginWindowSize_,
final String envelopStan) {
LinkStatsMeta.ConfusionMatrix confusionMatrix;
@@ -283,9 +286,6 @@ static Either, LinkInfo> linkInteraction(
} else {
LinkInfo linkInfo = null;
final List externalLinkCandidateList = new ArrayList<>();
- final var matchThreshold = externalLinkRange != null
- ? externalLinkRange.high()
- : matchThreshold_;
LinkerProbabilistic.checkUpdatedLinkMU();
final var candidateGoldenRecords = libMPI.findLinkCandidates(interaction.demographicData());
LOGGER.debug("{} : {}", envelopStan, candidateGoldenRecords.size());
@@ -313,17 +313,17 @@ static Either, LinkInfo> linkInteraction(
.parallel()
.mapToObj(i -> {
final var workCandidate = allCandidateScores.get(i);
- return FieldTallies.map(i == 0 && workCandidate.score >= matchThreshold,
+ return FieldTallies.map(i == 0 && workCandidate.score >= matchThreshold_,
interaction.demographicData(),
workCandidate.goldenRecord.demographicData());
})
.reduce(CUSTOM_FIELD_TALLIES_SUM_IDENTITY, FieldTallies::sum);
final var score = allCandidateScores.getFirst().score;
- if (score >= matchThreshold + 0.1) {
+ if (score >= maxThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(1.0, 0.0, 0.0, 0.0);
- } else if (score >= matchThreshold) {
+ } else if (score >= matchThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.80, 0.20, 0.0, 0.0);
- } else if (score >= matchThreshold - 0.1) {
+ } else if (score >= minThreshold_) {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.0, 0.0, 0.20, 0.80);
} else {
confusionMatrix = new LinkStatsMeta.ConfusionMatrix(0.0, 0.0, 1.0, 0.0);
@@ -340,12 +340,12 @@ static Either, LinkInfo> linkInteraction(
final var belowThresholdNotifications = new ArrayList();
final var aboveThresholdNotifications = new ArrayList();
final var candidatesAboveMatchThreshold = allCandidateScores.stream().peek(v -> {
- if (v.score() > matchThreshold - 0.1 && v.score() < matchThreshold) {
+ if (v.score() > minThreshold_ && v.score() < matchThreshold_) {
belowThresholdNotifications.add(new Notification.MatchData(v.goldenRecord().goldenId(), v.score()));
- } else if (v.score() >= matchThreshold && v.score() < matchThreshold + 0.1) {
+ } else if (v.score() >= matchThreshold_ && v.score() < maxThreshold_) {
aboveThresholdNotifications.add(new Notification.MatchData(v.goldenRecord().goldenId(), v.score()));
}
- }).filter(v -> v.score() >= matchThreshold).collect(Collectors.toCollection(ArrayList::new));
+ }).filter(v -> v.score() >= matchThreshold_).collect(Collectors.toCollection(ArrayList::new));
if (candidatesAboveMatchThreshold.isEmpty()) {
if (candidatesInExternalLinkRange.isEmpty()) {
linkInfo = libMPI.createInteractionAndLinkToClonedGoldenRecord(interaction, 1.0F);
@@ -376,7 +376,7 @@ static Either, LinkInfo> linkInteraction(
validated1,
validated2,
firstCandidate.linkingRule());
- if (linkToGoldenId.score() <= matchThreshold + 0.1) {
+ if (linkToGoldenId.score() <= maxThreshold_) {
sendNotification(Notification.NotificationType.ABOVE_THRESHOLD,
linkInfo.interactionUID(),
patientName(interaction),
@@ -388,7 +388,7 @@ static Either, LinkInfo> linkInteraction(
}
if (Boolean.TRUE.equals(firstCandidate.goldenRecord.auxGoldenRecordData().auxAutoUpdateEnabled())) {
updateGoldenRecordFields(libMPI,
- matchThreshold,
+ matchThreshold_,
linkInfo.interactionUID(),
linkInfo.goldenUID());
}
@@ -396,7 +396,7 @@ static Either, LinkInfo> linkInteraction(
if (candidatesInExternalLinkRange.isEmpty() && candidatesAboveMatchThreshold.size() > 1) {
for (var i = 1; i < candidatesAboveMatchThreshold.size(); i++) {
final var candidate = candidatesAboveMatchThreshold.get(i);
- if (firstCandidate.score - candidate.score <= 0.1) {
+ if (firstCandidate.score - candidate.score <= marginWindowSize_) {
marginCandidates.add(new Notification.MatchData(candidate.goldenRecord.goldenId(), candidate.score));
} else {
break;
diff --git a/devops/linux/docker/conf/stack/docker-stack-high-0.yml b/devops/linux/docker/conf/stack/docker-stack-high-0.yml
index 50ff7ba35..8fe72873c 100644
--- a/devops/linux/docker/conf/stack/docker-stack-high-0.yml
+++ b/devops/linux/docker/conf/stack/docker-stack-high-0.yml
@@ -611,7 +611,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
- LINKER_MATCH_THRESHOLD: 0.65
+ LINKER_MATCH_THRESHOLD: 0.7
+ LINKER_MIN_THRESHOLD: 0.65
+ LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
diff --git a/devops/linux/docker/conf/stack/docker-stack-high-1.yml b/devops/linux/docker/conf/stack/docker-stack-high-1.yml
index c9568b1c3..6ab2d9688 100644
--- a/devops/linux/docker/conf/stack/docker-stack-high-1.yml
+++ b/devops/linux/docker/conf/stack/docker-stack-high-1.yml
@@ -611,7 +611,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
- LINKER_MATCH_THRESHOLD: 0.65
+ LINKER_MATCH_THRESHOLD: 0.7
+ LINKER_MIN_THRESHOLD: 0.65
+ LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
diff --git a/devops/linux/docker/conf/stack/docker-stack-low-0.yml b/devops/linux/docker/conf/stack/docker-stack-low-0.yml
index e4db8c78a..f8c4d1daf 100644
--- a/devops/linux/docker/conf/stack/docker-stack-low-0.yml
+++ b/devops/linux/docker/conf/stack/docker-stack-low-0.yml
@@ -400,7 +400,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
- LINKER_MATCH_THRESHOLD: 0.65
+ LINKER_MATCH_THRESHOLD: 0.7
+ LINKER_MIN_THRESHOLD: 0.65
+ LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend
diff --git a/devops/linux/docker/conf/stack/docker-stack-low-1.yml b/devops/linux/docker/conf/stack/docker-stack-low-1.yml
index d31898ae7..f731c3997 100644
--- a/devops/linux/docker/conf/stack/docker-stack-low-1.yml
+++ b/devops/linux/docker/conf/stack/docker-stack-low-1.yml
@@ -400,7 +400,9 @@ services:
LINKER_HTTP_PORT: 50000
API_IP: api
API_HTTP_PORT: 50000
- LINKER_MATCH_THRESHOLD: 0.65
+ LINKER_MATCH_THRESHOLD: 0.7
+ LINKER_MIN_THRESHOLD: 0.65
+ LINKER_MAX_THRESHOLD: 0.75
LINKER_MATCH_THRESHOLD_MARGIN: 0.1
networks:
- backend