Deduplication engine integration demo fixes (#4253)

unicef · Sep 26, 2024 · fef8aa6 · fef8aa6
1 parent 629a32b
commit fef8aa6
Show file tree

Hide file tree

Showing 38 changed files with 1,022 additions and 561 deletions.
diff --git a/src/frontend/data/schema.graphql b/src/frontend/data/schema.graphql
@@ -895,26 +895,16 @@ scalar DateTime
 
 scalar Decimal
 
-type DeduplicationEngineSimilarityPairNode implements Node {
-  id: ID!
-  program: ProgramNode!
-  individual1: IndividualNode!
-  individual2: IndividualNode!
-  similarityScore: String
-  ticketneedsadjudicationdetailsSet(offset: Int, before: String, after: String, first: Int, last: Int): TicketNeedsAdjudicationDetailsNodeConnection!
-  isDuplicate: Boolean
-}
-
-type DeduplicationEngineSimilarityPairNodeConnection {
-  pageInfo: PageInfo!
-  edges: [DeduplicationEngineSimilarityPairNodeEdge]!
-  totalCount: Int
-  edgeCount: Int
+type DeduplicationEngineSimilarityPairIndividualNode {
+  photo: String
+  fullName: String
+  unicefId: String
 }
 
-type DeduplicationEngineSimilarityPairNodeEdge {
-  node: DeduplicationEngineSimilarityPairNode
-  cursor: String!
+type DeduplicationEngineSimilarityPairNode {
+  individual1: DeduplicationEngineSimilarityPairIndividualNode
+  individual2: DeduplicationEngineSimilarityPairIndividualNode
+  similarityScore: String
 }
 
 type DeduplicationResultNode {
@@ -2518,8 +2508,6 @@ type IndividualNode implements Node {
   householdsAndRoles: [IndividualRoleInHouseholdNode!]!
   copiedTo(offset: Int, before: String, after: String, first: Int, last: Int): IndividualNodeConnection!
   bankAccountInfo: BankAccountInfoNode
-  biometricDuplicates1(offset: Int, before: String, after: String, first: Int, last: Int): DeduplicationEngineSimilarityPairNodeConnection!
-  biometricDuplicates2(offset: Int, before: String, after: String, first: Int, last: Int): DeduplicationEngineSimilarityPairNodeConnection!
   paymentrecordSet(offset: Int, before: String, after: String, first: Int, last: Int): PaymentRecordNodeConnection!
   collectorPayments(offset: Int, before: String, after: String, first: Int, last: Int): PaymentNodeConnection!
   paymentSet(offset: Int, before: String, after: String, first: Int, last: Int): PaymentNodeConnection!
@@ -3774,7 +3762,6 @@ type ProgramNode implements Node {
   householdSet(offset: Int, before: String, after: String, first: Int, last: Int): HouseholdNodeConnection!
   individuals(offset: Int, before: String, after: String, first: Int, last: Int): IndividualNodeConnection!
   registrationImports(offset: Int, before: String, after: String, first: Int, last: Int): RegistrationDataImportNodeConnection!
-  deduplicationEngineSimilarityPairs(offset: Int, before: String, after: String, first: Int, last: Int): DeduplicationEngineSimilarityPairNodeConnection!
   paymentplanSet(offset: Int, before: String, after: String, first: Int, last: Int): PaymentPlanNodeConnection!
   cashplanSet(offset: Int, before: String, after: String, first: Int, last: Int): CashPlanNodeConnection!
   paymentSet(offset: Int, before: String, after: String, first: Int, last: Int): PaymentNodeConnection!
@@ -4185,7 +4172,7 @@ type RegistrationDataImportNode implements Node {
   goldenRecordPossibleDuplicatesCountAndPercentage: [CountAndPercentageNode]
   goldenRecordUniqueCountAndPercentage: [CountAndPercentageNode]
   totalHouseholdsCountWithValidPhoneNo: Int
-  isDeduplicated: String
+  biometricDeduplicated: String
   canMerge: Boolean
   biometricDeduplicationEnabled: Boolean
 }
@@ -5020,6 +5007,7 @@ type TicketIndividualDataUpdateDetailsNodeEdge {
 type TicketNeedsAdjudicationDetailsExtraDataNode {
   goldenRecords: [DeduplicationResultNode]
   possibleDuplicate: [DeduplicationResultNode]
+  dedupEngineSimilarityPair: DeduplicationEngineSimilarityPairNode
 }
 
 type TicketNeedsAdjudicationDetailsNode implements Node {
@@ -5036,7 +5024,6 @@ type TicketNeedsAdjudicationDetailsNode implements Node {
   scoreMin: Float!
   scoreMax: Float!
   isCrossArea: Boolean!
-  dedupEngineSimilarityPair: DeduplicationEngineSimilarityPairNode
   selectedIndividual: IndividualNode
   possibleDuplicate: IndividualNode
   hasDuplicatedDocument: Boolean

diff --git a/src/frontend/package.json b/src/frontend/package.json
@@ -12,7 +12,7 @@
     "test": "TZ=UTC jest --config jest.config.ts",
     "preview": "vite preview",
     "download-dev-schema": "wget --no-check-certificate -O data/schema.graphql https://dev-hct.unitst.org/api/graphql/schema.graphql",
-    "download-local-schema": "wget --no-check-certificate -O data/schema.graphql http://localhost:8080/api/graphql/schema.graphql",
+    "download-local-schema": "wget --no-check-certificate -O data/schema.graphql http://localhost:3000/api/graphql/schema.graphql",
     "generate-types": "yarn download-dev-schema && graphql-codegen --config codegen.yml --debug",
     "generate-types-local": "yarn download-local-schema && graphql-codegen --config codegen.yml --debug"
   },

diff --git a/src/frontend/src/__generated__/graphql.tsx b/src/frontend/src/__generated__/graphql.tsx
diff --git a/src/frontend/src/__generated__/introspection-result.ts b/src/frontend/src/__generated__/introspection-result.ts
diff --git a/src/frontend/src/apollo/fragments/GrievanceTicketFragment.ts b/src/frontend/src/apollo/fragments/GrievanceTicketFragment.ts
@@ -207,20 +207,6 @@ export const grievanceTicketDetailed = gql`
     }
     needsAdjudicationTicketDetails {
       id
-      dedupEngineSimilarityPair {
-        isDuplicate
-        similarityScore
-        individual1 {
-          unicefId
-          fullName
-          photo
-        }
-        individual2 {
-          unicefId
-          fullName
-          photo
-        }
-      }
       hasDuplicatedDocument
       extraData {
         goldenRecords {
@@ -233,6 +219,19 @@ export const grievanceTicketDetailed = gql`
           proximityToScore
           score
         }
+        dedupEngineSimilarityPair {
+          similarityScore
+          individual1 {
+            unicefId
+            fullName
+            photo
+          }
+          individual2 {
+            unicefId
+            fullName
+            photo
+          }
+        }
       }
       goldenRecordsIndividual {
         id

diff --git a/src/frontend/src/apollo/fragments/RegistrationFragments.ts b/src/frontend/src/apollo/fragments/RegistrationFragments.ts
@@ -27,7 +27,7 @@ export const registrationMinimal = gql`
     refuseReason
     totalHouseholdsCountWithValidPhoneNo
     adminUrl
-    isDeduplicated
+    biometricDeduplicated
   }
 `;
 

diff --git a/src/frontend/src/components/grievances/GrievancesTable/GrievancesFilters.tsx b/src/frontend/src/components/grievances/GrievancesTable/GrievancesFilters.tsx
@@ -90,6 +90,7 @@ export const GrievancesFilters = ({
   const showIssueType =
     filter.category === GRIEVANCE_CATEGORIES.SENSITIVE_GRIEVANCE ||
     filter.category === GRIEVANCE_CATEGORIES.DATA_CHANGE ||
+    filter.category === GRIEVANCE_CATEGORIES.NEEDS_ADJUDICATION ||
     filter.category === GRIEVANCE_CATEGORIES.GRIEVANCE_COMPLAINT;
 
   const updatedPriorityChoices = useMemo(() => {

diff --git a/src/frontend/src/components/grievances/NeedsAdjudication/BiometricsResults.tsx b/src/frontend/src/components/grievances/NeedsAdjudication/BiometricsResults.tsx
@@ -19,16 +19,15 @@ import { useTranslation } from 'react-i18next';
 import { hasPermissions, PERMISSIONS } from 'src/config/permissions';
 
 export interface Individual {
-  __typename?: 'IndividualNode';
+  __typename?: 'DeduplicationEngineSimilarityPairIndividualNode';
   unicefId?: string;
-  fullName: string;
+  fullName?: string;
   photo?: string;
 }
 
 export interface BiometricsResultsProps {
   ticketId: string;
   similarityScore: string;
-  faceMatchResult: 'Duplicates' | 'Uniqueness';
   individual1?: Individual;
   individual2?: Individual;
 }
@@ -50,7 +49,6 @@ const Placeholder: React.FC = () => (
 export const BiometricsResults = ({
   ticketId,
   similarityScore,
-  faceMatchResult,
   individual1,
   individual2,
 }: BiometricsResultsProps): React.ReactElement => {
@@ -146,7 +144,7 @@ export const BiometricsResults = ({
                 </strong>
               </div>
               <div>
-                {t('Face images matching suggests:')} {faceMatchResult}
+                {t('Face images matching suggests: Duplicates')}
               </div>
             </Box>
           </DialogContainer>

diff --git a/src/frontend/src/components/grievances/NeedsAdjudication/NeedsAdjudicationActions.tsx b/src/frontend/src/components/grievances/NeedsAdjudication/NeedsAdjudicationActions.tsx
@@ -52,7 +52,7 @@ export const NeedsAdjudicationActions: React.FC<
   const { isActiveProgram } = useProgramContext();
   const actionsDisabled =
     !isTicketForApproval || !isActiveProgram || !selectedIndividualIds.length;
-  const { dedupEngineSimilarityPair } = ticket.needsAdjudicationTicketDetails;
+  const { dedupEngineSimilarityPair } = ticket.needsAdjudicationTicketDetails.extraData;
 
   return (
     <Box
@@ -90,11 +90,6 @@ export const NeedsAdjudicationActions: React.FC<
           <BiometricsResults
             ticketId={ticket.id}
             similarityScore={dedupEngineSimilarityPair.similarityScore}
-            faceMatchResult={
-              dedupEngineSimilarityPair.isDuplicate
-                ? t('Duplicates')
-                : t('Uniqueness')
-            }
             individual1={dedupEngineSimilarityPair.individual1}
             individual2={dedupEngineSimilarityPair.individual2}
           />

diff --git a/src/frontend/src/containers/pages/rdi/RegistrationDataImportPage.tsx b/src/frontend/src/containers/pages/rdi/RegistrationDataImportPage.tsx
@@ -78,7 +78,7 @@ export function RegistrationDataImportPage(): React.ReactElement {
               disabled={deduplicationFlags.isDeduplicationDisabled}
               title={t('Deduplication engine already in progress')}
             >
-              {t('RUN DEDUPLICATION ENGINE')}
+              {t('START DEDUPLICATION ENGINE')}
             </ButtonTooltip>
           </Box>
         )}

diff --git a/src/frontend/src/containers/pages/rdi/people/PeopleRegistrationDataImportPage.tsx b/src/frontend/src/containers/pages/rdi/people/PeopleRegistrationDataImportPage.tsx
@@ -78,7 +78,7 @@ export function PeopleRegistrationDataImportPage(): React.ReactElement {
               disabled={deduplicationFlags.isDeduplicationDisabled}
               title={t('Deduplication engine already in progress')}
             >
-              {t('RUN DEDUPLICATION ENGINE')}
+              {t('START DEDUPLICATION ENGINE')}
             </ButtonTooltip>
           </Box>
         )}

diff --git a/...di/RegistrationDataImportForPeopleTable/RegistrationDataImportForPeopleTableHeadCells.tsx b/...di/RegistrationDataImportForPeopleTable/RegistrationDataImportForPeopleTableHeadCells.tsx
@@ -29,8 +29,8 @@ export const headCells: HeadCell<RegistrationDataImportNode>[] = [
   },
   {
     disablePadding: false,
-    label: 'Is Deduplicated?',
-    id: 'isDeduplicated',
+    label: 'Biometric Deduplicated',
+    id: 'biometricDeduplicated',
     numeric: false,
     disableSort: true,
   },

diff --git a/...bles/rdi/RegistrationDataImportForPeopleTable/RegistrationDataImportForPeopleTableRow.tsx b/...bles/rdi/RegistrationDataImportForPeopleTable/RegistrationDataImportForPeopleTableRow.tsx
@@ -80,7 +80,7 @@ export function RegistrationDataImportForPeopleTableRow({
         </UniversalMoment>
       </TableCell>
       <TableCell align="center">
-        {registrationDataImport.isDeduplicated}
+        {registrationDataImport.biometricDeduplicated}
       </TableCell>
       <TableCell align="right">
         {registrationDataImport.numberOfIndividuals}

diff --git a/...DataImportForPeopleTable/__snapshots__/RegistrationDataImportForPeopleTable.test.tsx.snap b/...DataImportForPeopleTable/__snapshots__/RegistrationDataImportForPeopleTable.test.tsx.snap
@@ -126,7 +126,7 @@ exports[`containers/tables/rdi/RegistrationDataImportTable should render loading
                     class="sc-iBdnpw jMIPuq"
                     data-cy="table-label"
                   >
-                    Is Deduplicated?
+                    Biometric Deduplicated
                   </span>
                 </th>
                 <th
@@ -580,7 +580,7 @@ exports[`containers/tables/rdi/RegistrationDataImportTable should render with da
                     class="sc-iBdnpw jMIPuq"
                     data-cy="table-label"
                   >
-                    Is Deduplicated?
+                    Biometric Deduplicated
                   </span>
                 </th>
                 <th

diff --git a/...end/src/containers/tables/rdi/RegistrationDataImportTable/RegistrationDataImportTable.tsx b/...end/src/containers/tables/rdi/RegistrationDataImportTable/RegistrationDataImportTable.tsx
@@ -69,8 +69,8 @@ export function RegistrationDataImportTable({
     if (deduplicationFlags?.canRunDeduplication) {
       header.splice(4, 0, {
         disablePadding: false,
-        label: 'Is Deduplicated?',
-        id: 'isDeduplicated',
+        label: 'Biometric Deduplicated',
+        id: 'biometricDeduplicated',
         numeric: false,
         disableSort: true,
       });

diff --git a/.../src/containers/tables/rdi/RegistrationDataImportTable/RegistrationDataImportTableRow.tsx b/.../src/containers/tables/rdi/RegistrationDataImportTable/RegistrationDataImportTableRow.tsx
@@ -87,7 +87,7 @@ export function RegistrationDataImportTableRow({
       </TableCell>
       {biometricDeduplicationEnabled && (
         <TableCell align="center">
-          {registrationDataImport.isDeduplicated}
+          {registrationDataImport.biometricDeduplicated}
         </TableCell>
       )}
       <TableCell align="right">

diff --git a/src/hct_mis_api/apps/grievance/migrations/0073_migration.py b/src/hct_mis_api/apps/grievance/migrations/0073_migration.py
@@ -0,0 +1,17 @@
+# Generated by Django 3.2.25 on 2024-09-25 17:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('grievance', '0072_migration'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='ticketneedsadjudicationdetails',
+            name='dedup_engine_similarity_pair',
+        ),
+    ]
diff --git a/src/hct_mis_api/apps/grievance/models.py b/src/hct_mis_api/apps/grievance/models.py
@@ -834,9 +834,6 @@ class TicketNeedsAdjudicationDetails(TimeStampedUUIDModel):
     score_min = models.FloatField(default=0.0)
     score_max = models.FloatField(default=0.0)
     is_cross_area = models.BooleanField(default=False)
-    dedup_engine_similarity_pair = models.ForeignKey(
-        "registration_data.DeduplicationEngineSimilarityPair", on_delete=models.CASCADE, null=True
-    )
 
     # deprecated and will remove soon
     selected_individual = models.ForeignKey(

diff --git a/src/hct_mis_api/apps/grievance/schema.py b/src/hct_mis_api/apps/grievance/schema.py
@@ -75,7 +75,10 @@
 from hct_mis_api.apps.payment.schema import PaymentRecordAndPaymentNode
 from hct_mis_api.apps.program.models import Program
 from hct_mis_api.apps.program.schema import ProgramNode
-from hct_mis_api.apps.registration_data.nodes import DeduplicationResultNode
+from hct_mis_api.apps.registration_data.nodes import (
+    DeduplicationEngineSimilarityPairNode,
+    DeduplicationResultNode,
+)
 from hct_mis_api.apps.utils.exceptions import log_and_raise
 from hct_mis_api.apps.utils.schema import Arg, ChartDatasetNode
 
@@ -395,6 +398,7 @@ def resolve_household_data(parent: TicketHouseholdDataUpdateDetails, info: Any)
 class TicketNeedsAdjudicationDetailsExtraDataNode(graphene.ObjectType):
     golden_records = graphene.List(DeduplicationResultNode)
     possible_duplicate = graphene.List(DeduplicationResultNode)
+    dedup_engine_similarity_pair = graphene.Field(DeduplicationEngineSimilarityPairNode)
 
     def resolve_golden_records(self, info: Any) -> List[Dict]:
         return encode_ids(self.golden_records, "Individual", "hit_id")
@@ -419,7 +423,10 @@ class Meta:
     def resolve_extra_data(parent, info: Any) -> TicketNeedsAdjudicationDetailsExtraDataNode:
         golden_records = parent.extra_data.get("golden_records")
         possible_duplicate = parent.extra_data.get("possible_duplicate")
-        return TicketNeedsAdjudicationDetailsExtraDataNode(golden_records, possible_duplicate)
+        dedup_engine_similarity_pair = parent.extra_data.get("dedup_engine_similarity_pair")
+        return TicketNeedsAdjudicationDetailsExtraDataNode(
+            golden_records, possible_duplicate, dedup_engine_similarity_pair
+        )
 
     def resolve_possible_duplicates(self, info: Any) -> QuerySet:
         return self.possible_duplicates.all()

diff --git a/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py b/src/hct_mis_api/apps/grievance/services/needs_adjudication_ticket_services.py
@@ -1,3 +1,4 @@
+import logging
 from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple
 
 from django.contrib.auth.models import AbstractUser
@@ -43,6 +44,9 @@
     from hct_mis_api.apps.program.models import Program
 
 
+logger = logging.getLogger(__name__)
+
+
 def _clear_deduplication_individuals_fields(individuals: Sequence[Individual]) -> None:
     for individual in individuals:
         individual.deduplication_golden_record_status = UNIQUE
@@ -97,6 +101,22 @@ def close_needs_adjudication_new_ticket(ticket_details: TicketNeedsAdjudicationD
             mark_as_distinct_individual(individual_to_distinct, user, ticket_details.ticket.programs.all())
         _clear_deduplication_individuals_fields(distinct_individuals)
 
+    if ticket_details.ticket.issue_type == GrievanceTicket.ISSUE_TYPE_BIOMETRICS_SIMILARITY:
+        # both individuals are distinct, report false positive
+        if not duplicate_individuals and distinct_individuals:
+            from hct_mis_api.apps.registration_datahub.services.biometric_deduplication import (
+                BiometricDeduplicationService,
+            )
+
+            ids = [str(individual.id) for individual in distinct_individuals]
+            service = BiometricDeduplicationService()
+            try:
+                service.report_false_positive_duplicate(
+                    ids[0], ids[1], ticket_details.ticket.registration_data_import.program.deduplication_set_id
+                )
+            except service.api.API_EXCEPTION_CLASS:
+                logger.exception("Failed to report false positive duplicate to Deduplication Engine")
+
 
 def close_needs_adjudication_ticket_service(grievance_ticket: GrievanceTicket, user: AbstractUser) -> None:
     ticket_details = grievance_ticket.ticket_details
@@ -168,6 +188,8 @@ def create_grievance_ticket_with_details(
         "golden_records": golden_records,
         "possible_duplicate": possible_duplicate.get_deduplication_golden_record(),
     }
+    if dedup_engine_similarity_pair:
+        extra_data["dedup_engine_similarity_pair"] = dedup_engine_similarity_pair.serialize_for_ticket()  # type: ignore
     score_min, score_max = _get_min_max_score(golden_records)
     ticket_details = TicketNeedsAdjudicationDetails.objects.create(
         ticket=ticket,
@@ -178,7 +200,6 @@ def create_grievance_ticket_with_details(
         extra_data=extra_data,
         score_min=score_min,
         score_max=score_max,
-        dedup_engine_similarity_pair=dedup_engine_similarity_pair,
     )
 
     ticket_details.possible_duplicates.add(*possible_duplicates)