Remove duplicated entried from relationships and nodes (#1333)

microsoft · Oct 29, 2024 · 83026bd · 83026bd
1 parent 083de12
commit 83026bd
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/.semversioner/next-release/patch-20241029003722814474.json b/.semversioner/next-release/patch-20241029003722814474.json
@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "Remove duplicated relationships and nodes"
+}
diff --git a/graphrag/index/flows/create_final_nodes.py b/graphrag/index/flows/create_final_nodes.py
@@ -69,4 +69,5 @@ async def create_final_nodes(
     )
     joined.rename(columns={"label": "title", "cluster": "community"}, inplace=True)
 
-    return joined
+    # TODO: Find duplication source
+    return joined.drop_duplicates(subset=["title", "community"])
diff --git a/graphrag/index/flows/create_final_relationships.py b/graphrag/index/flows/create_final_relationships.py
@@ -66,4 +66,5 @@ async def create_final_relationships(
         "text_unit_ids"
     ].str.split(",")
 
-    return edge_combined_degree
+    # TODO: Find duplication source
+    return edge_combined_degree.drop_duplicates(subset=["source", "target"])