diff --git a/src/neuroglancer/annotation/user_layer.ts b/src/neuroglancer/annotation/user_layer.ts
index 9a9b93b08..8a5b49219 100644
--- a/src/neuroglancer/annotation/user_layer.ts
+++ b/src/neuroglancer/annotation/user_layer.ts
@@ -135,9 +135,11 @@ class LinkedSegmentationLayers extends RefCounted {
       state.seenGeneration = generation;
     }
     if (!isLoading) {
+      const {relationshipStates} = this.annotationDisplayState;
       for (const [relationship, state] of map) {
         if (state.seenGeneration !== generation) {
           map.delete(relationship);
+          relationshipStates.delete(relationship);
           changed = true;
         }
       }