diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go
index 9730579fa920..fce2677ca922 100644
--- a/pkg/detector/detector.go
+++ b/pkg/detector/detector.go
@@ -1124,6 +1124,7 @@ func (d *ResourceDetector) CleanupLabels(objRef workv1alpha2.ObjectReference, la
 		return err
 	}
 
+	workload = workload.DeepCopy()
 	workloadLabels := workload.GetLabels()
 	for _, l := range labels {
 		delete(workloadLabels, l)
diff --git a/pkg/detector/policy.go b/pkg/detector/policy.go
index e25b7f77743a..c38262712642 100644
--- a/pkg/detector/policy.go
+++ b/pkg/detector/policy.go
@@ -272,9 +272,13 @@ func (d *ResourceDetector) removeResourceLabelsIfNotMatch(objectReference workv1
 		return false, nil
 	}
 
+	object = object.DeepCopy()
+	objLabels := object.GetLabels()
 	for _, labelKey := range labelKeys {
-		util.RemoveLabel(object, labelKey)
+		delete(objLabels, labelKey)
 	}
+	object.SetLabels(objLabels)
+
 	err = d.Client.Update(context.TODO(), object)
 	if err != nil {
 		return false, err