diff --git a/pkg/syncer/resource_syncer.go b/pkg/syncer/resource_syncer.go
index 8deeccd0..cc69ce2e 100644
--- a/pkg/syncer/resource_syncer.go
+++ b/pkg/syncer/resource_syncer.go
@@ -195,6 +195,7 @@ type ResourceSyncerConfig struct {
 type resourceSyncer struct {
 	workQueue         workqueue.Interface
 	hasSynced         func() bool
+	unregHandler      func()
 	informer          cache.Controller
 	store             cache.Store
 	config            ResourceSyncerConfig
@@ -261,6 +262,10 @@ func NewResourceSyncerWithSharedInformer(config *ResourceSyncerConfig, informer
 
 	syncer.hasSynced = reg.HasSynced
 
+	syncer.unregHandler = func() {
+		_ = informer.RemoveEventHandler(reg)
+	}
+
 	return syncer, nil
 }
 
@@ -349,6 +354,10 @@ func (r *resourceSyncer) Start(stopCh <-chan struct{}) error {
 				prometheus.Unregister(r.syncCounter)
 			}
 
+			if r.unregHandler != nil {
+				r.unregHandler()
+			}
+
 			r.stopped <- struct{}{}
 			r.log.V(log.LIBDEBUG).Infof("Syncer %q stopped", r.config.Name)
 		}()