diff --git a/controllers/runner_controller.go b/controllers/runner_controller.go
index 3753819493..03dbe50ec8 100644
--- a/controllers/runner_controller.go
+++ b/controllers/runner_controller.go
@@ -19,7 +19,7 @@ package controllers
 import (
 	"context"
 	"fmt"
-	"reflect"
+	"github.com/summerwind/actions-runner-controller/hash"
 	"strings"
 
 	"github.com/go-logr/logr"
@@ -39,6 +39,8 @@ import (
 const (
 	containerName = "runner"
 	finalizerName = "runner.actions.summerwind.dev"
+
+	runnerHashAnnotationKey = "runner-hash"
 )
 
 // RunnerReconciler reconciles a Runner object
@@ -198,7 +200,10 @@ func (r *RunnerReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			return ctrl.Result{}, nil
 		}
 
-		if !runnerBusy && (!reflect.DeepEqual(pod.Spec.Containers[0].Env, newPod.Spec.Containers[0].Env) || pod.Spec.Containers[0].Image != newPod.Spec.Containers[0].Image) {
+		curHash := pod.Annotations[runnerHashAnnotationKey]
+		newHash := newPod.Annotations[runnerHashAnnotationKey]
+
+		if !runnerBusy && curHash != newHash {
 			restart = true
 		}
 
@@ -357,12 +362,25 @@ func (r *RunnerReconciler) newPod(runner v1alpha1.Runner) (corev1.Pod, error) {
 	}
 
 	env = append(env, runner.Spec.Env...)
+
+	annotations := map[string]string{}
+
+	for k, v := range runner.Annotations {
+		annotations[k] = v
+	}
+
+	// This implies that we recreate the runner pod whenever the runner has changes in:
+	// - metadata.labels
+	// - metadata.annotations
+	// - metadata.spec (including image, env, organization, repository, group, token, and so on)
+	annotations[runnerHashAnnotationKey] = hash.FNVHashStringObjects(runner.Labels, runner.Annotations, runner.Spec)
+
 	pod := corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        runner.Name,
 			Namespace:   runner.Namespace,
 			Labels:      runner.Labels,
-			Annotations: runner.Annotations,
+			Annotations: annotations,
 		},
 		Spec: corev1.PodSpec{
 			RestartPolicy: "OnFailure",
diff --git a/hash/fnv.go b/hash/fnv.go
new file mode 100644
index 0000000000..a8382544a7
--- /dev/null
+++ b/hash/fnv.go
@@ -0,0 +1,17 @@
+package hash
+
+import (
+	"fmt"
+	"hash/fnv"
+	"k8s.io/apimachinery/pkg/util/rand"
+)
+
+func FNVHashStringObjects(objs ...interface{}) string {
+	hash := fnv.New32a()
+
+	for _, obj := range objs {
+		DeepHashObject(hash, obj)
+	}
+
+	return rand.SafeEncodeString(fmt.Sprint(hash.Sum32()))
+}
diff --git a/hash/hash.go b/hash/hash.go
new file mode 100644
index 0000000000..a6c3e1c62f
--- /dev/null
+++ b/hash/hash.go
@@ -0,0 +1,25 @@
+// Copyright 2015 The Kubernetes Authors.
+// hash.go is copied from kubernetes's pkg/util/hash.go
+// See https://github.com/kubernetes/kubernetes/blob/e1c617a88ec286f5f6cb2589d6ac562d095e1068/pkg/util/hash/hash.go#L25-L37
+
+package hash
+
+import (
+	"hash"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+// DeepHashObject writes specified object to hash using the spew library
+// which follows pointers and prints actual values of the nested objects
+// ensuring the hash does not change when a pointer changes.
+func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) {
+	hasher.Reset()
+	printer := spew.ConfigState{
+		Indent:         " ",
+		SortKeys:       true,
+		DisableMethods: true,
+		SpewKeys:       true,
+	}
+	printer.Fprintf(hasher, "%#v", objectToWrite)
+}