diff --git a/agent/agent.go b/agent/agent.go
index 6d0e4db..2c78d58 100644
--- a/agent/agent.go
+++ b/agent/agent.go
@@ -19,6 +19,7 @@ import (
 
 	keyfile "github.com/foxboron/go-tpm-keyfiles"
 	"github.com/foxboron/ssh-tpm-agent/key"
+	"github.com/foxboron/ssh-tpm-agent/signer"
 	"github.com/google/go-tpm/tpm2/transport"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
@@ -97,10 +98,12 @@ func (a *Agent) signers() ([]ssh.Signer, error) {
 	}
 
 	for _, k := range a.keys {
-		s, err := ssh.NewSignerFromSigner(keyfile.NewTPMKeySigner(k.TPMKey, a.op, a.tpm, func(k *keyfile.TPMKey) ([]byte, error) {
-			// Shimming the function to get the correct type
-			return a.pin(&key.SSHTPMKey{TPMKey: k})
-		}))
+		s, err := ssh.NewSignerFromSigner(
+			signer.NewSSHKeySigner(k, a.op, a.tpm,
+				func(_ *keyfile.TPMKey) ([]byte, error) {
+					// Shimming the function to get the correct type
+					return a.pin(k)
+				}))
 		if err != nil {
 			return nil, fmt.Errorf("failed to prepare signer: %w", err)
 		}
diff --git a/cmd/ssh-tpm-agent/main.go b/cmd/ssh-tpm-agent/main.go
index aadc0fa..9b1dd1e 100644
--- a/cmd/ssh-tpm-agent/main.go
+++ b/cmd/ssh-tpm-agent/main.go
@@ -46,6 +46,8 @@ Options:
 
     -o, --owner-password    Ask for the owner password.
 
+    --no-cache              The agent will not cache key passwords.
+
     -d                      Enable debug logging.
 
     --install-user-units    Installs systemd system units and sshd configs for using
@@ -104,6 +106,7 @@ func main() {
 		swtpmFlag, printSocketFlag       bool
 		installUserUnits, system, noLoad bool
 		askOwnerPassword, debugMode      bool
+		noCache                          bool
 	)
 
 	envSocketPath := func() string {
@@ -131,6 +134,7 @@ func main() {
 	flag.BoolVar(&askOwnerPassword, "o", false, "ask for the owner password")
 	flag.BoolVar(&askOwnerPassword, "owner-password", false, "ask for the owner password")
 	flag.BoolVar(&debugMode, "d", false, "debug mode")
+	flag.BoolVar(&noCache, "no-cache", false, "do not cache key passwords")
 	flag.Parse()
 
 	opts := &slog.HandlerOptions{
@@ -216,10 +220,21 @@ func main() {
 			}
 		},
 
-		// PIN Callback
+		// PIN Callback with caching
+		// SSHKeySigner in signer/signer.go resets this value if
+		// we get a TPMRCAuthFail
 		func(key *key.SSHTPMKey) ([]byte, error) {
+			if len(key.Userauth) != 0 {
+				slog.Debug("providing cached userauth for key", slog.String("desc", key.Description))
+				return key.Userauth, nil
+			}
 			keyInfo := fmt.Sprintf("Enter passphrase for (%s): ", key.Description)
-			return askpass.ReadPassphrase(keyInfo, askpass.RP_USE_ASKPASS)
+			userauth, err := askpass.ReadPassphrase(keyInfo, askpass.RP_USE_ASKPASS)
+			if !noCache && err == nil {
+				slog.Debug("caching userauth for key", slog.String("desc", key.Description))
+				key.Userauth = userauth
+			}
+			return userauth, err
 		},
 	)
 
diff --git a/key/key.go b/key/key.go
index bde5daf..e7307ce 100644
--- a/key/key.go
+++ b/key/key.go
@@ -18,6 +18,7 @@ var (
 // SSHTPMKey is a wrapper for TPMKey implementing the ssh.PublicKey specific parts
 type SSHTPMKey struct {
 	*keyfile.TPMKey
+	Userauth    []byte
 	Certificate *ssh.Certificate
 }
 
@@ -28,7 +29,7 @@ func NewSSHTPMKey(tpm transport.TPMCloser, alg tpm2.TPMAlgID, bits int, owneraut
 	if err != nil {
 		return nil, err
 	}
-	return &SSHTPMKey{k, nil}, nil
+	return &SSHTPMKey{k, nil, nil}, nil
 }
 
 // This assumes we are just getting a local PK.
@@ -50,7 +51,7 @@ func NewImportedSSHTPMKey(tpm transport.TPMCloser, pk any, ownerauth []byte, fn
 	if err != nil {
 		return nil, fmt.Errorf("failed turning imported key to loadable key: %v", err)
 	}
-	return &SSHTPMKey{k, nil}, nil
+	return &SSHTPMKey{k, nil, nil}, nil
 }
 
 func (k *SSHTPMKey) SSHPublicKey() (ssh.PublicKey, error) {
@@ -85,5 +86,5 @@ func Decode(b []byte) (*SSHTPMKey, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &SSHTPMKey{k, nil}, nil
+	return &SSHTPMKey{k, nil, nil}, nil
 }
diff --git a/signer/signer.go b/signer/signer.go
new file mode 100644
index 0000000..b8f4e0e
--- /dev/null
+++ b/signer/signer.go
@@ -0,0 +1,41 @@
+package signer
+
+import (
+	"crypto"
+	"errors"
+	"io"
+	"log/slog"
+
+	"github.com/google/go-tpm/tpm2"
+	"github.com/google/go-tpm/tpm2/transport"
+
+	keyfile "github.com/foxboron/go-tpm-keyfiles"
+	"github.com/foxboron/ssh-tpm-agent/key"
+)
+
+// Shim for keyfile.TPMKeySigner
+// We need access to the SSHTPMKey to change the userauth for caching
+type SSHKeySigner struct {
+	*keyfile.TPMKeySigner
+	key *key.SSHTPMKey
+}
+
+// func (t *SSHKeySigner) Public() crypto.PublicKey {
+// 	return t.TPMKeySigner.Public()
+// }
+
+func (t *SSHKeySigner) Sign(r io.Reader, digest []byte, opts crypto.SignerOpts) ([]byte, error) {
+	b, err := t.TPMKeySigner.Sign(r, digest, opts)
+	if errors.Is(err, tpm2.TPMRCAuthFail) {
+		slog.Debug("removed cached userauth for key", slog.Any("err", err), slog.String("desc", t.key.Description))
+		t.key.Userauth = []byte(nil)
+	}
+	return b, err
+}
+
+func NewSSHKeySigner(k *key.SSHTPMKey, ownerAuth func() ([]byte, error), tpm func() transport.TPMCloser, auth func(*keyfile.TPMKey) ([]byte, error)) *SSHKeySigner {
+	return &SSHKeySigner{
+		TPMKeySigner: keyfile.NewTPMKeySigner(k.TPMKey, ownerAuth, tpm, auth),
+		key:          k,
+	}
+}