diff --git a/src/Prometheus/Storage/APC.php b/src/Prometheus/Storage/APC.php
index c45f1940..c8a3933f 100644
--- a/src/Prometheus/Storage/APC.php
+++ b/src/Prometheus/Storage/APC.php
@@ -123,11 +123,32 @@ public function updateSummary(array $data): void
     public function updateGauge(array $data): void
     {
         $valueKey = $this->valueKey($data);
+        $old = apcu_fetch($valueKey);
         if ($data['command'] === Adapter::COMMAND_SET) {
-            apcu_store($valueKey, $this->toBinaryRepresentationAsInteger($data['value']));
-            apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
+            $new = $this->toBinaryRepresentationAsInteger($data['value']);
+            if ($old === false) {
+                apcu_store($valueKey, $new);
+                apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
+                return;
+            } else {
+                // Taken from https://github.com/prometheus/client_golang/blob/66058aac3a83021948e5fb12f1f408ff556b9037/prometheus/value.go#L91
+                while (true) {
+                    if ($old !== false) {
+                        if (apcu_cas($valueKey, $old, $new)) {
+                            return;
+                        } else {
+                            $old = apcu_fetch($valueKey);
+                        }
+                    } else {
+                        // Cache got evicted under our feet? Just consider it a fresh/new insert and move on.
+                        apcu_store($valueKey, $new);
+                        apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
+                        return;
+                    }
+                }
+            }
         } else {
-            if (!apcu_exists($valueKey)) {
+            if ($old === false) {
                 $new = apcu_add($valueKey, $this->toBinaryRepresentationAsInteger(0));
                 if ($new) {
                     apcu_store($this->metaKey($data), json_encode($this->metaData($data)));
diff --git a/src/Prometheus/Storage/APCng.php b/src/Prometheus/Storage/APCng.php
index 6628ef97..5448fc95 100644
--- a/src/Prometheus/Storage/APCng.php
+++ b/src/Prometheus/Storage/APCng.php
@@ -169,16 +169,34 @@ public function updateSummary(array $data): void
     public function updateGauge(array $data): void
     {
         $valueKey = $this->valueKey($data);
+        $old = apcu_fetch($valueKey);
         if ($data['command'] === Adapter::COMMAND_SET) {
-            apcu_store($valueKey, $this->convertToIncrementalInteger($data['value']), 0);
-            $this->storeMetadata($data);
-            $this->storeLabelKeys($data);
+            $new = $this->convertToIncrementalInteger($data['value']);
+            if ($old === false) {
+                apcu_store($valueKey, $new, 0);
+                $this->storeMetadata($data);
+                $this->storeLabelKeys($data);
+
+                return;
+            }
+
+            for ($loops = 0; $loops < self::MAX_LOOPS; $loops++) {
+                if (apcu_cas($valueKey, $old, $new)) {
+                    break;
+                }
+                $old = apcu_fetch($valueKey);
+                if ($old === false) {
+                    apcu_store($valueKey, $new, 0);
+                    $this->storeMetadata($data);
+                    $this->storeLabelKeys($data);
+
+                    return;
+                }
+            }
 
             return;
         }
 
-        $old = apcu_fetch($valueKey);
-
         if ($old === false) {
             apcu_add($valueKey, 0, 0);
             $this->storeMetadata($data);
@@ -896,6 +914,10 @@ private function decodeLabelKey(string $str): string
     private function storeMetadata(array $data, bool $encoded = true): void
     {
         $metaKey = $this->metaKey($data);
+        if (apcu_exists($metaKey)) {
+            return;
+        }
+
         $metaData = $this->metaData($data);
         $toStore = $metaData;