diff --git a/src/ray/rpc/retryable_grpc_client.h b/src/ray/rpc/retryable_grpc_client.h
index 99f64dea1922c..0bcdd5e76a3d3 100644
--- a/src/ray/rpc/retryable_grpc_client.h
+++ b/src/ray/rpc/retryable_grpc_client.h
@@ -250,8 +250,8 @@ class RetryableGrpcClient : public std::enable_shared_from_this<RetryableGrpcCli
       if (!server_is_unavailable_) {
         server_is_unavailable_ = true;
       } else {
-        uint64_t server_unavailable_duration_seconds = static_cast<uint64_t>(
-            absl::ToInt64Seconds(absl::Now() - server_last_available_time_));
+        uint64_t server_unavailable_duration_seconds = static_cast<uint64_t>(std::max(
+            absl::ToInt64Seconds(absl::Now() - server_last_available_time_), 0ll));
         if (server_unavailable_duration_seconds >= server_unavailable_timeout_seconds_) {
           RAY_LOG(WARNING) << server_name_ << " has been unavailable for "
                            << server_unavailable_duration_seconds << " seconds";