From 366e7087a01ff9a5ba59e53c4dba037348411cd6 Mon Sep 17 00:00:00 2001
From: Christopher Vittal <cvittal@broadinstitute.org>
Date: Tue, 9 Jul 2024 10:34:04 -0400
Subject: [PATCH] [hailtop] Don't log rate limit errors (#14605)

---
 hail/python/hailtop/utils/utils.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/hail/python/hailtop/utils/utils.py b/hail/python/hailtop/utils/utils.py
index 4fce7a5047c..f12dca8ae43 100644
--- a/hail/python/hailtop/utils/utils.py
+++ b/hail/python/hailtop/utils/utils.py
@@ -699,6 +699,18 @@ def is_transient_error(e: BaseException) -> bool:
     return False
 
 
+def is_rate_limit_error(e: BaseException) -> bool:
+    import hailtop.httpx  # pylint: disable=import-outside-toplevel,cyclic-import
+
+    if isinstance(e, aiohttp.ClientResponseError) and e.status == 429:
+        return True
+    if isinstance(e, hailtop.httpx.ClientResponseError) and (
+        e.status == 429 or e.status == 403 and 'rateLimitExceeded' in e.body
+    ):
+        return True
+    return False
+
+
 def is_delayed_warning_error(e: BaseException) -> bool:
     if isinstance(e, aiohttp.ClientResponseError) and e.status in (503, 429):
         # 503 service unavailable
@@ -791,6 +803,7 @@ async def retry_transient_errors_with_debug_string(
 ) -> T:
     start_time = time_msecs()
     tries = 0
+
     while True:
         try:
             return await f(*args, **kwargs)
@@ -805,6 +818,8 @@ async def retry_transient_errors_with_debug_string(
                     f'{5 - tries} more times. Do not be alarmed. (next delay: '
                     f'{delay}s). The most recent error was {type(e)} {e}. {debug_string}'
                 )
+            elif is_rate_limit_error(e):
+                pass
             elif not is_transient_error(e):
                 raise
             else: