[hailtop] Don't log rate limit errors (#14605)

hail/python/hailtop/utils/utils.py

@@ -699,6 +699,18 @@ def is_transient_error(e: BaseException) -> bool:
     return False
 
 
+def is_rate_limit_error(e: BaseException) -> bool:
+    import hailtop.httpx  # pylint: disable=import-outside-toplevel,cyclic-import
+
+    if isinstance(e, aiohttp.ClientResponseError) and e.status == 429:
+        return True
+    if isinstance(e, hailtop.httpx.ClientResponseError) and (
+        e.status == 429 or e.status == 403 and 'rateLimitExceeded' in e.body
+    ):
+        return True
+    return False
+
+
 def is_delayed_warning_error(e: BaseException) -> bool:
     if isinstance(e, aiohttp.ClientResponseError) and e.status in (503, 429):
         # 503 service unavailable
@@ -791,6 +803,7 @@ async def retry_transient_errors_with_debug_string(
 ) -> T:
     start_time = time_msecs()
     tries = 0
+
     while True:
         try:
             return await f(*args, **kwargs)
@@ -805,6 +818,8 @@ async def retry_transient_errors_with_debug_string(
                     f'{5 - tries} more times. Do not be alarmed. (next delay: '
                     f'{delay}s). The most recent error was {type(e)} {e}. {debug_string}'
                 )
+            elif is_rate_limit_error(e):
+                pass
             elif not is_transient_error(e):
                 raise
             else: