From 2afd642ba9df7f1f56ada7d45b17746fd3f25571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20K=C3=B6lling?= Date: Thu, 19 Sep 2024 17:46:57 -0400 Subject: [PATCH] use aiohttp-retry to try again on specific request errors --- ipfsspec/async_ipfs.py | 11 ++++++----- pyproject.toml | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ipfsspec/async_ipfs.py b/ipfsspec/async_ipfs.py index 572a388..ad5e8d3 100644 --- a/ipfsspec/async_ipfs.py +++ b/ipfsspec/async_ipfs.py @@ -8,6 +8,7 @@ import asyncio import aiohttp +import aiohttp_retry from fsspec.asyn import AsyncFileSystem, sync, sync_wrapper from fsspec.exceptions import FSTimeoutError @@ -147,12 +148,12 @@ def _raise_not_found_for_status(self, response, url): response.raise_for_status() - - async def get_client(**kwargs): - timeout = aiohttp.ClientTimeout(sock_connect=1, sock_read=5) - kwargs = {"timeout": timeout, **kwargs} - return aiohttp.ClientSession(**kwargs) + retry_options = aiohttp_retry.ExponentialRetry( + attempts=5, + exceptions={OSError, aiohttp.ServerDisconnectedError, asyncio.TimeoutError}) + retry_client = aiohttp_retry.RetryClient(raise_for_status=False, retry_options=retry_options) + return retry_client def gateway_from_file(gateway_path, protocol="ipfs"): diff --git a/pyproject.toml b/pyproject.toml index c0e1141..0a868a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ dependencies = [ "fsspec>=0.9.0", "requests", "aiohttp", + "aiohttp-retry", "multiformats", "dag-cbor >= 0.2.2", "pure-protobuf >= 2.1.0, <3",