From f3e0507caff7e5def0c3621cfa5cbd14f6c0f537 Mon Sep 17 00:00:00 2001 From: Lukas Kluft Date: Sat, 31 Aug 2024 16:25:01 -0100 Subject: [PATCH 1/2] Add AsyncIPNSFileSystem --- ipfsspec/__init__.py | 4 ++-- ipfsspec/async_ipfs.py | 35 ++++++++++++++++++++--------------- setup.py | 1 + 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ipfsspec/__init__.py b/ipfsspec/__init__.py index cf90301..c1e8c7e 100644 --- a/ipfsspec/__init__.py +++ b/ipfsspec/__init__.py @@ -1,7 +1,7 @@ -from .async_ipfs import AsyncIPFSFileSystem +from .async_ipfs import AsyncIPFSFileSystem, AsyncIPNSFileSystem from ._version import get_versions __version__ = get_versions()['version'] del get_versions -__all__ = ["__version__", "AsyncIPFSFileSystem"] +__all__ = ["__version__", "AsyncIPFSFileSystem", "AsyncIPNSFileSystem"] diff --git a/ipfsspec/async_ipfs.py b/ipfsspec/async_ipfs.py index 5f9c7c4..d466553 100644 --- a/ipfsspec/async_ipfs.py +++ b/ipfsspec/async_ipfs.py @@ -21,7 +21,7 @@ def __init__(self, retry_after=None): self.retry_after = retry_after -class AsyncIPFSGatewayBase: +class AsyncGatewayBase: async def stat(self, path, session): res = await self.api_get("files/stat", session, arg=path) self._raise_not_found_for_status(res, path) @@ -87,11 +87,12 @@ def _raise_not_found_for_status(self, response, url): response.raise_for_status() -class AsyncIPFSGateway(AsyncIPFSGatewayBase): +class AsyncGateway(AsyncGatewayBase): resolution = "path" - def __init__(self, url): + def __init__(self, url, protocol="ipfs"): self.url = url + self.protocol = protocol async def api_get(self, endpoint, session, **kwargs): res = await session.get(self.url + "/api/v0/" + endpoint, params=kwargs, trace_request_ctx={'gateway': self.url}) @@ -106,7 +107,7 @@ async def api_post(self, endpoint, session, **kwargs): async def _cid_req(self, method, path, headers=None, **kwargs): headers = headers or {} if self.resolution == "path": - res = await method(self.url + "/ipfs/" + path, trace_request_ctx={'gateway': self.url}, headers=headers, **kwargs) + res = await method("/".join((self.url, self.protocol, path)), trace_request_ctx={'gateway': self.url}, headers=headers, **kwargs) elif self.resolution == "subdomain": raise NotImplementedError("subdomain resolution is not yet implemented") else: @@ -145,17 +146,17 @@ async def get_client(**kwargs): return aiohttp.ClientSession(**kwargs) -def gateway_from_file(gateway_path): +def gateway_from_file(gateway_path, protocol="ipfs"): if gateway_path.exists(): with open(gateway_path) as gw_file: ipfs_gateway = gw_file.readline().strip() logger.debug("using IPFS gateway from %s: %s", gateway_path, ipfs_gateway) - return AsyncIPFSGateway(ipfs_gateway) + return AsyncGateway(ipfs_gateway, protocol=protocol) return None @lru_cache -def get_gateway(): +def get_gateway(protocol="ipfs"): """ Get IPFS gateway according to IPIP-280 @@ -166,29 +167,29 @@ def get_gateway(): ipfs_gateway = os.environ.get("IPFS_GATEWAY", "") if ipfs_gateway: logger.debug("using IPFS gateway from IPFS_GATEWAY environment variable: %s", ipfs_gateway) - return AsyncIPFSGateway(ipfs_gateway) + return AsyncGateway(ipfs_gateway, protocol) # internal configuration: accept IPFSSPEC_GATEWAYS for backwards compatibility if ipfsspec_gateways := os.environ.get("IPFSSPEC_GATEWAYS", ""): ipfs_gateway = ipfsspec_gateways.split()[0] logger.debug("using IPFS gateway from IPFSSPEC_GATEWAYS environment variable: %s", ipfs_gateway) warnings.warn("The IPFSSPEC_GATEWAYS environment variable is deprecated, please configure your IPFS Gateway according to IPIP-280, e.g. by using the IPFS_GATEWAY environment variable or using the ~/.ipfs/gateway file.", DeprecationWarning) - return AsyncIPFSGateway(ipfs_gateway) + return AsyncGateway(ipfs_gateway, protocol) # check various well-known files for possible gateway configurations if ipfs_path := os.environ.get("IPFS_PATH", ""): - if ipfs_gateway := gateway_from_file(Path(ipfs_path) / "gateway"): + if ipfs_gateway := gateway_from_file(Path(ipfs_path) / "gateway", protocol): return ipfs_gateway if home := os.environ.get("HOME", ""): - if ipfs_gateway := gateway_from_file(Path(home) / ".ipfs" / "gateway"): + if ipfs_gateway := gateway_from_file(Path(home) / ".ipfs" / "gateway", protocol): return ipfs_gateway if config_home := os.environ.get("XDG_CONFIG_HOME", ""): - if ipfs_gateway := gateway_from_file(Path(config_home) / "ipfs" / "gateway"): + if ipfs_gateway := gateway_from_file(Path(config_home) / "ipfs" / "gateway", protocol): return ipfs_gateway - if ipfs_gateway := gateway_from_file(Path("/etc") / "ipfs" / "gateway"): + if ipfs_gateway := gateway_from_file(Path("/etc") / "ipfs" / "gateway", protocol): return ipfs_gateway system = platform.system() @@ -213,7 +214,7 @@ def get_gateway(): candidates = [] for candidate in candidates: - if ipfs_gateway := gateway_from_file(candidate): + if ipfs_gateway := gateway_from_file(candidate, protocol): return ipfs_gateway # if we reach this point, no gateway is configured @@ -249,7 +250,7 @@ def __init__(self, asynchronous=False, loop=None, client_kwargs=None, **storage_ @property def gateway(self): - return get_gateway() + return get_gateway(self.protocol) @staticmethod def close_session(loop, session): @@ -300,3 +301,7 @@ def open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs): def ukey(self, path): """returns the CID, which is by definition an unchanging identitifer""" return self.info(path)["CID"] + + +class AsyncIPNSFileSystem(AsyncIPFSFileSystem): + protocol = "ipns" diff --git a/setup.py b/setup.py index 7243c24..832f002 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ entry_points={ 'fsspec.specs': [ 'ipfs=ipfsspec.AsyncIPFSFileSystem', + 'ipns=ipfsspec.AsyncIPNSFileSystem', ], }, ) From c58f756aad9a5158dfd687a2ec430bf1dece8c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20K=C3=B6lling?= Date: Thu, 12 Sep 2024 15:50:32 -0400 Subject: [PATCH 2/2] undo naming change to keep tests happy --- ipfsspec/async_ipfs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ipfsspec/async_ipfs.py b/ipfsspec/async_ipfs.py index d466553..a6d003d 100644 --- a/ipfsspec/async_ipfs.py +++ b/ipfsspec/async_ipfs.py @@ -21,7 +21,7 @@ def __init__(self, retry_after=None): self.retry_after = retry_after -class AsyncGatewayBase: +class AsyncIPFSGatewayBase: async def stat(self, path, session): res = await self.api_get("files/stat", session, arg=path) self._raise_not_found_for_status(res, path) @@ -87,7 +87,7 @@ def _raise_not_found_for_status(self, response, url): response.raise_for_status() -class AsyncGateway(AsyncGatewayBase): +class AsyncIPFSGateway(AsyncIPFSGatewayBase): resolution = "path" def __init__(self, url, protocol="ipfs"): @@ -151,7 +151,7 @@ def gateway_from_file(gateway_path, protocol="ipfs"): with open(gateway_path) as gw_file: ipfs_gateway = gw_file.readline().strip() logger.debug("using IPFS gateway from %s: %s", gateway_path, ipfs_gateway) - return AsyncGateway(ipfs_gateway, protocol=protocol) + return AsyncIPFSGateway(ipfs_gateway, protocol=protocol) return None