Truncate token uri in logging (#99)

Some URLs are very long (e.g. data urls), so we truncate them to make them easier to read in logs and use less space.

This PR also fixes the foundation parser, which was broken, because the API it relied on doesn't exist anymore. It's now falling back to contract calls.

docs/changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## v0.3.1
+
+- Trim token_uri in some log outputs, this is mainly useful for data uris that are too long and make logs unreadable
+- Fix `FoundationParser`, the API it relied on doesn't exist anymore, so we are falling back to contract calls to get the metadata
+
 ## v0.3.0
 
 - Upgrade web3 to 6.11.3

docs/index.md

@@ -1,6 +1,6 @@
 # Getting Started
 
-Documentation for version: **v0.3.0**
+Documentation for version: **v0.3.1**
 
 ## Overview
 

offchain/metadata/parsers/collection/foundation.py

@@ -12,24 +12,69 @@
 class FoundationParser(CollectionParser):
     _COLLECTION_ADDRESSES: list[str] = [CollectionAddress.FOUNDATION]
 
-    def parse_metadata(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
-        if token.uri is None or raw_data is None:
-            token.uri = f"https://api.foundation.app/opensea/{token.token_id}"
-            raw_data = self.fetcher.fetch_content(token.uri)  # type: ignore[assignment]
-        metadata = DefaultCatchallParser(self.fetcher).parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501
+    def _normalize_metadata(self, metadata: Optional[Metadata]) -> Optional[Metadata]:
+        if metadata is None:
+            return None
+
         metadata.standard = None  # type: ignore[union-attr]
-        if metadata.content.uri.endswith("glb"):  # type: ignore[union-attr]
-            metadata.content.mime_type = "model/gltf-binary"  # type: ignore[union-attr]
+        if (
+            metadata
+            and metadata.image
+            and metadata.image.uri
+            and metadata.image.uri.endswith("glb")
+        ):
+            metadata.image.mime_type = "model/gltf-binary"
 
         return metadata
 
-    async def _gen_parse_metadata_impl(self, token: Token, raw_data: Optional[dict], *args, **kwargs) -> Optional[Metadata]:  # type: ignore[no-untyped-def, type-arg]  # noqa: E501
+    def parse_metadata(
+        self, token: Token, raw_data: Optional[dict], *args, **kwargs
+    ) -> Optional[Metadata]:
         if token.uri is None or raw_data is None:
-            token.uri = f"https://api.foundation.app/opensea/{token.token_id}"
-            raw_data = await self.fetcher.gen_fetch_content(token.uri)  # type: ignore[assignment]
-        metadata = await DefaultCatchallParser(self.fetcher).gen_parse_metadata(token=token, raw_data=raw_data)  # type: ignore[arg-type]  # noqa: E501
-        metadata.standard = None  # type: ignore[union-attr]
-        if metadata.content.uri.endswith("glb"):  # type: ignore[union-attr]
-            metadata.content.mime_type = "model/gltf-binary"  # type: ignore[union-attr]
+            token.uri = self.contract_caller.single_address_single_fn_many_args(
+                token.collection_address,
+                "tokenURI(uint256)",
+                ["string"],
+                [[token.token_id]],
+            )[0]
+            if token.uri is None:
+                return None
 
-        return metadata
+            content = self.fetcher.fetch_content(token.uri)
+            if content and isinstance(content, dict):
+                raw_data = content
+
+        if raw_data is None:
+            return None
+
+        metadata = DefaultCatchallParser(self.fetcher).parse_metadata(
+            token=token, raw_data=raw_data
+        )
+
+        return self._normalize_metadata(metadata)
+
+    async def _gen_parse_metadata_impl(
+        self, token: Token, raw_data: Optional[dict], *args, **kwargs
+    ) -> Optional[Metadata]:
+        if token.uri is None or raw_data is None:
+            token.uri = await self.contract_caller.rpc.async_reader.call_function(
+                token.collection_address,
+                "tokenURI(uint256)",
+                ["string"],
+                [token.token_id],
+            )
+            if token.uri is None:
+                return None
+
+            content = await self.fetcher.gen_fetch_content(token.uri)
+            if content and isinstance(content, dict):
+                raw_data = content
+
+        if raw_data is None:
+            return None
+
+        metadata = await DefaultCatchallParser(self.fetcher).gen_parse_metadata(
+            token=token, raw_data=raw_data
+        )
+
+        return self._normalize_metadata(metadata)

offchain/metadata/pipelines/metadata_pipeline.py

@@ -25,6 +25,15 @@
 )
 
 
+def _truncate_uri(uri: str, max_length: int = 100) -> str:
+    if len(uri) <= max_length:
+        return uri
+
+    keep_length = (max_length - 3) // 2  # 3 is for the '...'
+
+    return uri[:keep_length] + "..." + uri[-keep_length:]
+
+
 class MetadataPipeline(BasePipeline):
     """Pipeline for processing NFT metadata.
 
@@ -161,7 +170,7 @@ def fetch_token_metadata(
             try:
                 raw_data = self.fetcher.fetch_content(token.uri)
             except Exception as e:
-                error_message = f"({token.chain_identifier}-{token.collection_address}-{token.token_id}) Failed to parse token uri: {token.uri}. {str(e)}"  # noqa: E501
+                error_message = f"({token.chain_identifier}-{token.collection_address}-{token.token_id}) Failed to parse token uri: {_truncate_uri(token.uri)}. {str(e)}"  # noqa: E501
                 logger.error(error_message)
                 possible_metadatas_or_errors.append(
                     MetadataProcessingError.from_token_and_error(
@@ -216,9 +225,9 @@ async def gen_fetch_token_metadata(
             Union[Metadata, MetadataProcessingError]: returns either a Metadata
                 or a MetadataProcessingError if unable to parse.
         """
-        possible_metadatas_or_errors: list[
-            Union[Metadata, MetadataProcessingError]
-        ] = []
+        possible_metadatas_or_errors: list[Union[Metadata, MetadataProcessingError]] = (
+            []
+        )
 
         if not token.uri:
             return MetadataProcessingError.from_token_and_error(
@@ -230,7 +239,7 @@ async def gen_fetch_token_metadata(
         try:
             raw_data = await self.fetcher.gen_fetch_content(token.uri)
         except Exception as e:
-            error_message = f"({token.chain_identifier}-{token.collection_address}-{token.token_id}) Failed to parse token uri: {token.uri}. {str(e)}"  # noqa: E501
+            error_message = f"({token.chain_identifier}-{token.collection_address}-{token.token_id}) Failed to parse token uri: {_truncate_uri(token.uri)}. {str(e)}"  # noqa: E501
             logger.error(error_message)
             possible_metadatas_or_errors.append(
                 MetadataProcessingError.from_token_and_error(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "offchain"
-version = "0.3.0"
+version = "0.3.1"
 description = "Open source metadata processing framework"
 authors = ["Zora eng <[email protected]>"]
 readme = "README.md"

tests/metadata/fetchers/test_metadata_fetcher.py

@@ -54,7 +54,7 @@ async def test_gen_fetch_base_adapter(self):  # type: ignore[no-untyped-def]
     async def test_gen_fetch_mime_type_and_size(self):  # type: ignore[no-untyped-def]
         fetcher = MetadataFetcher()
         result = await fetcher.gen_fetch_mime_type_and_size(
-            "https://ipfs.io/ipfs/QmQaYaf3Q2oCBaUfUvV6mBP58EjbUTbMk6dC1o4YGjeWCo"
+            "https://ipfs.decentralized-content.com/ipfs/QmQaYaf3Q2oCBaUfUvV6mBP58EjbUTbMk6dC1o4YGjeWCo"
         )
         assert result == ("image/png", "2887641")  # type: ignore[comparison-overlap]
         print(result)

tests/metadata/parsers/test_foundation_parser.py

@@ -22,9 +22,7 @@ class TestFoundationParser:
     raw_data = {
         "name": "Experiment #0004",
         "description": "They rise again!",
-        "image": "https://d1hiserqh6k9o1.cloudfront.net/Ax/kk/QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.png",
-        "animation_url": "ipfs://QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.glb",
-        "external_url": "https://foundation.app/@pw_3Dlab/foundation/113384",
+        "image": "ipfs://QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.glb",
     }
 
     def test_foundation_parser_should_parse_token(self):  # type: ignore[no-untyped-def]
@@ -37,40 +35,33 @@ def test_foundation_parser_parses_metadata(self):  # type: ignore[no-untyped-def
         fetcher = MetadataFetcher()
         contract_caller = ContractCaller()
         fetcher.fetch_mime_type_and_size = MagicMock(return_value=("application/json", 0))  # type: ignore[assignment]
-        fetcher.fetch_content = MagicMock(return_value=self.raw_data)  # type: ignore[assignment]
+        fetcher.fetch_content = MagicMock(return_value=None)  # type: ignore[assignment]
         parser = FoundationParser(fetcher=fetcher, contract_caller=contract_caller)  # type: ignore[abstract]
         metadata = parser.parse_metadata(token=self.token, raw_data=self.raw_data)
         assert metadata == Metadata(
             token=Token(
                 chain_identifier="ETHEREUM-MAINNET",
                 collection_address="0x3b3ee1931dc30c1957379fac9aba94d1c48a5405",
                 token_id=113384,
-                uri="https://api.foundation.app/opensea/113384",
+                uri="ipfs://QmRxAiR7FsT78mLcyMiE1p86a77gBQVJGWfGRADtMwqyEe/metadata.json",
             ),
             raw_data={
                 "name": "Experiment #0004",
                 "description": "They rise again!",
-                "image": "https://d1hiserqh6k9o1.cloudfront.net/Ax/kk/QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.png",
-                "animation_url": "ipfs://QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.glb",
-                "external_url": "https://foundation.app/@pw_3Dlab/foundation/113384",
+                "image": "ipfs://QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.glb",
             },
             attributes=[],
             standard=None,
             name="Experiment #0004",
             description="They rise again!",
             mime_type="application/json",
             image=MediaDetails(
-                size=0,
-                sha256=None,
-                uri="https://d1hiserqh6k9o1.cloudfront.net/Ax/kk/QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.png",
-                mime_type="application/json",
-            ),
-            content=MediaDetails(
                 size=0,
                 sha256=None,
                 uri="ipfs://QmWwB2LXk7VKu5KtDrtUYdwpHK1NvJ49XrQvFRJxqiAxkk/nft.glb",
                 mime_type="model/gltf-binary",
             ),
+            content=None,
             additional_fields=[],
         )