From 4315ac24256d80812415afe60c4e4ecb45a7a171 Mon Sep 17 00:00:00 2001 From: mhh Date: Tue, 15 Aug 2023 16:45:07 +0200 Subject: [PATCH] increase test coverage, add CACHE_FILES_PATH to settings --- src/aleph/sdk/cache.py | 12 +- src/aleph/sdk/conf.py | 4 + src/aleph/sdk/node.py | 10 +- tests/unit/conftest.py | 70 ++++++++++ tests/unit/test_cache.py | 64 +-------- tests/unit/test_node.py | 289 +++++++++++++++++++-------------------- 6 files changed, 231 insertions(+), 218 deletions(-) diff --git a/src/aleph/sdk/cache.py b/src/aleph/sdk/cache.py index 526b81b8..13dbe966 100644 --- a/src/aleph/sdk/cache.py +++ b/src/aleph/sdk/cache.py @@ -290,7 +290,7 @@ async def get_posts( conditions = [] if types: - conditions.append(query_post_types(types)) + conditions.append(query_message_types(types)) if refs: conditions.append(query_refs(refs)) if addresses: @@ -345,7 +345,7 @@ async def get_messages( conditions = [] if message_type: - conditions.append(MessageModel.type == message_type.value) + conditions.append(query_message_types(message_type)) if content_types: conditions.append(query_content_types(content_types)) if content_keys: @@ -455,10 +455,10 @@ async def watch_messages( yield model_to_message(item) -def query_post_types(types: Union[str, Iterable[str]]): - if isinstance(types, str): - return MessageModel.content_type == types - return MessageModel.content_type.in_(types) +def query_message_types(message_types: Union[str, Iterable[str]]): + if isinstance(message_types, str): + return MessageModel.type == message_types + return MessageModel.type.in_(message_types) def query_content_types(content_types: Union[str, Iterable[str]]): diff --git a/src/aleph/sdk/conf.py b/src/aleph/sdk/conf.py index 84ce4991..2ae20e45 100644 --- a/src/aleph/sdk/conf.py +++ b/src/aleph/sdk/conf.py @@ -37,6 +37,10 @@ class Settings(BaseSettings): default=Path(":memory:"), # can also be :memory: for in-memory caching description="Path to the cache database", ) + CACHE_FILES_PATH: Path = Field( + default=Path("cache", "files"), + description="Path to the cache files", + ) class Config: env_prefix = "ALEPH_" diff --git a/src/aleph/sdk/node.py b/src/aleph/sdk/node.py index 3c425fd7..e7337af3 100644 --- a/src/aleph/sdk/node.py +++ b/src/aleph/sdk/node.py @@ -16,6 +16,7 @@ from aleph.sdk import AuthenticatedAlephClient from aleph.sdk.base import AuthenticatedAlephClientBase from aleph.sdk.cache import MessageCache +from aleph.sdk.conf import settings from aleph.sdk.types import StorageEnum @@ -107,13 +108,14 @@ async def download_file(self, file_hash: str) -> bytes: return f.read() except FileNotFoundError: file = await self.session.download_file(file_hash) + self._file_path(file_hash).parent.mkdir(parents=True, exist_ok=True) with open(self._file_path(file_hash), "wb") as f: f.write(file) return file - def _file_path(self, file_hash: str) -> Path: - # TODO: Make this configurable (and not be an ugly hack) - return Path("cache", "files", file_hash) + @staticmethod + def _file_path(file_hash: str) -> Path: + return settings.CACHE_FILES_PATH / Path(file_hash) async def create_post( self, @@ -271,5 +273,5 @@ async def submit( ) # TODO: this can cause inconsistencies if the message is dropped if status in [MessageStatus.PROCESSED, MessageStatus.PENDING]: - self.add(resp["message"]) + self.add(resp) return resp, status diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9952f847..311d32f3 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,7 +1,9 @@ from pathlib import Path from tempfile import NamedTemporaryFile +from typing import List import pytest as pytest +from aleph_message.models import AggregateMessage, AlephMessage, PostMessage import aleph.sdk.chains.ethereum as ethereum import aleph.sdk.chains.sol as solana @@ -34,3 +36,71 @@ def tezos_account() -> tezos.TezosAccount: with NamedTemporaryFile(delete=False) as private_key_file: private_key_file.close() yield tezos.get_fallback_account(path=Path(private_key_file.name)) + + +@pytest.fixture +def messages() -> List[AlephMessage]: + return [ + AggregateMessage.parse_obj( + { + "item_hash": "5b26d949fe05e38f535ef990a89da0473f9d700077cced228f2d36e73fca1fd6", + "type": "AGGREGATE", + "chain": "ETH", + "sender": "0x51A58800b26AA1451aaA803d1746687cB88E0501", + "signature": "0xca5825b6b93390482b436cb7f28b4628f8c9f56dc6af08260c869b79dd6017c94248839bd9fd0ffa1230dc3b1f4f7572a8d1f6fed6c6e1fb4d70ccda0ab5d4f21b", + "item_type": "inline", + "item_content": '{"address":"0x51A58800b26AA1451aaA803d1746687cB88E0501","key":"0xce844d79e5c0c325490c530aa41e8f602f0b5999binance","content":{"1692026263168":{"version":"x25519-xsalsa20-poly1305","nonce":"RT4Lbqs7Xzk+op2XC+VpXgwOgg21BotN","ephemPublicKey":"CVW8ECE3m8BepytHMTLan6/jgIfCxGdnKmX47YirF08=","ciphertext":"VuGJ9vMkJSbaYZCCv6Zemx4ixeb+9IW8H1vFB9vLtz1a8d87R4BfYUisLoCQxRkeUXqfW0/KIGQ5idVjr8Yj7QnKglW5AJ8UX7wEWMhiRFLatpWP8P9FI2n8Z7Rblu7Oz/OeKnuljKL3KsalcUQSsFa/1qACsIoycPZ6Wq6t1mXxVxxJWzClLyKRihv1pokZGT9UWxh7+tpoMGlRdYainyAt0/RygFw+r8iCMOilHnyv4ndLkKQJXyttb0tdNr/gr57+9761+trioGSysLQKZQWW6Ih6aE8V9t3BenfzYwiCnfFw3YAAKBPMdm9QdIETyrOi7YhD/w==","sha256":"bbeb499f681aed2bc18b6f3b6a30d25254bd30fbfde43444e9085f3bcd075c3c"}},"time":1692026263.662}', + "content": { + "key": "0xce844d79e5c0c325490c530aa41e8f602f0b5999binance", + "time": 1692026263.662, + "address": "0x51A58800b26AA1451aaA803d1746687cB88E0501", + "content": { + "hello": "world", + }, + }, + "time": 1692026263.662, + "channel": "UNSLASHED", + "size": 734, + "confirmations": [], + "confirmed": False, + } + ), + PostMessage.parse_obj( + { + "item_hash": "70f3798fdc68ce0ee03715a5547ee24e2c3e259bf02e3f5d1e4bf5a6f6a5e99f", + "type": "POST", + "chain": "SOL", + "sender": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4", + "signature": "0x91616ee45cfba55742954ff87ebf86db4988bcc5e3334b49a4caa6436e28e28d4ab38667cbd4bfb8903abf8d71f70d9ceb2c0a8d0a15c04fc1af5657f0050c101b", + "item_type": "storage", + "item_content": None, + "content": { + "time": 1692026021.1257718, + "type": "aleph-network-metrics", + "address": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4", + "ref": "0123456789abcdef", + "content": { + "tags": ["mainnet"], + "hello": "world", + "version": "1.0", + }, + }, + "time": 1692026021.132849, + "channel": "aleph-scoring", + "size": 122537, + "confirmations": [], + "confirmed": False, + } + ), + ] + + +@pytest.fixture +def raw_messages_response(messages): + return { + "messages": [message.dict() for message in messages], + "pagination_item": "messages", + "pagination_page": 1, + "pagination_per_page": 20, + "pagination_total": 2, + } diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 1c960d83..6b44bf03 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -4,7 +4,6 @@ import pytest from aleph_message.models import ( - AggregateMessage, AlephMessage, Chain, MessageType, @@ -16,63 +15,6 @@ from aleph.sdk.chains.ethereum import get_fallback_account -@pytest.fixture(scope="session") -def messages(): - return [ - AggregateMessage.parse_obj( - { - "item_hash": "5b26d949fe05e38f535ef990a89da0473f9d700077cced228f2d36e73fca1fd6", - "type": "AGGREGATE", - "chain": "ETH", - "sender": "0x51A58800b26AA1451aaA803d1746687cB88E0501", - "signature": "0xca5825b6b93390482b436cb7f28b4628f8c9f56dc6af08260c869b79dd6017c94248839bd9fd0ffa1230dc3b1f4f7572a8d1f6fed6c6e1fb4d70ccda0ab5d4f21b", - "item_type": "inline", - "item_content": '{"address":"0x51A58800b26AA1451aaA803d1746687cB88E0501","key":"0xce844d79e5c0c325490c530aa41e8f602f0b5999binance","content":{"1692026263168":{"version":"x25519-xsalsa20-poly1305","nonce":"RT4Lbqs7Xzk+op2XC+VpXgwOgg21BotN","ephemPublicKey":"CVW8ECE3m8BepytHMTLan6/jgIfCxGdnKmX47YirF08=","ciphertext":"VuGJ9vMkJSbaYZCCv6Zemx4ixeb+9IW8H1vFB9vLtz1a8d87R4BfYUisLoCQxRkeUXqfW0/KIGQ5idVjr8Yj7QnKglW5AJ8UX7wEWMhiRFLatpWP8P9FI2n8Z7Rblu7Oz/OeKnuljKL3KsalcUQSsFa/1qACsIoycPZ6Wq6t1mXxVxxJWzClLyKRihv1pokZGT9UWxh7+tpoMGlRdYainyAt0/RygFw+r8iCMOilHnyv4ndLkKQJXyttb0tdNr/gr57+9761+trioGSysLQKZQWW6Ih6aE8V9t3BenfzYwiCnfFw3YAAKBPMdm9QdIETyrOi7YhD/w==","sha256":"bbeb499f681aed2bc18b6f3b6a30d25254bd30fbfde43444e9085f3bcd075c3c"}},"time":1692026263.662}', - "content": { - "key": "0xce844d79e5c0c325490c530aa41e8f602f0b5999binance", - "time": 1692026263.662, - "address": "0x51A58800b26AA1451aaA803d1746687cB88E0501", - "content": { - "hello": "world", - }, - }, - "time": 1692026263.662, - "channel": "UNSLASHED", - "size": 734, - "confirmations": [], - "confirmed": False, - } - ), - PostMessage.parse_obj( - { - "item_hash": "70f3798fdc68ce0ee03715a5547ee24e2c3e259bf02e3f5d1e4bf5a6f6a5e99f", - "type": "POST", - "chain": "SOL", - "sender": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4", - "signature": "0x91616ee45cfba55742954ff87ebf86db4988bcc5e3334b49a4caa6436e28e28d4ab38667cbd4bfb8903abf8d71f70d9ceb2c0a8d0a15c04fc1af5657f0050c101b", - "item_type": "storage", - "item_content": None, - "content": { - "time": 1692026021.1257718, - "type": "aleph-network-metrics", - "address": "0x4D52380D3191274a04846c89c069E6C3F2Ed94e4", - "ref": "0123456789abcdef", - "content": { - "tags": ["mainnet"], - "hello": "world", - "version": "1.0", - }, - }, - "time": 1692026021.132849, - "channel": "aleph-scoring", - "size": 122537, - "confirmations": [], - "confirmed": False, - } - ), - ] - - @pytest.mark.asyncio async def test_base(messages): # test add_many @@ -171,6 +113,12 @@ async def test_chains(self): 0 ] == self.messages[1] + @pytest.mark.asyncio + async def test_content_keys(self): + assert ( + await self.cache.get_messages(content_keys=self.messages[0].content.key) + ).messages[0] == self.messages[0] + @pytest.mark.asyncio async def test_message_cache_listener(): diff --git a/tests/unit/test_node.py b/tests/unit/test_node.py index e4f045a4..0b844e50 100644 --- a/tests/unit/test_node.py +++ b/tests/unit/test_node.py @@ -1,10 +1,14 @@ import json +import os +from pathlib import Path +from typing import Any, Dict from unittest.mock import AsyncMock, MagicMock import pytest as pytest from aleph_message.models import ( AggregateMessage, ForgetMessage, + MessageType, PostMessage, ProgramMessage, StoreMessage, @@ -12,163 +16,102 @@ from aleph_message.status import MessageStatus from aleph.sdk import AuthenticatedAlephClient +from aleph.sdk.conf import settings from aleph.sdk.node import DomainNode from aleph.sdk.types import Account, StorageEnum -@pytest.fixture -def mock_node_with_post_success( - ethereum_account: Account, -) -> DomainNode: - class MockPostResponse: - def __init__(self, sync: bool): - self.sync = sync - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - ... - - @property - def status(self): - return 200 if self.sync else 202 - - def raise_for_status(self): - if self.status not in [200, 202]: - raise Exception("Bad status code") - - async def json(self): - message_status = "processed" if self.sync else "pending" - return { - "message_status": message_status, - "publication_status": {"status": "success", "failed": []}, - "hash": "QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy", - "message": { - "type": "post", - "channel": "TEST", - "content": {"Hello": "World"}, - "key": "QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy", - "item_hash": "QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy", - }, - } - - async def text(self): - return json.dumps(await self.json()) - - class MockGetResponse: - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - ... - - @property - def status(self): - return 200 - - def raise_for_status(self): - if self.status != 200: - raise Exception("Bad status code") - - async def json(self): - return { - "messages": [ - { - "item_hash": "", - "type": "AGGREGATE", - "chain": "ETH", - "sender": "", - "signature": "", - "item_type": "inline", - "item_content": "", - "content": { - "key": "", - "time": 1692026263.662, - "address": "", - "content": { - "1692026263168": { - "nonce": "", - "sha256": "", - "version": "x25519-xsalsa20-poly1305", - "ciphertext": "", - "ephemPublicKey": "", - } - }, - }, - "time": 1692026263.662, - "channel": "UNSLASHED", - "size": 734, - "confirmations": [], - "confirmed": False, - }, - { - "item_hash": "", - "type": "POST", - "chain": "ETH", - "sender": "", - "signature": "", - "item_type": "storage", - "item_content": None, - "content": { - "time": 1692026021.1257718, - "type": "aleph-network-metrics", - "address": "", - "content": { - "tags": ["mainnet"], - "metrics": { - "ccn": [ - { - "asn": 24940, - "url": "", - "as_name": "", - "node_id": "", - "version": "", - "txs_total": 0, - "measured_at": 1692025827.943929, - "base_latency": 0.1020817756652832, - "metrics_latency": 0.28051209449768066, - "pending_messages": 0, - "aggregate_latency": 0.06148695945739746, - "base_latency_ipv4": 0.1020817756652832, - "eth_height_remaining": 276, - "file_download_latency": 0.10703206062316895, - } - ], - "server": "151.115.63.76", - "server_asn": 12876, - "server_as_name": "Online SAS, FR", - }, - "version": "1.0", - }, - }, - "time": 1692026021.132849, - "channel": "aleph-scoring", - "size": 122537, - "confirmations": [], - "confirmed": False, - }, - ], - "pagination_item": "messages", - "pagination_page": 1, - "pagination_per_page": 20, - "pagination_total": 1, - } +class MockPostResponse: + def __init__(self, response_message: Any, sync: bool): + self.response_message = response_message + self.sync = sync + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + ... + + @property + def status(self): + return 200 if self.sync else 202 + + def raise_for_status(self): + if self.status not in [200, 202]: + raise Exception("Bad status code") + + async def json(self): + message_status = "processed" if self.sync else "pending" + return { + "message_status": message_status, + "publication_status": {"status": "success", "failed": []}, + "hash": "QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy", + "message": self.response_message, + } + + async def text(self): + return json.dumps(await self.json()) + + +class MockGetResponse: + def __init__(self, response): + self.response = response + + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc_val, exc_tb): + ... + + @property + def status(self): + return 200 + + def raise_for_status(self): + if self.status != 200: + raise Exception("Bad status code") + + async def json(self): + return self.response + + +@pytest.fixture +def mock_session_with_two_messages( + ethereum_account: Account, raw_messages_response: Dict[str, Any] +) -> AuthenticatedAlephClient: http_session = AsyncMock() http_session.post = MagicMock() http_session.post.side_effect = lambda *args, **kwargs: MockPostResponse( - sync=kwargs.get("sync", False) + response_message={ + "type": "post", + "channel": "TEST", + "content": {"Hello": "World"}, + "key": "QmBlahBlahBlah", + "item_hash": "QmBlahBlahBlah", + }, + sync=kwargs.get("sync", False), ) http_session.get = MagicMock() - http_session.get.return_value = MockGetResponse() + http_session.get.return_value = MockGetResponse(raw_messages_response) client = AuthenticatedAlephClient( account=ethereum_account, api_server="http://localhost" ) client.http_session = http_session - node = DomainNode(session=client) + return client + + +@pytest.mark.asyncio +def test_node_init(mock_session_with_two_messages): + node = DomainNode(session=mock_session_with_two_messages) + assert node.session == mock_session_with_two_messages + assert len(node) >= 2 + + +@pytest.fixture +def mock_node_with_post_success(mock_session_with_two_messages) -> DomainNode: + node = DomainNode(session=mock_session_with_two_messages) return node @@ -209,14 +152,14 @@ async def test_create_store(mock_node_with_post_success): mock_node_with_post_success.ipfs_push_file = mock_ipfs_push_file - async with mock_node_with_post_success as session: - _ = await session.create_store( + async with mock_node_with_post_success as node: + _ = await node.create_store( file_content=b"HELLO", channel="TEST", storage_engine=StorageEnum.ipfs, ) - _ = await session.create_store( + _ = await node.create_store( file_hash="QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy", channel="TEST", storage_engine=StorageEnum.ipfs, @@ -227,8 +170,8 @@ async def test_create_store(mock_node_with_post_success): "QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy" ) mock_node_with_post_success.storage_push_file = mock_storage_push_file - async with mock_node_with_post_success as session: - store_message, message_status = await session.create_store( + async with mock_node_with_post_success as node: + store_message, message_status = await node.create_store( file_content=b"HELLO", channel="TEST", storage_engine=StorageEnum.storage, @@ -240,8 +183,8 @@ async def test_create_store(mock_node_with_post_success): @pytest.mark.asyncio async def test_create_program(mock_node_with_post_success): - async with mock_node_with_post_success as session: - program_message, message_status = await session.create_program( + async with mock_node_with_post_success as node: + program_message, message_status = await node.create_program( program_ref="cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe", entrypoint="main:app", runtime="facefacefacefacefacefacefacefacefacefacefacefacefacefacefaceface", @@ -255,8 +198,8 @@ async def test_create_program(mock_node_with_post_success): @pytest.mark.asyncio async def test_forget(mock_node_with_post_success): - async with mock_node_with_post_success as session: - forget_message, message_status = await session.forget( + async with mock_node_with_post_success as node: + forget_message, message_status = await node.forget( hashes=["QmRTV3h1jLcACW4FRfdisokkQAk4E4qDhUzGpgdrd4JAFy"], reason="GDPR", channel="TEST", @@ -264,3 +207,49 @@ async def test_forget(mock_node_with_post_success): assert mock_node_with_post_success.session.http_session.post.called_once assert isinstance(forget_message, ForgetMessage) + + +@pytest.mark.asyncio +async def test_download_file(mock_node_with_post_success): + mock_node_with_post_success.session.download_file = AsyncMock() + mock_node_with_post_success.session.download_file.return_value = b"HELLO" + + # remove file locally + if os.path.exists(settings.CACHE_FILES_PATH / Path("QmAndSoOn")): + os.remove(settings.CACHE_FILES_PATH / Path("QmAndSoOn")) + + # fetch from mocked response + async with mock_node_with_post_success as node: + file_content = await node.download_file( + file_hash="QmAndSoOn", + ) + + assert mock_node_with_post_success.session.http_session.get.called_once + assert file_content == b"HELLO" + + # fetch cached + async with mock_node_with_post_success as node: + file_content = await node.download_file( + file_hash="QmAndSoOn", + ) + + assert file_content == b"HELLO" + + +@pytest.mark.asyncio +async def test_submit_message(mock_node_with_post_success): + content = {"Hello": "World"} + async with mock_node_with_post_success as node: + message, status = await node.submit( + content={ + "address": "0x1234567890123456789012345678901234567890", + "time": 1234567890, + "type": "TEST", + "content": content, + }, + message_type=MessageType.post, + ) + + assert mock_node_with_post_success.session.http_session.post.called_once + assert message.content.content == content + assert status == MessageStatus.PENDING