From 1ffa698b0606da2e13ace9d23c675727d6a51fe7 Mon Sep 17 00:00:00 2001 From: Akshay Avinash Date: Sun, 6 Oct 2024 09:35:19 +0530 Subject: [PATCH] Add get public URL method for kv store --- src/crawlee/storages/_key_value_store.py | 12 ++++++++++++ tests/unit/storages/test_key_value_store.py | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py index b012ea74f..e5cbbc0ed 100644 --- a/src/crawlee/storages/_key_value_store.py +++ b/src/crawlee/storages/_key_value_store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING, Any, AsyncIterator, TypeVar, overload from typing_extensions import override @@ -157,3 +158,14 @@ async def set_value( return await self._resource_client.delete_record(key) return await self._resource_client.set_record(key, value, content_type) + + def get_public_url(self, key: str) -> str: + """Get the public URL for the given key. + + Args: + key: Key of the record for which URL is required + Returns: + The public URL for the given key. + """ + name = self.name or self._configuration.default_key_value_store_id + return f'file://{os.getcwd()}/storage/key_value_stores/{name}/{key}' diff --git a/tests/unit/storages/test_key_value_store.py b/tests/unit/storages/test_key_value_store.py index 5e7355832..6fb1506a0 100644 --- a/tests/unit/storages/test_key_value_store.py +++ b/tests/unit/storages/test_key_value_store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import AsyncGenerator import pytest @@ -100,3 +101,9 @@ async def test_static_get_set_value(key_value_store: KeyValueStore) -> None: await key_value_store.set_value('test-static', 'static') value = await key_value_store.get_value('test-static') assert value == 'static' + + +async def test_static_get_public_url(key_value_store: KeyValueStore) -> None: + await key_value_store.set_value('test-static', 'static') + public_url = key_value_store.get_public_url('test-static') + assert public_url == f'file://{os.getcwd()}/storage/key_value_stores/default/test-static'