From 4c267c34e202ae35773489860e56c5f944123d00 Mon Sep 17 00:00:00 2001 From: jmmshn Date: Fri, 21 Aug 2020 16:34:14 -0700 Subject: [PATCH 1/8] typo in test --- tests/stores/test_aws.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 544a5fbbc..b62e9004d 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -137,7 +137,7 @@ def tests_msonable_read_write(s3store): def test_remove(s3store): s3store.update([{"task_id": "mp-2", "data": "asd"}]) s3store.update([{"task_id": "mp-4", "data": "asd"}]) - s3store.update([{"task_id": "mp-5", "data": "aaa"}]) + s3store.update({"task_id": "mp-5", "data": "aaa"}) assert s3store.query_one({"task_id": "mp-2"}) is not None assert s3store.query_one({"task_id": "mp-4"}) is not None From f39fb19fd127fe8708a6257dbe5b0d0afe96d4de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 07:03:29 +0000 Subject: [PATCH 2/8] Bump pydocstyle from 5.0.2 to 5.1.0 --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 811048522..dfbb67d54 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -5,7 +5,7 @@ pytest-cov==2.10.1 pytest-mock==3.2.0 moto==1.3.14 pycodestyle==2.6.0 -pydocstyle==5.0.2 +pydocstyle==5.1.0 flake8==3.8.3 mypy==0.782 mypy-extensions==0.4.3 From ccffb332949792eb932792b692b8f0b83d704f03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 07:03:43 +0000 Subject: [PATCH 3/8] Bump pytest-mock from 3.2.0 to 3.3.0 --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index dfbb67d54..51353f693 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -2,7 +2,7 @@ pre-commit==2.6.0 pytest==6.0.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 -pytest-mock==3.2.0 +pytest-mock==3.3.0 moto==1.3.14 pycodestyle==2.6.0 pydocstyle==5.1.0 From bbcb2ba244f3c6627545686a0be35fa8e6751a0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 07:03:46 +0000 Subject: [PATCH 4/8] Bump pre-commit from 2.6.0 to 2.7.1 --- requirements-testing.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-testing.txt b/requirements-testing.txt index 51353f693..a90c75110 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -1,4 +1,4 @@ -pre-commit==2.6.0 +pre-commit==2.7.1 pytest==6.0.1 pytest-asyncio==0.14.0 pytest-cov==2.10.1 From 3ac64b3bb4c23f14c8d0475fb5ec0fbb78b553dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2020 07:03:55 +0000 Subject: [PATCH 5/8] Bump mkdocs-material from 5.5.7 to 5.5.8 --- requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index b11e562b4..5b7b6b710 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,5 +1,5 @@ mkdocs==1.1.2 -mkdocs-material==5.5.7 +mkdocs-material==5.5.8 mkdocs-minify-plugin==0.3.0 mkdocstrings==0.13.0 pymdown-extensions==8.0 From e66c906c50f4bea63f4535e70e96f8f06eeeb773 Mon Sep 17 00:00:00 2001 From: jmmshn Date: Tue, 25 Aug 2020 21:08:16 -0700 Subject: [PATCH 6/8] added context manager for stores and minor bugfix for S3Store.key --- src/maggma/core/store.py | 7 +++++++ src/maggma/stores/aws.py | 5 +++-- tests/stores/test_aws.py | 15 ++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/maggma/core/store.py b/src/maggma/core/store.py index 7ad530937..3d0b6b57f 100644 --- a/src/maggma/core/store.py +++ b/src/maggma/core/store.py @@ -346,6 +346,13 @@ def __setstate__(self, d): d = MontyDecoder().process_decoded(d) self.__init__(**d) + def __enter__(self): + self.connect() + return self + + def __exit__(self, exception_type, exception_value, traceback): + self.close() + class StoreError(Exception): """ General Store-related error """ diff --git a/src/maggma/stores/aws.py b/src/maggma/stores/aws.py index 881f5267f..2a1385887 100644 --- a/src/maggma/stores/aws.py +++ b/src/maggma/stores/aws.py @@ -39,6 +39,7 @@ def __init__( endpoint_url: str = None, sub_dir: str = None, s3_workers: int = 1, + key: str = "task_id", **kwargs, ): """ @@ -66,10 +67,10 @@ def __init__( self.s3_bucket = None # type: Any self.s3_workers = s3_workers # Force the key to be the same as the index + self.index.key = key kwargs["key"] = str(index.key) self._thread_local = threading.local() - super(S3Store, self).__init__(**kwargs) def name(self) -> str: @@ -279,7 +280,7 @@ def update(self, docs: Union[List[Dict], Dict], key: Union[List, str, None] = No for sdoc in fs: search_docs.append(sdoc.result()) # Use store's update to remove key clashes - self.index.update(search_docs) + self.index.update(search_docs, key=self.key) def get_bucket(self): if threading.current_thread().name == "MainThread": diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index b62e9004d..21c85223a 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -63,13 +63,26 @@ def s3store_multi(): conn = boto3.client("s3") conn.create_bucket(Bucket="bucket1") - index = MemoryStore("index'") + index = MemoryStore("index") store = S3Store(index, "bucket1", s3_workers=4) store.connect() yield store +def test_keys(): + with mock_s3(): + conn = boto3.client("s3") + conn.create_bucket(Bucket="bucket1") + index = MemoryStore("index") + store = S3Store(index, "bucket1", s3_workers=4, key="key1") + store.connect() + with pytest.raises(KeyError, match=r".*key1.*"): + store.update({"task_id": "mp-1", "data": "1234"}) + store.update({"key1": "mp-2", "data": "1234"}) + assert store.key == store.index.key == "key1" + + def test_multi_update(s3store, s3store_multi): data = [{"task_id": str(j), "data": "DATA"} for j in range(32)] From 8aac0dd2adf412ef4d8759eeab59e639d19235c4 Mon Sep 17 00:00:00 2001 From: jmmshn Date: Wed, 26 Aug 2020 11:13:49 -0700 Subject: [PATCH 7/8] fixed handling of S3Store keys --- src/maggma/stores/aws.py | 10 +++++++++- tests/stores/test_aws.py | 15 +++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/maggma/stores/aws.py b/src/maggma/stores/aws.py index 2a1385887..8f34d8aea 100644 --- a/src/maggma/stores/aws.py +++ b/src/maggma/stores/aws.py @@ -7,6 +7,7 @@ import zlib from concurrent.futures import wait from concurrent.futures.thread import ThreadPoolExecutor +import warnings from typing import Any, Dict, Iterator, List, Optional, Tuple, Union import msgpack # type: ignore @@ -67,7 +68,14 @@ def __init__( self.s3_bucket = None # type: Any self.s3_workers = s3_workers # Force the key to be the same as the index - self.index.key = key + assert isinstance( + index.key, str + ), "Since we are using the key as a file name in S3, they key must be a string" + if key != index.key: + warnings.warn( + f'The desired S3Store key "{key}" does not match the index key "{index.key}", the index key will be used', + UserWarning, + ) kwargs["key"] = str(index.key) self._thread_local = threading.local() diff --git a/tests/stores/test_aws.py b/tests/stores/test_aws.py index 21c85223a..1a15d0167 100644 --- a/tests/stores/test_aws.py +++ b/tests/stores/test_aws.py @@ -74,12 +74,16 @@ def test_keys(): with mock_s3(): conn = boto3.client("s3") conn.create_bucket(Bucket="bucket1") - index = MemoryStore("index") - store = S3Store(index, "bucket1", s3_workers=4, key="key1") + index = MemoryStore("index", key=1) + with pytest.raises(AssertionError, match=r"Since we are.*"): + store = S3Store(index, "bucket1", s3_workers=4, key=1) + index = MemoryStore("index", key="key1") + with pytest.warns(UserWarning, match=r"The desired S3Store.*$"): + store = S3Store(index, "bucket1", s3_workers=4, key="key2") store.connect() - with pytest.raises(KeyError, match=r".*key1.*"): - store.update({"task_id": "mp-1", "data": "1234"}) - store.update({"key1": "mp-2", "data": "1234"}) + store.update({"key1": "mp-1", "data": "1234"}) + with pytest.raises(KeyError): + store.update({"key2": "mp-2", "data": "1234"}) assert store.key == store.index.key == "key1" @@ -151,7 +155,6 @@ def test_remove(s3store): s3store.update([{"task_id": "mp-2", "data": "asd"}]) s3store.update([{"task_id": "mp-4", "data": "asd"}]) s3store.update({"task_id": "mp-5", "data": "aaa"}) - assert s3store.query_one({"task_id": "mp-2"}) is not None assert s3store.query_one({"task_id": "mp-4"}) is not None From 4084988fe335a97b0838bc5e701ff74ebeb943c3 Mon Sep 17 00:00:00 2001 From: jmmshn Date: Wed, 26 Aug 2020 11:20:47 -0700 Subject: [PATCH 8/8] linting --- src/maggma/stores/aws.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/maggma/stores/aws.py b/src/maggma/stores/aws.py index 8f34d8aea..9b3b12bf8 100644 --- a/src/maggma/stores/aws.py +++ b/src/maggma/stores/aws.py @@ -73,7 +73,8 @@ def __init__( ), "Since we are using the key as a file name in S3, they key must be a string" if key != index.key: warnings.warn( - f'The desired S3Store key "{key}" does not match the index key "{index.key}", the index key will be used', + f'The desired S3Store key "{key}" does not match the index key "{index.key},"' + "the index key will be used", UserWarning, ) kwargs["key"] = str(index.key)