From 915dc9abd8332ff1ac50bca75cac441a0e05f7bb Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Fri, 26 Jan 2024 09:38:08 -0500 Subject: [PATCH 01/10] preparing dev workflow for mongomock use * also add more python version testing * both modeled on conftrak changes --- .github/workflows/testing.yml | 7 ++----- requirements-dev.txt | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a258940..7f88111 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] dependencies: ["pip", "conda"] fail-fast: false @@ -29,9 +29,6 @@ jobs: - name: Checkout the code uses: actions/checkout@v2 - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.6.0 - - name: Set up Python ${{ matrix.python-version }} with conda uses: conda-incubator/setup-miniconda@v2 with: @@ -56,7 +53,7 @@ jobs: if: matrix.dependencies == 'conda' run: | set -vxeo pipefail - conda install -y -c conda-forge six mongoquery doct jsonschema mock pymongo pytest pyyaml requests tornado ujson + conda install -y -c conda-forge six mongoquery doct jsonschema mock mongomock pymongo pytest pyyaml requests tornado ujson - name: Install the package run: | diff --git a/requirements-dev.txt b/requirements-dev.txt index 93253de..6c32384 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ mock +mongomock pytest From a415687fbc4de0560bd69e5daec4c2d6a4a8befb Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 7 Feb 2024 16:50:21 -0500 Subject: [PATCH 02/10] enable testing mode for AStore * mongomock - will be used for tests --- analysisstore/server/astore.py | 39 +++++++--------------------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/analysisstore/server/astore.py b/analysisstore/server/astore.py index 8a2891e..81c964f 100644 --- a/analysisstore/server/astore.py +++ b/analysisstore/server/astore.py @@ -5,7 +5,7 @@ class AStore: - def __init__(self, config): + def __init__(self, config, testing=False): """Given a database configuration that consists of uri and database, instantiate an AStore object that handles the connections to the database. @@ -15,36 +15,13 @@ def __init__(self, config): config: dict uri in string format, and database """ - self.client = MongoClient(config['uri']) - self.database = self.client[config['database']] - self.database.analysis_header.create_index([('uid', DESCENDING)], - unique=True, - background=True) - self.database.analysis_header.create_index([('time', DESCENDING)], - unique=False, - background=True) - self.database.analysis_tail.create_index([('analysis_header', - DESCENDING)], - unique=True, background=True) - self.database.analysis_tail.create_index([('uid', DESCENDING)], - unique=True, background=True) - self.database.analysis_tail.create_index([('time', DESCENDING)], - unique=False, background=True) - self.database.data_reference_header.create_index([('analysis_header', - DESCENDING)], - unique=True, background=False) - self.database.data_reference_header.create_index([('uid', DESCENDING)], - unique=True, background=False) - self.database.data_reference_header.create_index([('time', DESCENDING)], - unique=False) - self.database.data_reference.create_index([('time', DESCENDING), - ('data_reference_header', - DESCENDING)]) - self.database.data_reference.create_index([('uid', DESCENDING)], - unique=True) - self.database.data_reference.create_index([('data_reference_header', - DESCENDING)], - unique=False) + if not testing: + self.client = MongoClient(config["uri"]) + else: + import mongomock + + self.client = mongomock.MongoClient(config["uri"]) + self.database = self.client[config["database"]] def doc_or_uid_to_uid(self, doc_or_uid): """Given Document or uid return the uid From f3a979ca14bbfe2a8dbf5b446f641a53239ea892 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Fri, 26 Jan 2024 11:06:54 -0500 Subject: [PATCH 03/10] add conftest * similar structure as used for conftrak tests --- analysisstore/test/conftest.py | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 analysisstore/test/conftest.py diff --git a/analysisstore/test/conftest.py b/analysisstore/test/conftest.py new file mode 100644 index 0000000..be34a65 --- /dev/null +++ b/analysisstore/test/conftest.py @@ -0,0 +1,49 @@ +import time as ttime +import uuid + +import pytest +import requests +import subprocess +import os +import sys +import uuid +from analysisstore.client.commands import AnalysisClient +import contextlib + +testing_config = dict( + host="localhost", + port=7601, + timezone="US/Eastern", + service_port=7601, + database="astoretest{0}".format(str(uuid.uuid4())), + mongo_uri="mongodb://localhost", + mongo_host="localhost", + mongo_port=27017, +) + + +@contextlib.contextmanager +def astore_startup(): + ps = subprocess.Popen( + [ + sys.executable, + "-c", + f"from analysisstore.ignition import start_server; start_server(args={testing_config}, testing=True) ", + ], + ) + ttime.sleep(1.3) # make sure the process is started + yield ps + + +@pytest.fixture(scope="session") +def astore_server(): + with astore_startup() as astore_fixture: + yield + + +@pytest.fixture(scope="function") +def astore_client(): + c = AnalysisClient( + host=testing_config["mongo_host"], port=testing_config["service_port"] + ) + return c From 9096dc2a52271e4ad39a63c0a5704942c9ee6f51 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 7 Feb 2024 17:00:06 -0500 Subject: [PATCH 04/10] fix commandline options for ignition, move test mode into config --- analysisstore/ignition.py | 13 ++++++++++--- analysisstore/test/conftest.py | 7 +++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/analysisstore/ignition.py b/analysisstore/ignition.py index 3bba012..b6f8939 100644 --- a/analysisstore/ignition.py +++ b/analysisstore/ignition.py @@ -46,6 +46,9 @@ def start_server(config=None): help='port listen to for clients') parser.add_argument('--log-file_prefix', dest='log_file_prefix', type=str, help='Log file name that tornado logs are dumped') + parser.add_argument( + "--testing", dest="testing", type=bool, help="Run server in test mode" + ) args = parser.parse_args() if args.database is not None: config['database'] = args.database @@ -55,11 +58,15 @@ def start_server(config=None): config['timezone'] = args.timezone if args.service_port is not None: config['service_port'] = args.service_port + config["testing"] = args.testing or None + config["log_file_prefix"] = args.log_file_prefix or None if not config: - raise KeyError('No configuration provided. Provide config file or command line args') - tornado.options.parse_command_line({'log_file_prefix': args.log_file_prefix}) + raise KeyError( + "No configuration provided. Provide config file or command line args" + ) + tornado.options.parse_command_line({'log_file_prefix': config["log_file_prefix"]}) cfg = dict(uri=config['mongo_uri'], database=config['database']) - astore = AStore(cfg) + astore = AStore(cfg, testing=config["testing"]) application = tornado.web.Application([(r'/analysis_header', AnalysisHeaderHandler), (r'/data_reference', DataReferenceHandler), (r'/data_reference_header', diff --git a/analysisstore/test/conftest.py b/analysisstore/test/conftest.py index be34a65..adde528 100644 --- a/analysisstore/test/conftest.py +++ b/analysisstore/test/conftest.py @@ -2,9 +2,7 @@ import uuid import pytest -import requests import subprocess -import os import sys import uuid from analysisstore.client.commands import AnalysisClient @@ -19,6 +17,7 @@ mongo_uri="mongodb://localhost", mongo_host="localhost", mongo_port=27017, + testing=True, ) @@ -28,7 +27,7 @@ def astore_startup(): [ sys.executable, "-c", - f"from analysisstore.ignition import start_server; start_server(args={testing_config}, testing=True) ", + f"from analysisstore.ignition import start_server; start_server(config={testing_config}) ", ], ) ttime.sleep(1.3) # make sure the process is started @@ -44,6 +43,6 @@ def astore_server(): @pytest.fixture(scope="function") def astore_client(): c = AnalysisClient( - host=testing_config["mongo_host"], port=testing_config["service_port"] + {"host": testing_config["mongo_host"], "port": testing_config["service_port"]} ) return c From 76b4025c86160a8423d7e0fe9131c09a3dcdb790 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Mon, 29 Jan 2024 11:46:51 -0500 Subject: [PATCH 05/10] restructure for fixtures and functions * model on conftrak's tests --- analysisstore/test/test_client.py | 199 ++++++++++++++---------------- 1 file changed, 93 insertions(+), 106 deletions(-) diff --git a/analysisstore/test/test_client.py b/analysisstore/test/test_client.py index c82f70f..5b0f5b0 100644 --- a/analysisstore/test/test_client.py +++ b/analysisstore/test/test_client.py @@ -1,114 +1,101 @@ from ..client.commands import AnalysisClient from .testing import TESTING_CONFIG import pytest -from .testing import astore_setup, astore_teardown import time import requests -from doct import Document import uuid -class TestConnClient: - def setup_class(self): - self.proc = astore_setup() - self.conn = AnalysisClient(TESTING_CONFIG) - - def test_conn_switch(self): - w_conf = dict(host='wrong_host', - port=0) - tmp_conn = AnalysisClient(w_conf) - tmp_conn.host == w_conf['host'] - tmp_conn.port == 0 - pytest.raises(requests.exceptions.ConnectionError, - tmp_conn.connection_status) - - def test_urls(self): - """Catch potentially annoying and difficult to debug typos""" - base_test_url = 'http://{}:{}/'.format(TESTING_CONFIG['host'], - TESTING_CONFIG['port']) - self.conn._host_url == base_test_url - self.conn.aheader_url == base_test_url + 'analysis_header' - self.conn.atail_url == base_test_url + 'analysis_tail' - self.conn.dref_url == base_test_url + 'data_reference' - self.conn.dref_header_url == base_test_url + 'data_reference_header' - - def test_doc_or_uid_to_uid(self): - m_uid=str(uuid.uuid4()) - test_dict = {'name': 'test_doc', 'uid': m_uid} - m_uid == self.conn._doc_or_uid_to_uid(test_dict) - - def test_post_fact(self): - pld = {'data': 'bogus'} - sig = 'bogus' - res = self.conn._post_factory(signature=sig, - payload=pld) - res['payload'] == pld - res['signature'] == sig - - def test_query_fact(self): - pld = {'data': 'bogus'} - sig = 'bogus' - res = self.conn._query_factory(signature=sig, - payload=pld) - res['payload'] == pld - res['signature'] == sig - - def test_header_insert(self): - pytest.raises(TypeError, self.conn.insert_analysis_header) - m_uid = str(uuid.uuid4()) - rid = self.conn.insert_analysis_header(uid=m_uid, time=time.time(), - provenance={'version': 1.1}, - custom=False) - rid == m_uid - - def generate_ahdr(self): - hid = self.conn.insert_analysis_header(uid=str(uuid.uuid4()), - time=time.time(), - provenance={'version': 1.1}, - custom=False) - return hid - - def generate_dref(self, hdr_id): - did = self.conn.insert_analysis_tail(uid=str(uuid.uuid4()), - analysis_header=hdr_id, - time=time.time(), - exit_status='test') - return did - - def test_tail_insert(self): - pytest.raises(TypeError, self.conn.insert_analysis_tail) - t_uid = str(uuid.uuid4()) - t = self.conn.insert_analysis_tail(uid=t_uid, - analysis_header=self.generate_ahdr(), - time=time.time(), exit_status='test') - t_uid == t - - - - def test_dref_header_insert(self): - pytest.raises(TypeError, self.conn.insert_data_reference_header) - dh_uid = str(uuid.uuid4()) - dh_id = self.conn.insert_data_reference_header(analysis_header=self.generate_ahdr(), - time=time.time(), - uid=dh_uid, - data_keys={}) - dh_id == dh_uid - - - def test_dref_insert(self): - pass - - def test_header_find(self): - pass - - def test_tail_find(self): - pass - - def test_dref_header_find(self): - pass - - def test_dref_find(self): - pass - - def teardown_class(self): - astore_teardown(self.proc) +def test_conn_switch(astore_client): + w_conf = dict(host="wrong_host", port=0) + tmp_conn = AnalysisClient(w_conf) + tmp_conn.host == w_conf["host"] + tmp_conn.port == 0 + pytest.raises(requests.exceptions.ConnectionError, tmp_conn.connection_status) + + +def test_urls(astore_client): + """Catch potentially annoying and difficult to debug typos""" + base_test_url = "http://{}:{}/".format( + TESTING_CONFIG["host"], TESTING_CONFIG["port"] + ) + astore_client._host_url == base_test_url + astore_client.aheader_url == base_test_url + "analysis_header" + astore_client.atail_url == base_test_url + "analysis_tail" + astore_client.dref_url == base_test_url + "data_reference" + astore_client.dref_header_url == base_test_url + "data_reference_header" + + +def test_doc_or_uid_to_uid(astore_client): + m_uid = str(uuid.uuid4()) + test_dict = {"name": "test_doc", "uid": m_uid} + m_uid == astore_client._doc_or_uid_to_uid(test_dict) + + +def test_post_fact(astore_client): + pld = {"data": "bogus"} + sig = "bogus" + res = astore_client._post_factory(signature=sig, payload=pld) + res["payload"] == pld + res["signature"] == sig + + +def test_query_fact(astore_client): + pld = {"data": "bogus"} + sig = "bogus" + res = astore_client._query_factory(signature=sig, payload=pld) + res["payload"] == pld + res["signature"] == sig + + +def test_header_insert(astore_server, astore_client): + pytest.raises(TypeError, astore_client.insert_analysis_header) + m_uid = str(uuid.uuid4()) + rid = astore_client.insert_analysis_header( + uid=m_uid, time=time.time(), provenance={"version": 1.1}, custom=False + ) + rid == m_uid + + +def generate_ahdr(astore_client): + hid = astore_client.insert_analysis_header( + uid=str(uuid.uuid4()), + time=time.time(), + provenance={"version": 1.1}, + custom=False, + ) + return hid + + +def generate_dref(astore_client, hdr_id): + did = astore_client.insert_analysis_tail( + uid=str(uuid.uuid4()), + analysis_header=hdr_id, + time=time.time(), + exit_status="test", + ) + return did + + +def test_tail_insert(astore_client): + pytest.raises(TypeError, astore_client.insert_analysis_tail) + t_uid = str(uuid.uuid4()) + t = astore_client.insert_analysis_tail( + uid=t_uid, + analysis_header=generate_ahdr(astore_client), + time=time.time(), + exit_status="test", + ) + t_uid == t + + +def test_dref_header_insert(astore_client): + pytest.raises(TypeError, astore_client.insert_data_reference_header) + dh_uid = str(uuid.uuid4()) + dh_id = astore_client.insert_data_reference_header( + analysis_header=generate_ahdr(astore_client), + time=time.time(), + uid=dh_uid, + data_keys={}, + ) + dh_id == dh_uid From e76a61134cf6be9c08d89ecd6972ea10e522a9a0 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Mon, 29 Jan 2024 11:55:59 -0500 Subject: [PATCH 06/10] simplify test --- analysisstore/test/test_conn_pool.py | 32 ++++++---------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/analysisstore/test/test_conn_pool.py b/analysisstore/test/test_conn_pool.py index e01cb0e..aa5ece0 100644 --- a/analysisstore/test/test_conn_pool.py +++ b/analysisstore/test/test_conn_pool.py @@ -1,30 +1,12 @@ from ..client.commands import AnalysisClient from .testing import TESTING_CONFIG import pytest -from .testing import astore_setup, astore_teardown - -class TestConnPool: - def setup_class(self): - self.proc = astore_setup() - - def test_client_default(self): - pytest.raises(TypeError, AnalysisClient) - - - def test_client_wconf(self): - config = {'host': TESTING_CONFIG['host'], - 'port': TESTING_CONFIG['port']} - conn = AnalysisClient(config) - - def test_client_badconf(self): - config = {'host': 'localhost'} - pytest.raises(KeyError, AnalysisClient, config) - config['port'] = TESTING_CONFIG['port'] - conn = AnalysisClient(config) - conn.host == TESTING_CONFIG['host'] - conn.port == TESTING_CONFIG['port'] - - def teardown_class(self): - astore_teardown(self.proc) +def test_client_badconf(): + config = {"host": "localhost"} + pytest.raises(KeyError, AnalysisClient, config) + config["port"] = TESTING_CONFIG["port"] + conn = AnalysisClient(config) + conn.host == TESTING_CONFIG["host"] + conn.port == TESTING_CONFIG["port"] From 2ee53174ae9ce6eb17ec01dd90c7604f3c8dba06 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Mon, 5 Feb 2024 13:50:20 -0500 Subject: [PATCH 07/10] move to use conftest parameters * other file has been removed --- analysisstore/test/test_client.py | 16 ++++++++-------- analysisstore/test/test_conn_pool.py | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/analysisstore/test/test_client.py b/analysisstore/test/test_client.py index 5b0f5b0..e4cf615 100644 --- a/analysisstore/test/test_client.py +++ b/analysisstore/test/test_client.py @@ -1,12 +1,12 @@ from ..client.commands import AnalysisClient -from .testing import TESTING_CONFIG +from .conftest import testing_config import pytest import time import requests import uuid -def test_conn_switch(astore_client): +def test_conn_switch(astore_server, astore_client): w_conf = dict(host="wrong_host", port=0) tmp_conn = AnalysisClient(w_conf) tmp_conn.host == w_conf["host"] @@ -17,7 +17,7 @@ def test_conn_switch(astore_client): def test_urls(astore_client): """Catch potentially annoying and difficult to debug typos""" base_test_url = "http://{}:{}/".format( - TESTING_CONFIG["host"], TESTING_CONFIG["port"] + testing_config["host"], testing_config["port"] ) astore_client._host_url == base_test_url astore_client.aheader_url == base_test_url + "analysis_header" @@ -26,13 +26,13 @@ def test_urls(astore_client): astore_client.dref_header_url == base_test_url + "data_reference_header" -def test_doc_or_uid_to_uid(astore_client): +def test_doc_or_uid_to_uid(astore_server, astore_client): m_uid = str(uuid.uuid4()) test_dict = {"name": "test_doc", "uid": m_uid} m_uid == astore_client._doc_or_uid_to_uid(test_dict) -def test_post_fact(astore_client): +def test_post_fact(astore_server, astore_client): pld = {"data": "bogus"} sig = "bogus" res = astore_client._post_factory(signature=sig, payload=pld) @@ -40,7 +40,7 @@ def test_post_fact(astore_client): res["signature"] == sig -def test_query_fact(astore_client): +def test_query_fact(astore_server, astore_client): pld = {"data": "bogus"} sig = "bogus" res = astore_client._query_factory(signature=sig, payload=pld) @@ -77,7 +77,7 @@ def generate_dref(astore_client, hdr_id): return did -def test_tail_insert(astore_client): +def test_tail_insert(astore_server, astore_client): pytest.raises(TypeError, astore_client.insert_analysis_tail) t_uid = str(uuid.uuid4()) t = astore_client.insert_analysis_tail( @@ -89,7 +89,7 @@ def test_tail_insert(astore_client): t_uid == t -def test_dref_header_insert(astore_client): +def test_dref_header_insert(astore_server, astore_client): pytest.raises(TypeError, astore_client.insert_data_reference_header) dh_uid = str(uuid.uuid4()) dh_id = astore_client.insert_data_reference_header( diff --git a/analysisstore/test/test_conn_pool.py b/analysisstore/test/test_conn_pool.py index aa5ece0..7d18207 100644 --- a/analysisstore/test/test_conn_pool.py +++ b/analysisstore/test/test_conn_pool.py @@ -1,12 +1,12 @@ from ..client.commands import AnalysisClient -from .testing import TESTING_CONFIG +from .conftest import testing_config import pytest def test_client_badconf(): config = {"host": "localhost"} pytest.raises(KeyError, AnalysisClient, config) - config["port"] = TESTING_CONFIG["port"] + config["port"] = testing_config["port"] conn = AnalysisClient(config) - conn.host == TESTING_CONFIG["host"] - conn.port == TESTING_CONFIG["port"] + conn.host == testing_config["host"] + conn.port == testing_config["port"] From 55ee44df922a585877989da30513a18e24565d35 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 7 Feb 2024 17:06:59 -0500 Subject: [PATCH 08/10] fix import of AStore --- analysisstore/ignition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysisstore/ignition.py b/analysisstore/ignition.py index b6f8939..d38e7b0 100644 --- a/analysisstore/ignition.py +++ b/analysisstore/ignition.py @@ -4,7 +4,7 @@ import sys import tornado.ioloop import tornado.options -from server.astore import AStore +from analysisstore.server.astore import AStore from analysisstore.server.engine import (AnalysisHeaderHandler, AnalysisTailHandler, DataReferenceHeaderHandler, From 58f80d739bf374d9aff003b34090d26a3e18a49c Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 7 Feb 2024 17:07:15 -0500 Subject: [PATCH 09/10] set log_file_prefix for testing --- analysisstore/test/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/analysisstore/test/conftest.py b/analysisstore/test/conftest.py index adde528..655b565 100644 --- a/analysisstore/test/conftest.py +++ b/analysisstore/test/conftest.py @@ -18,6 +18,7 @@ mongo_host="localhost", mongo_port=27017, testing=True, + log_file_prefix="testing", ) From 61dfeea6665c239959af90f51935967ea760c6d2 Mon Sep 17 00:00:00 2001 From: Jun Aishima Date: Wed, 7 Feb 2024 17:10:08 -0500 Subject: [PATCH 10/10] fix to relative import --- analysisstore/ignition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analysisstore/ignition.py b/analysisstore/ignition.py index d38e7b0..ee89a02 100644 --- a/analysisstore/ignition.py +++ b/analysisstore/ignition.py @@ -4,7 +4,7 @@ import sys import tornado.ioloop import tornado.options -from analysisstore.server.astore import AStore +from .server.astore import AStore from analysisstore.server.engine import (AnalysisHeaderHandler, AnalysisTailHandler, DataReferenceHeaderHandler,