From 5441eb45f9abeebcf7c6868668c6a534b6716db5 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 15:21:58 +0000 Subject: [PATCH 01/17] start digifeeds cli --- Dockerfile | 9 +++++++++ aim/cli/__init__.py | 0 aim/cli/digifeeds.py | 8 ++++++++ aim/cli/main.py | 9 +++++++++ poetry.lock | 2 +- pyproject.toml | 4 ++++ 6 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 aim/cli/__init__.py create mode 100644 aim/cli/digifeeds.py create mode 100644 aim/cli/main.py diff --git a/Dockerfile b/Dockerfile index 071007d..c2c092c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,8 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ # Set the working directory to /app WORKDIR /app +ENV PYTHONPATH="/app" + CMD ["tail", "-f", "/dev/null"] # Both build and development need poetry, so it is its own step. @@ -76,6 +78,13 @@ FROM base as production # Switch to the non-root user "user" # RUN mkdir -p /venv && chown ${UID}:${GID} /venv +<<<<<<< HEAD +======= +# By adding /venv/bin to the PATH the dependencies in the virtual environment +# are used +ENV VIRTUAL_ENV=/venv \ + PATH="/venv/bin:$PATH" +>>>>>>> 2f9cc22 (start digifeeds cli) COPY --chown=${UID}:${GID} . /app COPY --chown=${UID}:${GID} --from=build "/app/requirements.txt" /app/requirements.txt diff --git a/aim/cli/__init__.py b/aim/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py new file mode 100644 index 0000000..bedf735 --- /dev/null +++ b/aim/cli/digifeeds.py @@ -0,0 +1,8 @@ +import typer + +app = typer.Typer() + + +@app.command() +def add_to_db(barcode: str): + print(f'Adding barcode "{barcode}" to database') diff --git a/aim/cli/main.py b/aim/cli/main.py new file mode 100644 index 0000000..9c4602b --- /dev/null +++ b/aim/cli/main.py @@ -0,0 +1,9 @@ +import typer +import aim.cli.digifeeds as digifeeds + +app = typer.Typer() +app.add_typer(digifeeds.app, name="digifeeds") + + +if __name__ == "__main__": + app() diff --git a/poetry.lock b/poetry.lock index 2d5ede9..75fd570 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1506,4 +1506,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "9bbea301b59c5368e45e7f10ab3322d608f1b200fb42c3148f4f92d5b21d8367" +content-hash = "1cabf735a788b82ee17d88fb45f49bec2380c36ffe3e929003f3f118f0a9aaf6" diff --git a/pyproject.toml b/pyproject.toml index 16feacf..a86f67d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,9 @@ authors = ["Monique Rio ", "David Fulmer "] readme = "README.md" packages = [{include = "aim"}] +[tool.poetry.scripts] +aim = "aim.cli.main:app" + [tool.poetry.dependencies] python = "^3.11" requests = "^2.31.0" @@ -14,6 +17,7 @@ mysqlclient = "^2.2.4" fastapi = {extras = ["standard"], version = "^0.114.2"} httpx = "^0.27.2" alembic = "^1.13.2" +typer = "^0.12.5" [tool.poetry.group.dev.dependencies] From bbc5a8f021b87f043945cf2712c98ef13c51c2be Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 15:33:44 +0000 Subject: [PATCH 02/17] getting aim cli to run with python -m --- aim/__main__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 aim/__main__.py diff --git a/aim/__main__.py b/aim/__main__.py new file mode 100644 index 0000000..09fbc41 --- /dev/null +++ b/aim/__main__.py @@ -0,0 +1,3 @@ +from aim.cli.main import app + +app() From 47cf55dd46935fa857415d97b42ecdc05982d566 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 15:54:30 +0000 Subject: [PATCH 03/17] moves load_statuses to the aim cli --- aim/cli/digifeeds.py | 7 +++++++ aim/digifeeds/bin/load_statuses.py | 12 ------------ 2 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 aim/digifeeds/bin/load_statuses.py diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index bedf735..342fe04 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -1,4 +1,5 @@ import typer +from aim.digifeeds.database import models, main app = typer.Typer() @@ -6,3 +7,9 @@ @app.command() def add_to_db(barcode: str): print(f'Adding barcode "{barcode}" to database') + + +@app.command() +def load_statuses(): + with main.SessionLocal() as db_session: + models.load_statuses(session=db_session) diff --git a/aim/digifeeds/bin/load_statuses.py b/aim/digifeeds/bin/load_statuses.py deleted file mode 100644 index df189a2..0000000 --- a/aim/digifeeds/bin/load_statuses.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys -from aim.digifeeds.database.models import load_statuses -from aim.digifeeds.database.main import SessionLocal - - -def main(): - with SessionLocal() as db_session: - load_statuses(session=db_session) - - -if __name__ == "__main__": - sys.exit(main()) From 56f069babc108a4f10fe0844357251f66a58ef58 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 18:03:50 +0000 Subject: [PATCH 04/17] WIP: start adding a script --- aim/digifeeds/add_to_db.py | 2 ++ poetry.lock | 35 ++++++++++++++++++++++++++++++- pyproject.toml | 7 ++++++- tests/digifeeds/test_add_to_db.py | 6 ++++++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 aim/digifeeds/add_to_db.py create mode 100644 tests/digifeeds/test_add_to_db.py diff --git a/aim/digifeeds/add_to_db.py b/aim/digifeeds/add_to_db.py new file mode 100644 index 0000000..4b9a480 --- /dev/null +++ b/aim/digifeeds/add_to_db.py @@ -0,0 +1,2 @@ +def add_to_db(barcode: str): + pass diff --git a/poetry.lock b/poetry.lock index 75fd570..871a2eb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -917,6 +917,20 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-socket" +version = "0.7.0" +description = "Pytest Plugin to disable socket calls during tests" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pytest_socket-0.7.0-py3-none-any.whl", hash = "sha256:7e0f4642177d55d317bbd58fc68c6bd9048d6eadb2d46a89307fa9221336ce45"}, + {file = "pytest_socket-0.7.0.tar.gz", hash = "sha256:71ab048cbbcb085c15a4423b73b619a8b35d6a307f46f78ea46be51b1b7e11b3"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + [[package]] name = "python-dotenv" version = "1.0.1" @@ -1028,6 +1042,25 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "responses" +version = "0.25.3" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "responses-0.25.3-py3-none-any.whl", hash = "sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb"}, + {file = "responses-0.25.3.tar.gz", hash = "sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "rich" version = "13.8.1" @@ -1506,4 +1539,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "1cabf735a788b82ee17d88fb45f49bec2380c36ffe3e929003f3f118f0a9aaf6" +content-hash = "f030c8e46e0eec746562b89ea73f20a7a1338e4d72fc3c100085d3fc8ea88f38" diff --git a/pyproject.toml b/pyproject.toml index a86f67d..312e7cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,10 +24,15 @@ typer = "^0.12.5" pytest = "^8.0.2" pytest-cov = "^5.0.0" ruff = "^0.6.6" +responses = "^0.25.3" +pytest-socket = "^0.7.0" [tool.pytest.ini_options] -addopts = "--cov=aim --cov-report=html --cov-report=term:skip-covered" +addopts = "--cov=aim --cov-report=html --cov-report=term:skip-covered --disable-socket --allow-unix-socket" +testpaths = [ + "tests" +] [build-system] requires = ["poetry-core"] diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py new file mode 100644 index 0000000..2706782 --- /dev/null +++ b/tests/digifeeds/test_add_to_db.py @@ -0,0 +1,6 @@ +import requests + + +def test_response(): + response = requests.get("http://google.com") + assert (response.status_code) == 200 From e3d7c2ff1dd0732a72c65b52bf0681be36765eb4 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 19:11:01 +0000 Subject: [PATCH 05/17] fixes Dockerfile --- Dockerfile | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2c092c..6786a78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ # wheel packages, which are both needed for installing applications like Pandas and Numpy. # The base layer will contain the dependencies shared by the other layers -FROM python:3.11-slim-bookworm as base +FROM python:3.11-slim-bookworm AS base # Allowing the argumenets to be read into the dockerfile. Ex: .env > compose.yml > Dockerfile ARG POETRY_VERSION=1.8.3 @@ -39,7 +39,7 @@ ENV PYTHONPATH="/app" CMD ["tail", "-f", "/dev/null"] # Both build and development need poetry, so it is its own step. -FROM base as poetry +FROM base AS poetry RUN pip install poetry==${POETRY_VERSION} @@ -58,7 +58,7 @@ ENV PYTHONUNBUFFERED=1\ POETRY_VIRTUALENVS_IN_PROJECT=1 \ POETRY_CACHE_DIR=/tmp/poetry_cache -FROM poetry as build +FROM poetry AS build # Just copy the files needed to install the dependencies COPY pyproject.toml poetry.lock README.md ./ @@ -66,7 +66,7 @@ COPY pyproject.toml poetry.lock README.md ./ RUN poetry export --without dev -f requirements.txt --output requirements.txt # We want poetry on in development -FROM poetry as development +FROM poetry AS development RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ git @@ -74,17 +74,10 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ USER app # We don't want poetry on in production, so we copy the needed files form the build stage -FROM base as production +FROM base AS production # Switch to the non-root user "user" # RUN mkdir -p /venv && chown ${UID}:${GID} /venv -<<<<<<< HEAD -======= -# By adding /venv/bin to the PATH the dependencies in the virtual environment -# are used -ENV VIRTUAL_ENV=/venv \ - PATH="/venv/bin:$PATH" ->>>>>>> 2f9cc22 (start digifeeds cli) COPY --chown=${UID}:${GID} . /app COPY --chown=${UID}:${GID} --from=build "/app/requirements.txt" /app/requirements.txt From b2e061e870c5df3894c76ca057dd334281bd8a05 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 21:44:56 +0000 Subject: [PATCH 06/17] adds db client object --- aim/digifeeds/db_client.py | 22 +++++++++++++++ aim/services.py | 1 + compose.yml | 24 ++++++++++++++-- tests/digifeeds/test_db_client.py | 47 +++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 aim/digifeeds/db_client.py create mode 100644 tests/digifeeds/test_db_client.py diff --git a/aim/digifeeds/db_client.py b/aim/digifeeds/db_client.py new file mode 100644 index 0000000..85daa24 --- /dev/null +++ b/aim/digifeeds/db_client.py @@ -0,0 +1,22 @@ +import requests +from aim.services import S + + +class DBClient: + def get_item(self, barcode: str): + url = f"{S.digifeeds_api_url}/{barcode}" + print(url) + response = requests.get(url) + if response.status_code == 200: + return response.json() + elif response.status_code == 404: + return None + else: + response.raise_for_status() + + def add_item(self, barcode: str): + response = requests.post(f"{S.digifeeds_api_url}/{barcode}") + if response.status_code == 200: + return response.json() + else: + response.raise_for_status() diff --git a/aim/services.py b/aim/services.py index b3dffdc..0525bbb 100644 --- a/aim/services.py +++ b/aim/services.py @@ -12,3 +12,4 @@ ) S.test_database = "sqlite:///:memory:" S.ci_on = os.getenv("CI") +S.digifeeds_api_url = os.getenv("DIGIFEEDS_API_URL") or "http://api:8000" diff --git a/compose.yml b/compose.yml index fbdf30a..9a26327 100644 --- a/compose.yml +++ b/compose.yml @@ -9,8 +9,6 @@ services: GID: ${GID:-1000} DEV: ${DEV:-false} POETRY_VERSION: ${POETRY_VERSION:-1.8.3} - ports: - - 8000:8000 env_file: - env.development - .env @@ -31,5 +29,27 @@ services: - MARIADB_PASSWORD=test@123 - MARIADB_DATABASE=database + api: + build: + context: . + target: development + dockerfile: Dockerfile + args: + UID: ${UID:-1000} + GID: ${GID:-1000} + DEV: ${DEV:-false} + POETRY_VERSION: ${POETRY_VERSION:-1.8.3} + ports: + - 8000:8000 + env_file: + - env.development + - .env + volumes: + - .:/app + tty: true + stdin_open: true + command: "poetry run uvicorn aim.digifeeds.database.main:app --host 0.0.0.0 --reload" + + volumes: database: diff --git a/tests/digifeeds/test_db_client.py b/tests/digifeeds/test_db_client.py new file mode 100644 index 0000000..3530f36 --- /dev/null +++ b/tests/digifeeds/test_db_client.py @@ -0,0 +1,47 @@ +import responses +import pytest +from aim.services import S +from aim.digifeeds.db_client import DBClient +from requests.exceptions import HTTPError + + +@responses.activate +def test_get_item_success(): + url = f"{S.digifeeds_api_url}/my_barcode" + responses.get(url, json={"item": "my_item"}, status=200) + item = DBClient().get_item(barcode="my_barcode") + assert item == {"item": "my_item"} + + +@responses.activate +def test_get_item_not_found(): + url = f"{S.digifeeds_api_url}/my_barcode" + responses.get(url, json={"item": "my_item"}, status=404) + item = DBClient().get_item(barcode="my_barcode") + assert item == None + + +@responses.activate +def test_get_item_raises_error(): + url = f"{S.digifeeds_api_url}/my_barcode" + responses.get(url, json={"item": "my_item"}, status=500) + with pytest.raises(Exception) as exc_info: + DBClient().get_item(barcode="my_barcode") + assert exc_info.type is HTTPError + + +@responses.activate +def test_add_item_success(): + url = f"{S.digifeeds_api_url}/my_barcode" + responses.post(url, json={"item": "my_item"}, status=200) + item = DBClient().add_item(barcode="my_barcode") + assert item == {"item": "my_item"} + + +@responses.activate +def test_add_item_failure(): + url = f"{S.digifeeds_api_url}/my_barcode" + responses.post(url, json={"item": "my_item"}, status=404) + with pytest.raises(Exception) as exc_info: + DBClient().add_item(barcode="my_barcode") + assert exc_info.type is HTTPError From c37791e558943b348b47d8ef5062608bd1c5a669 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Wed, 25 Sep 2024 22:08:21 +0000 Subject: [PATCH 07/17] changes SimpleNamespace to NamedTuple --- aim/services.py | 44 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/aim/services.py b/aim/services.py index 0525bbb..b3893d1 100644 --- a/aim/services.py +++ b/aim/services.py @@ -1,15 +1,39 @@ from types import SimpleNamespace +from typing import NamedTuple import os import sqlalchemy as sa -S = SimpleNamespace() -S.mysql_database = sa.engine.URL.create( - drivername="mysql+mysqldb", - username=os.environ["MARIADB_USER"], - password=os.environ["MARIADB_PASSWORD"], - host=os.environ["DATABASE_HOST"], - database=os.environ["MARIADB_DATABASE"], +Services = NamedTuple( + "Services", + [ + ("mysql_database", sa.engine.URL), + ("test_database", str), + ("ci_on", str | None), + ("digifeeds_api_url", str), + ], ) -S.test_database = "sqlite:///:memory:" -S.ci_on = os.getenv("CI") -S.digifeeds_api_url = os.getenv("DIGIFEEDS_API_URL") or "http://api:8000" + +S = Services( + mysql_database=sa.engine.URL.create( + drivername="mysql+mysqldb", + username=os.environ["MARIADB_USER"], + password=os.environ["MARIADB_PASSWORD"], + host=os.environ["DATABASE_HOST"], + database=os.environ["MARIADB_DATABASE"], + ), + test_database="sqlite:///:memory:", + ci_on=os.getenv("CI"), + digifeeds_api_url=os.getenv("DIGIFEEDS_API_URL") or "http://api:8000", +) + +# S = SimpleNamespace() +# S.mysql_database = sa.engine.URL.create( +# drivername="mysql+mysqldb", +# username=os.environ["MARIADB_USER"], +# password=os.environ["MARIADB_PASSWORD"], +# host=os.environ["DATABASE_HOST"], +# database=os.environ["MARIADB_DATABASE"], +# ) +# S.test_database = "sqlite:///:memory:" +# S.ci_on = os.getenv("CI") +# S.digifeeds_api_url = os.getenv("DIGIFEEDS_API_URL") or "http://api:8000" From 7a5fbb10052b459af96036f3f150fb6cb659b25f Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Thu, 26 Sep 2024 13:53:53 +0000 Subject: [PATCH 08/17] adds add_item_status method to db_client --- aim/digifeeds/db_client.py | 26 +++++++++++++++--- tests/digifeeds/test_db_client.py | 44 +++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/aim/digifeeds/db_client.py b/aim/digifeeds/db_client.py index 85daa24..0b9abe5 100644 --- a/aim/digifeeds/db_client.py +++ b/aim/digifeeds/db_client.py @@ -3,9 +3,11 @@ class DBClient: + def __init__(self) -> None: + self.base_url = S.digifeeds_api_url + def get_item(self, barcode: str): - url = f"{S.digifeeds_api_url}/{barcode}" - print(url) + url = self._url(f"items/{barcode}") response = requests.get(url) if response.status_code == 200: return response.json() @@ -15,8 +17,26 @@ def get_item(self, barcode: str): response.raise_for_status() def add_item(self, barcode: str): - response = requests.post(f"{S.digifeeds_api_url}/{barcode}") + url = self._url(f"items/{barcode}") + response = requests.post(url) + if response.status_code == 200: + return response.json() + else: + response.raise_for_status() + + def get_or_add_item(self, barcode: str): + item = self.get_item(barcode) + if not item: + item = self.add_item(barcode) + return item + + def add_item_status(self, barcode: str, status: str): + url = self._url(f"items/{barcode}/status/{status}") + response = requests.put(url) if response.status_code == 200: return response.json() else: response.raise_for_status() + + def _url(self, path) -> str: + return f"{self.base_url}/{path}" diff --git a/tests/digifeeds/test_db_client.py b/tests/digifeeds/test_db_client.py index 3530f36..935a9be 100644 --- a/tests/digifeeds/test_db_client.py +++ b/tests/digifeeds/test_db_client.py @@ -7,7 +7,7 @@ @responses.activate def test_get_item_success(): - url = f"{S.digifeeds_api_url}/my_barcode" + url = f"{S.digifeeds_api_url}/items/my_barcode" responses.get(url, json={"item": "my_item"}, status=200) item = DBClient().get_item(barcode="my_barcode") assert item == {"item": "my_item"} @@ -15,7 +15,7 @@ def test_get_item_success(): @responses.activate def test_get_item_not_found(): - url = f"{S.digifeeds_api_url}/my_barcode" + url = f"{S.digifeeds_api_url}/items/my_barcode" responses.get(url, json={"item": "my_item"}, status=404) item = DBClient().get_item(barcode="my_barcode") assert item == None @@ -23,7 +23,7 @@ def test_get_item_not_found(): @responses.activate def test_get_item_raises_error(): - url = f"{S.digifeeds_api_url}/my_barcode" + url = f"{S.digifeeds_api_url}/items/my_barcode" responses.get(url, json={"item": "my_item"}, status=500) with pytest.raises(Exception) as exc_info: DBClient().get_item(barcode="my_barcode") @@ -32,7 +32,7 @@ def test_get_item_raises_error(): @responses.activate def test_add_item_success(): - url = f"{S.digifeeds_api_url}/my_barcode" + url = f"{S.digifeeds_api_url}/items/my_barcode" responses.post(url, json={"item": "my_item"}, status=200) item = DBClient().add_item(barcode="my_barcode") assert item == {"item": "my_item"} @@ -40,8 +40,42 @@ def test_add_item_success(): @responses.activate def test_add_item_failure(): - url = f"{S.digifeeds_api_url}/my_barcode" + url = f"{S.digifeeds_api_url}/items/my_barcode" responses.post(url, json={"item": "my_item"}, status=404) with pytest.raises(Exception) as exc_info: DBClient().add_item(barcode="my_barcode") assert exc_info.type is HTTPError + + +@responses.activate +def test_get_or_add_item_for_existing_item(): + url = f"{S.digifeeds_api_url}/items/my_barcode" + responses.get(url, json={"item": "my_item"}, status=200) + item = DBClient().get_or_add_item(barcode="my_barcode") + assert item == {"item": "my_item"} + + +@responses.activate +def test_get_or_add_item_for_new_item(): + url = f"{S.digifeeds_api_url}/items/my_barcode" + responses.get(url, json={}, status=404) + responses.post(url, json={"item": "my_item"}, status=200) + item = DBClient().get_or_add_item(barcode="my_barcode") + assert item == {"item": "my_item"} + + +@responses.activate +def test_add_item_status_success(): + url = f"{S.digifeeds_api_url}/items/my_barcode/status/in_zephir" + responses.put(url, json={"item": "my_item"}, status=200) + item = DBClient().add_item_status(barcode="my_barcode", status="in_zephir") + assert item == {"item": "my_item"} + + +@responses.activate +def test_add_item_status_failure(): + url = f"{S.digifeeds_api_url}/items/my_barcode/status/in_zephir" + responses.put(url, json={}, status=500) + with pytest.raises(Exception) as exc_info: + DBClient().add_item_status(barcode="my_barcode", status="in_zephir") + assert exc_info.type is HTTPError From b8a8061a3c17459e623056a4877e4d88152e4a42 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Thu, 26 Sep 2024 15:37:12 +0000 Subject: [PATCH 09/17] adds alma client; refactors db_client --- aim/digifeeds/alma_client.py | 32 ++++++++++++++++++++++ aim/digifeeds/db_client.py | 17 +++++------- aim/services.py | 4 +++ tests/digifeeds/test_alma_client.py | 42 +++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 aim/digifeeds/alma_client.py create mode 100644 tests/digifeeds/test_alma_client.py diff --git a/aim/digifeeds/alma_client.py b/aim/digifeeds/alma_client.py new file mode 100644 index 0000000..68fe902 --- /dev/null +++ b/aim/digifeeds/alma_client.py @@ -0,0 +1,32 @@ +import requests +from aim.services import S + + +class AlmaClient: + def __init__(self) -> None: + self.session = requests.Session() + self.session.headers.update( + { + "content": "application/json", + "Accept": "application/json", + "Authorization": f"apikey { S.alma_api_key }", + } + ) + self.base_url = "https://api-na.hosted.exlibrisgroup.com/almaws/v1" + self.digifeeds_set_id = S.digifeeds_set_id + + def add_barcode_to_digifeeds_set(self, barcode: str) -> None: + url = self._url(f"conf/sets/{self.digifeeds_set_id}") + query = { + "id_type": "BARCODE", + "op": "add_members", + "fail_on_invalid_id": "true", + } + body = {"members": {"member": [{"id": barcode}]}} + response = self.session.post(url, params=query, json=body) + if response.status_code != 200: + response.raise_for_status() + return None + + def _url(self, path: str) -> str: + return f"{self.base_url}/{path}" diff --git a/aim/digifeeds/db_client.py b/aim/digifeeds/db_client.py index 0b9abe5..966d3e2 100644 --- a/aim/digifeeds/db_client.py +++ b/aim/digifeeds/db_client.py @@ -9,20 +9,18 @@ def __init__(self) -> None: def get_item(self, barcode: str): url = self._url(f"items/{barcode}") response = requests.get(url) - if response.status_code == 200: - return response.json() - elif response.status_code == 404: + if response.status_code == 404: return None - else: + elif response.status_code != 200: response.raise_for_status() + return response.json() def add_item(self, barcode: str): url = self._url(f"items/{barcode}") response = requests.post(url) - if response.status_code == 200: - return response.json() - else: + if response.status_code != 200: response.raise_for_status() + return response.json() def get_or_add_item(self, barcode: str): item = self.get_item(barcode) @@ -33,10 +31,9 @@ def get_or_add_item(self, barcode: str): def add_item_status(self, barcode: str, status: str): url = self._url(f"items/{barcode}/status/{status}") response = requests.put(url) - if response.status_code == 200: - return response.json() - else: + if response.status_code != 200: response.raise_for_status() + return response.json() def _url(self, path) -> str: return f"{self.base_url}/{path}" diff --git a/aim/services.py b/aim/services.py index b3893d1..35ff93a 100644 --- a/aim/services.py +++ b/aim/services.py @@ -9,7 +9,9 @@ ("mysql_database", sa.engine.URL), ("test_database", str), ("ci_on", str | None), + ("alma_api_key", str), ("digifeeds_api_url", str), + ("digifeeds_set_id", str), ], ) @@ -24,6 +26,8 @@ test_database="sqlite:///:memory:", ci_on=os.getenv("CI"), digifeeds_api_url=os.getenv("DIGIFEEDS_API_URL") or "http://api:8000", + digifeeds_set_id=os.getenv("DIGIFEEDS_SET_ID") or "digifeeds_set_id", + alma_api_key=os.getenv("ALMA_API_KEY") or "alma_api_key", ) # S = SimpleNamespace() diff --git a/tests/digifeeds/test_alma_client.py b/tests/digifeeds/test_alma_client.py new file mode 100644 index 0000000..d77b5d1 --- /dev/null +++ b/tests/digifeeds/test_alma_client.py @@ -0,0 +1,42 @@ +import responses +from responses import matchers +import pytest +from aim.services import S +from aim.digifeeds.alma_client import AlmaClient +from requests.exceptions import HTTPError + + +@pytest.fixture +def alma_base_url(): + return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" + + +@responses.activate +def test_add_barcode_to_digifeeds_set_success(alma_base_url): + url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + query = {"id_type": "BARCODE", "op": "add_members", "fail_on_invalid_id": "true"} + body = {"members": {"member": [{"id": "my_barcode"}]}} + responses.post( + url, + match=[ + matchers.query_param_matcher(query), + matchers.json_params_matcher(body), + ], + json={}, + status=200, + ) + response = AlmaClient().add_barcode_to_digifeeds_set("my_barcode") + assert response is None + + +@responses.activate +def test_add_barcode_to_digifeeds_set_failure(alma_base_url): + url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + responses.post( + url, + json={}, + status=500, + ) + with pytest.raises(Exception) as exc_info: + AlmaClient().add_barcode_to_digifeeds_set("my_barcode") + assert exc_info.type is HTTPError From 8e4205f58d5b7e399d00584b2707863842e9aaea Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Thu, 26 Sep 2024 21:48:06 +0000 Subject: [PATCH 10/17] adds integration tests --- aim/__main__.py | 4 +- aim/cli/digifeeds.py | 2 + aim/cli/main.py | 2 +- aim/digifeeds/add_to_db.py | 17 ++++++- poetry.lock | 19 ++++++- pyproject.toml | 1 + tests/cli/test_digifeeds.py | 58 +++++++++++++++++++++ tests/digifeeds/test_add_to_db.py | 81 ++++++++++++++++++++++++++++-- tests/digifeeds/test_item.py | 20 ++++++++ tests/fixtures/digifeeds/item.json | 11 ++++ 10 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 tests/cli/test_digifeeds.py create mode 100644 tests/digifeeds/test_item.py create mode 100644 tests/fixtures/digifeeds/item.json diff --git a/aim/__main__.py b/aim/__main__.py index 09fbc41..a625434 100644 --- a/aim/__main__.py +++ b/aim/__main__.py @@ -1,3 +1,3 @@ -from aim.cli.main import app +from aim.cli.main import app # pragma: no cover -app() +app() # pragma: no cover diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index 342fe04..d306dbd 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -1,4 +1,5 @@ import typer +from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db from aim.digifeeds.database import models, main app = typer.Typer() @@ -6,6 +7,7 @@ @app.command() def add_to_db(barcode: str): + add_to_digifeeds_db(barcode) print(f'Adding barcode "{barcode}" to database') diff --git a/aim/cli/main.py b/aim/cli/main.py index 9c4602b..49e62a1 100644 --- a/aim/cli/main.py +++ b/aim/cli/main.py @@ -5,5 +5,5 @@ app.add_typer(digifeeds.app, name="digifeeds") -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover app() diff --git a/aim/digifeeds/add_to_db.py b/aim/digifeeds/add_to_db.py index 4b9a480..ae64423 100644 --- a/aim/digifeeds/add_to_db.py +++ b/aim/digifeeds/add_to_db.py @@ -1,2 +1,17 @@ +from aim.digifeeds.alma_client import AlmaClient +from aim.digifeeds.db_client import DBClient +from aim.digifeeds.item import Item +from requests.exceptions import HTTPError + + def add_to_db(barcode: str): - pass + item = Item(DBClient().get_or_add_item(barcode)) + if not item.has_status("added_to_digifeeds_set"): + try: + AlmaClient().add_barcode_to_digifeeds_set(barcode) + except HTTPError as ext_inst: + errorList = ext_inst.response.json()["errorList"]["error"] + if any(e["errorCode"] == "60120" for e in errorList): + DBClient().add_item_status(barcode=barcode, status="not_found_in_alma") + return None + DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") diff --git a/poetry.lock b/poetry.lock index 871a2eb..d57a4bb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -917,6 +917,23 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "pytest-socket" version = "0.7.0" @@ -1539,4 +1556,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "f030c8e46e0eec746562b89ea73f20a7a1338e4d72fc3c100085d3fc8ea88f38" +content-hash = "c81d62a4b37be18de0c3ce50c48dcd72b486284e456578ca9d6eb35514e55816" diff --git a/pyproject.toml b/pyproject.toml index 312e7cd..e98a117 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ pytest-cov = "^5.0.0" ruff = "^0.6.6" responses = "^0.25.3" pytest-socket = "^0.7.0" +pytest-mock = "^3.14.0" [tool.pytest.ini_options] diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py new file mode 100644 index 0000000..c6dbe46 --- /dev/null +++ b/tests/cli/test_digifeeds.py @@ -0,0 +1,58 @@ +import responses +import pytest +import json +from responses import matchers +from typer.testing import CliRunner +from aim.cli.main import app +from aim.services import S + +runner = CliRunner() + + +@pytest.fixture +def item_data(): + with open("tests/fixtures/digifeeds/item.json") as f: + output = json.load(f) + return output + + +@pytest.fixture +def alma_base_url(): + return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" + + +@responses.activate +def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data, alma_base_url): + item_data["statuses"][0]["name"] = "in_zephir" + get_add_url = f"{S.digifeeds_api_url}/items/some_barcode" + + get_item = responses.get(get_add_url, json={}, status=404) + post_item = responses.post(get_add_url, json=item_data, status=200) + add_item_status = responses.put( + f"{get_add_url}/status/added_to_digifeeds_set", json=item_data, status=200 + ) + + add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_query = { + "id_type": "BARCODE", + "op": "add_members", + "fail_on_invalid_id": "true", + } + add_to_digifeeds_body = {"members": {"member": [{"id": "some_barcode"}]}} + add_to_digifeeds_set = responses.post( + add_to_digifeeds_url, + match=[ + matchers.query_param_matcher(add_to_digifeeds_query), + matchers.json_params_matcher(add_to_digifeeds_body), + ], + json={}, + status=200, + ) + + result = runner.invoke(app, ["digifeeds", "add-to-db", "some_barcode"]) + assert get_item.call_count == 1 + assert post_item.call_count == 1 + assert add_to_digifeeds_set.call_count == 1 + assert add_item_status.call_count == 1 + assert result.exit_code == 0 + assert 'Adding barcode "some_barcode" to database' in result.stdout diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index 2706782..6ed42bb 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -1,6 +1,79 @@ -import requests +# from unittest.mock import patch +# # from aim.digifeeds.alma_client import AlmaClient +import json +import pytest +import responses +from aim.digifeeds.add_to_db import add_to_db +from aim.services import S -def test_response(): - response = requests.get("http://google.com") - assert (response.status_code) == 200 +@pytest.fixture +def alma_base_url(): + return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" + + +@pytest.fixture +def item_data(): + with open("tests/fixtures/digifeeds/item.json") as f: + output = json.load(f) + return output + + +def test_add_to_db_barcode_thats_in_the_digifeeds_set(mocker, item_data): + get_item_mock = mocker.patch( + "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + ) + result = add_to_db("my_barcode") + get_item_mock.assert_called_once() + assert result is None + + +def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): + item_data["statuses"][0]["name"] = "some_other_status" + get_item_mock = mocker.patch( + "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + ) + add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") + add_to_digifeeds_set_mock = mocker.patch( + "aim.digifeeds.alma_client.AlmaClient.add_barcode_to_digifeeds_set" + ) + result = add_to_db("my_barcode") + get_item_mock.assert_called_once() + add_status_mock.assert_called_once() + add_to_digifeeds_set_mock.assert_called_once() + assert result is None + + +@responses.activate +def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data, alma_base_url): + item_data["statuses"][0]["name"] = "some_other_status" + get_item_mock = mocker.patch( + "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + ) + add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") + error_body = { + "errorsExist": True, + "errorList": { + "error": [ + { + "errorCode": "60120", + "errorMessage": "The ID whatever is not valid for content type ITEM and identifier type BARCODE.", + "trackingId": "E01-2609211329-8EKLP-AWAE1781893571", + } + ] + }, + } + add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_set = responses.post( + add_to_digifeeds_url, + json=error_body, + status=400, + ) + + result = add_to_db("my_barcode") + get_item_mock.assert_called_once() + add_status_mock.assert_called_once_with( + barcode="my_barcode", status="not_found_in_alma" + ) + assert add_to_digifeeds_set.call_count == 1 + assert result is None diff --git a/tests/digifeeds/test_item.py b/tests/digifeeds/test_item.py new file mode 100644 index 0000000..c392401 --- /dev/null +++ b/tests/digifeeds/test_item.py @@ -0,0 +1,20 @@ +import pytest +import json +from aim.digifeeds.item import Item + + +@pytest.fixture +def item_data(): + with open("tests/fixtures/digifeeds/item.json") as f: + output = json.load(f) + return output + + +def test_has_status_is_true(item_data): + result = Item(item_data).has_status("added_to_digifeeds_set") + assert result is True + + +def test_has_status_is_false(item_data): + result = Item(item_data).has_status("in_zephir") + assert result is False diff --git a/tests/fixtures/digifeeds/item.json b/tests/fixtures/digifeeds/item.json new file mode 100644 index 0000000..6763228 --- /dev/null +++ b/tests/fixtures/digifeeds/item.json @@ -0,0 +1,11 @@ +{ + "barcode": "some_barcode", + "created_at": "2024-09-25T17:12:39", + "statuses": [ + { + "name": "added_to_digifeeds_set", + "description": "Item has been added to the digifeeds set", + "created_at": "2024-09-25T17:13:28" + } + ] + } \ No newline at end of file From 4282f235ded645d73bd31aa41f099d44bbdf37b1 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 08:54:08 -0400 Subject: [PATCH 11/17] adds way to set not_found_in_alma; tests load_statuses --- .vscode/settings.json | 7 +++++++ aim/digifeeds/add_to_db.py | 2 ++ aim/digifeeds/item.py | 6 ++++++ tests/cli/test_digifeeds.py | 9 ++++++++ tests/digifeeds/test_add_to_db.py | 34 +++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 .vscode/settings.json create mode 100644 aim/digifeeds/item.py diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9b38853 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/aim/digifeeds/add_to_db.py b/aim/digifeeds/add_to_db.py index ae64423..3580bb2 100644 --- a/aim/digifeeds/add_to_db.py +++ b/aim/digifeeds/add_to_db.py @@ -14,4 +14,6 @@ def add_to_db(barcode: str): if any(e["errorCode"] == "60120" for e in errorList): DBClient().add_item_status(barcode=barcode, status="not_found_in_alma") return None + else: + raise ext_inst DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") diff --git a/aim/digifeeds/item.py b/aim/digifeeds/item.py new file mode 100644 index 0000000..2b019cf --- /dev/null +++ b/aim/digifeeds/item.py @@ -0,0 +1,6 @@ +class Item: + def __init__(self, data: dict) -> None: + self.data = data + + def has_status(self, status: str) -> bool: + return any(s["name"] == status for s in self.data["statuses"]) diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index c6dbe46..ac8878d 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -56,3 +56,12 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data, alma_base_url): assert add_item_status.call_count == 1 assert result.exit_code == 0 assert 'Adding barcode "some_barcode" to database' in result.stdout + + +def test_load_statuses(mocker): + session_local_mock = mocker.patch("aim.digifeeds.database.main.SessionLocal") + load_statuse_mock = mocker.patch("aim.digifeeds.database.models.load_statuses") + result = runner.invoke(app, ["digifeeds", "load-statuses"]) + assert session_local_mock.call_count == 1 + assert load_statuse_mock.call_count == 1 + assert result.exit_code == 0 diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index 6ed42bb..628b200 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -4,6 +4,7 @@ import pytest import responses from aim.digifeeds.add_to_db import add_to_db +from requests.exceptions import HTTPError from aim.services import S @@ -77,3 +78,36 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data, alma_base_url) ) assert add_to_digifeeds_set.call_count == 1 assert result is None + + +@responses.activate +def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data, alma_base_url): + item_data["statuses"][0]["name"] = "some_other_status" + get_item_mock = mocker.patch( + "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + ) + add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") + error_body = { + "errorsExist": True, + "errorList": { + "error": [ + { + "errorCode": "60125", + "errorMessage": "some_other_error", + "trackingId": "E01-2609211329-8EKLP-AWAE1781893571", + } + ] + }, + } + add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_set = responses.post( + add_to_digifeeds_url, + json=error_body, + status=400, + ) + + with pytest.raises(Exception) as exc_info: + add_to_db("my_barcode") + assert exc_info.type is HTTPError + get_item_mock.assert_called_once() + add_status_mock.assert_not_called() From a77dc8b2adbfca27a3e213f71151de53083d9a56 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 13:08:09 +0000 Subject: [PATCH 12/17] linting --- aim/services.py | 13 ------------- tests/digifeeds/test_add_to_db.py | 1 + tests/digifeeds/test_db_client.py | 2 +- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/aim/services.py b/aim/services.py index 35ff93a..5149374 100644 --- a/aim/services.py +++ b/aim/services.py @@ -1,4 +1,3 @@ -from types import SimpleNamespace from typing import NamedTuple import os import sqlalchemy as sa @@ -29,15 +28,3 @@ digifeeds_set_id=os.getenv("DIGIFEEDS_SET_ID") or "digifeeds_set_id", alma_api_key=os.getenv("ALMA_API_KEY") or "alma_api_key", ) - -# S = SimpleNamespace() -# S.mysql_database = sa.engine.URL.create( -# drivername="mysql+mysqldb", -# username=os.environ["MARIADB_USER"], -# password=os.environ["MARIADB_PASSWORD"], -# host=os.environ["DATABASE_HOST"], -# database=os.environ["MARIADB_DATABASE"], -# ) -# S.test_database = "sqlite:///:memory:" -# S.ci_on = os.getenv("CI") -# S.digifeeds_api_url = os.getenv("DIGIFEEDS_API_URL") or "http://api:8000" diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index 628b200..250ef60 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -109,5 +109,6 @@ def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data, alma_base_u with pytest.raises(Exception) as exc_info: add_to_db("my_barcode") assert exc_info.type is HTTPError + assert add_to_digifeeds_set.call_count == 1 get_item_mock.assert_called_once() add_status_mock.assert_not_called() diff --git a/tests/digifeeds/test_db_client.py b/tests/digifeeds/test_db_client.py index 935a9be..df9b180 100644 --- a/tests/digifeeds/test_db_client.py +++ b/tests/digifeeds/test_db_client.py @@ -18,7 +18,7 @@ def test_get_item_not_found(): url = f"{S.digifeeds_api_url}/items/my_barcode" responses.get(url, json={"item": "my_item"}, status=404) item = DBClient().get_item(barcode="my_barcode") - assert item == None + assert item is None @responses.activate From 30f1ac3c6e8320f2e0ce45d4a4632594c6d7837d Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 13:17:09 +0000 Subject: [PATCH 13/17] move SessionLocal out of hidden if statement for CI --- aim/digifeeds/database/main.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/aim/digifeeds/database/main.py b/aim/digifeeds/database/main.py index 774bf5f..c0e9ce2 100644 --- a/aim/digifeeds/database/main.py +++ b/aim/digifeeds/database/main.py @@ -4,23 +4,29 @@ from aim.digifeeds.database import crud, schemas from aim.services import S -if S.ci_on is None: # pragma: no cover - engine = create_engine(S.mysql_database) - SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) -# models.Base.metadata.create_all(bind=engine) +# This is here so SessionLocal won't have a problem in tests in github +if S.ci_on: # pragma: no cover + engine = create_engine(S.test_database) +else: # pragma: no cover + engine = create_engine(S.mysql_database) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) app = FastAPI() + # Dependency -def get_db(): # pragma: no cover +def get_db(): # pragma: no cover db = SessionLocal() try: yield db finally: db.close() + @app.get("/items/", response_model_by_alias=False) -def get_items(in_zephir: bool | None = None, db: Session = Depends(get_db)) -> list[schemas.Item]: +def get_items( + in_zephir: bool | None = None, db: Session = Depends(get_db) +) -> list[schemas.Item]: db_items = crud.get_items(in_zephir=in_zephir, db=db) return db_items @@ -32,6 +38,7 @@ def get_item(barcode: str, db: Session = Depends(get_db)) -> schemas.Item: raise HTTPException(status_code=404, detail="Item not found") return db_item + @app.post("/items/{barcode}", response_model_by_alias=False) def create_item(barcode: str, db: Session = Depends(get_db)) -> schemas.Item: item = schemas.ItemCreate(barcode=barcode) @@ -41,18 +48,21 @@ def create_item(barcode: str, db: Session = Depends(get_db)) -> schemas.Item: db_item = crud.add_item(item=item, db=db) return db_item + @app.put("/items/{barcode}/status/{status_name}", response_model_by_alias=False) -def update_item(barcode: str, status_name: str, db: Session=Depends(get_db)) -> schemas.Item: +def update_item( + barcode: str, status_name: str, db: Session = Depends(get_db) +) -> schemas.Item: db_status = crud.get_status(name=status_name, db=db) if db_status is None: raise HTTPException(status_code=404, detail="Status not found") db_item = crud.get_item(barcode=barcode, db=db) if db_item is None: raise HTTPException(status_code=404, detail="Item not found") - return crud.add_item_status(db=db,item=db_item,status=db_status) + return crud.add_item_status(db=db, item=db_item, status=db_status) + @app.get("/statuses") -def get_statuses(db: Session=Depends(get_db)) -> list[schemas.Status]: +def get_statuses(db: Session = Depends(get_db)) -> list[schemas.Status]: db_statuses = crud.get_statuses(db=db) - return db_statuses - + return db_statuses From dc7d809efc1141e2d4af852af75475db4ebcf15a Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 13:29:06 +0000 Subject: [PATCH 14/17] refactor: alma_api_url moved to services --- aim/digifeeds/alma_client.py | 2 +- aim/services.py | 2 ++ tests/cli/test_digifeeds.py | 9 ++------- tests/digifeeds/test_add_to_db.py | 13 ++++--------- tests/digifeeds/test_alma_client.py | 13 ++++--------- 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/aim/digifeeds/alma_client.py b/aim/digifeeds/alma_client.py index 68fe902..dcc3505 100644 --- a/aim/digifeeds/alma_client.py +++ b/aim/digifeeds/alma_client.py @@ -12,7 +12,7 @@ def __init__(self) -> None: "Authorization": f"apikey { S.alma_api_key }", } ) - self.base_url = "https://api-na.hosted.exlibrisgroup.com/almaws/v1" + self.base_url = S.alma_api_url self.digifeeds_set_id = S.digifeeds_set_id def add_barcode_to_digifeeds_set(self, barcode: str) -> None: diff --git a/aim/services.py b/aim/services.py index 5149374..a33b134 100644 --- a/aim/services.py +++ b/aim/services.py @@ -9,6 +9,7 @@ ("test_database", str), ("ci_on", str | None), ("alma_api_key", str), + ("alma_api_url", str), ("digifeeds_api_url", str), ("digifeeds_set_id", str), ], @@ -27,4 +28,5 @@ digifeeds_api_url=os.getenv("DIGIFEEDS_API_URL") or "http://api:8000", digifeeds_set_id=os.getenv("DIGIFEEDS_SET_ID") or "digifeeds_set_id", alma_api_key=os.getenv("ALMA_API_KEY") or "alma_api_key", + alma_api_url="https://api-na.hosted.exlibrisgroup.com/almaws/v1", ) diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index ac8878d..d5eb1c6 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -16,13 +16,8 @@ def item_data(): return output -@pytest.fixture -def alma_base_url(): - return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" - - @responses.activate -def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data, alma_base_url): +def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): item_data["statuses"][0]["name"] = "in_zephir" get_add_url = f"{S.digifeeds_api_url}/items/some_barcode" @@ -32,7 +27,7 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data, alma_base_url): f"{get_add_url}/status/added_to_digifeeds_set", json=item_data, status=200 ) - add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" add_to_digifeeds_query = { "id_type": "BARCODE", "op": "add_members", diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index 250ef60..f14d243 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -8,11 +8,6 @@ from aim.services import S -@pytest.fixture -def alma_base_url(): - return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" - - @pytest.fixture def item_data(): with open("tests/fixtures/digifeeds/item.json") as f: @@ -46,7 +41,7 @@ def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): @responses.activate -def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data, alma_base_url): +def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch( "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data @@ -64,7 +59,7 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data, alma_base_url) ] }, } - add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" add_to_digifeeds_set = responses.post( add_to_digifeeds_url, json=error_body, @@ -81,7 +76,7 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data, alma_base_url) @responses.activate -def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data, alma_base_url): +def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch( "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data @@ -99,7 +94,7 @@ def test_add_to_db_barcode_that_causes_alma_error(mocker, item_data, alma_base_u ] }, } - add_to_digifeeds_url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" + add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" add_to_digifeeds_set = responses.post( add_to_digifeeds_url, json=error_body, diff --git a/tests/digifeeds/test_alma_client.py b/tests/digifeeds/test_alma_client.py index d77b5d1..0104ad1 100644 --- a/tests/digifeeds/test_alma_client.py +++ b/tests/digifeeds/test_alma_client.py @@ -6,14 +6,9 @@ from requests.exceptions import HTTPError -@pytest.fixture -def alma_base_url(): - return "https://api-na.hosted.exlibrisgroup.com/almaws/v1" - - @responses.activate -def test_add_barcode_to_digifeeds_set_success(alma_base_url): - url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" +def test_add_barcode_to_digifeeds_set_success(): + url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" query = {"id_type": "BARCODE", "op": "add_members", "fail_on_invalid_id": "true"} body = {"members": {"member": [{"id": "my_barcode"}]}} responses.post( @@ -30,8 +25,8 @@ def test_add_barcode_to_digifeeds_set_success(alma_base_url): @responses.activate -def test_add_barcode_to_digifeeds_set_failure(alma_base_url): - url = f"{alma_base_url}/conf/sets/{S.digifeeds_set_id}" +def test_add_barcode_to_digifeeds_set_failure(): + url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" responses.post( url, json={}, From b5a32fee48a9aa27b21e65f80317f9a10a551d62 Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 18:15:20 +0000 Subject: [PATCH 15/17] adds a list barcodes in input bucket method to digifeeds cli --- aim/cli/digifeeds.py | 7 + aim/digifeeds/list_barcodes_in_bucket.py | 19 ++ aim/services.py | 11 + poetry.lock | 321 +++++++++++++++++- pyproject.toml | 2 + tests/cli/test_digifeeds.py | 11 + .../digifeeds/test_list_barcodes_in_bucket.py | 26 ++ 7 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 aim/digifeeds/list_barcodes_in_bucket.py create mode 100644 tests/digifeeds/test_list_barcodes_in_bucket.py diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index d306dbd..83b8049 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -1,7 +1,9 @@ import typer from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db +from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket from aim.digifeeds.database import models, main + app = typer.Typer() @@ -15,3 +17,8 @@ def add_to_db(barcode: str): def load_statuses(): with main.SessionLocal() as db_session: models.load_statuses(session=db_session) + + +@app.command() +def list_barcodes_in_input_bucket(): + print(list_barcodes_in_bucket(), end="") diff --git a/aim/digifeeds/list_barcodes_in_bucket.py b/aim/digifeeds/list_barcodes_in_bucket.py new file mode 100644 index 0000000..61bcb80 --- /dev/null +++ b/aim/digifeeds/list_barcodes_in_bucket.py @@ -0,0 +1,19 @@ +import boto3 +from aim.services import S + + +def list_barcodes_in_bucket(): + s3 = boto3.client( + "s3", + aws_access_key_id=S.digifeeds_s3_access_key, + aws_secret_access_key=S.digifeeds_s3_secret_access_key, + ) + prefix = S.digifeeds_s3_input_path + "/" + response = s3.list_objects_v2( + Bucket=S.digifeeds_s3_bucket, + Prefix=prefix, + Delimiter="/", + ) + paths = [object["Prefix"] for object in response["CommonPrefixes"]] + barcodes = [path.split("/")[1] for path in paths] + return barcodes diff --git a/aim/services.py b/aim/services.py index a33b134..44487c8 100644 --- a/aim/services.py +++ b/aim/services.py @@ -12,6 +12,10 @@ ("alma_api_url", str), ("digifeeds_api_url", str), ("digifeeds_set_id", str), + ("digifeeds_s3_access_key", str), + ("digifeeds_s3_secret_access_key", str), + ("digifeeds_s3_bucket", str), + ("digifeeds_s3_input_path", str), ], ) @@ -29,4 +33,11 @@ digifeeds_set_id=os.getenv("DIGIFEEDS_SET_ID") or "digifeeds_set_id", alma_api_key=os.getenv("ALMA_API_KEY") or "alma_api_key", alma_api_url="https://api-na.hosted.exlibrisgroup.com/almaws/v1", + digifeeds_s3_access_key=os.getenv("DIGIFEEDS_S3_ACCESS_KEY") + or "digifeeds_s3_access_key", + digifeeds_s3_secret_access_key=os.getenv("DIGIFEEDS_S3_SECRET_ACCESS_KEY") + or "digifeeds_s3_secret_access_key", + digifeeds_s3_bucket=os.getenv("DIGIFEEDS_S3_BUCKET") or "digifeeds_s3_bucket", + digifeeds_s3_input_path=os.getenv("DIGIFEEDS_S3_INPUT_PATH") + or "path_to_input_barcodes", ) diff --git a/poetry.lock b/poetry.lock index d57a4bb..fa53736 100644 --- a/poetry.lock +++ b/poetry.lock @@ -50,6 +50,44 @@ doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] trio = ["trio (>=0.26.1)"] +[[package]] +name = "boto3" +version = "1.35.28" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.35.28-py3-none-any.whl", hash = "sha256:dc088b86a14f17d3cd2e96915c6ccfd31bce640dfe9180df579ed311bc6bf0fc"}, + {file = "boto3-1.35.28.tar.gz", hash = "sha256:8960fc458b9ba3c8a9890a607c31cee375db821f39aefaec9ff638248e81644a"}, +] + +[package.dependencies] +botocore = ">=1.35.28,<1.36.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.35.28" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.35.28-py3-none-any.whl", hash = "sha256:b66c78f3d6379bd16f0362f07168fa7699cdda3921fc880047192d96f2c8c527"}, + {file = "botocore-1.35.28.tar.gz", hash = "sha256:115d13f2172d8e9fa92e8d913f0e80092b97624d190f46772ed2930d4a355d55"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.21.5)"] + [[package]] name = "certifi" version = "2024.8.30" @@ -61,6 +99,85 @@ files = [ {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] +[[package]] +name = "cffi" +version = "1.17.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, + {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6"}, + {file = "cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e"}, + {file = "cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be"}, + {file = "cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c"}, + {file = "cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c"}, + {file = "cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1"}, + {file = "cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8"}, + {file = "cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16"}, + {file = "cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0"}, + {file = "cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a"}, + {file = "cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e"}, + {file = "cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7"}, + {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -269,6 +386,55 @@ files = [ [package.extras] toml = ["tomli"] +[[package]] +name = "cryptography" +version = "43.0.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, + {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, + {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "dnspython" version = "2.6.1" @@ -574,6 +740,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "mako" version = "1.3.5" @@ -697,6 +874,53 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] +[[package]] +name = "moto" +version = "5.0.15" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "moto-5.0.15-py2.py3-none-any.whl", hash = "sha256:fa1e92ffb55dbfb9fa92a2115a88c32481b75aa3fbd24075d1f29af2f9becffa"}, + {file = "moto-5.0.15.tar.gz", hash = "sha256:57aa8c2af417cc64a0ddfe63e5bcd1ada90f5079b73cdd1f74c4e9fb30a1a7e6"}, +] + +[package.dependencies] +boto3 = ">=1.9.201" +botocore = ">=1.14.0" +cryptography = ">=3.3.1" +Jinja2 = ">=2.10.1" +py-partiql-parser = {version = "0.5.6", optional = true, markers = "extra == \"s3\""} +python-dateutil = ">=2.1,<3.0.0" +PyYAML = {version = ">=5.1", optional = true, markers = "extra == \"s3\""} +requests = ">=2.5" +responses = ">=0.15.0" +werkzeug = ">=0.5,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1" +xmltodict = "*" + +[package.extras] +all = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +apigateway = ["PyYAML (>=5.1)", "joserfc (>=0.9.0)", "openapi-spec-validator (>=0.5.0)"] +apigatewayv2 = ["PyYAML (>=5.1)", "openapi-spec-validator (>=0.5.0)"] +appsync = ["graphql-core"] +awslambda = ["docker (>=3.0.0)"] +batch = ["docker (>=3.0.0)"] +cloudformation = ["PyYAML (>=5.1)", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +cognitoidp = ["joserfc (>=0.9.0)"] +dynamodb = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.6)"] +dynamodbstreams = ["docker (>=3.0.0)", "py-partiql-parser (==0.5.6)"] +events = ["jsonpath-ng"] +glue = ["pyparsing (>=3.0.7)"] +iotdata = ["jsondiff (>=1.1.2)"] +proxy = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=2.5.1)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "multipart", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +resourcegroupstaggingapi = ["PyYAML (>=5.1)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)"] +s3 = ["PyYAML (>=5.1)", "py-partiql-parser (==0.5.6)"] +s3crc32c = ["PyYAML (>=5.1)", "crc32c", "py-partiql-parser (==0.5.6)"] +server = ["PyYAML (>=5.1)", "antlr4-python3-runtime", "aws-xray-sdk (>=0.93,!=0.96)", "cfn-lint (>=0.40.0)", "docker (>=3.0.0)", "flask (!=2.2.0,!=2.2.1)", "flask-cors", "graphql-core", "joserfc (>=0.9.0)", "jsondiff (>=1.1.2)", "jsonpath-ng", "openapi-spec-validator (>=0.5.0)", "py-partiql-parser (==0.5.6)", "pyparsing (>=3.0.7)", "setuptools"] +ssm = ["PyYAML (>=5.1)"] +stepfunctions = ["antlr4-python3-runtime", "jsonpath-ng"] +xray = ["aws-xray-sdk (>=0.93,!=0.96)", "setuptools"] + [[package]] name = "mysqlclient" version = "2.2.4" @@ -741,6 +965,31 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "py-partiql-parser" +version = "0.5.6" +description = "Pure Python PartiQL Parser" +optional = false +python-versions = "*" +files = [ + {file = "py_partiql_parser-0.5.6-py2.py3-none-any.whl", hash = "sha256:622d7b0444becd08c1f4e9e73b31690f4b1c309ab6e5ed45bf607fe71319309f"}, + {file = "py_partiql_parser-0.5.6.tar.gz", hash = "sha256:6339f6bf85573a35686529fc3f491302e71dd091711dfe8df3be89a93767f97b"}, +] + +[package.extras] +dev = ["black (==22.6.0)", "flake8", "mypy", "pytest"] + +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "2.9.2" @@ -948,6 +1197,20 @@ files = [ [package.dependencies] pytest = ">=6.2.5" +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-dotenv" version = "1.0.1" @@ -1123,6 +1386,23 @@ files = [ {file = "ruff-0.6.6.tar.gz", hash = "sha256:0fc030b6fd14814d69ac0196396f6761921bd20831725c7361e1b8100b818034"}, ] +[[package]] +name = "s3transfer" +version = "0.10.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.8" +files = [ + {file = "s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:eca1c20de70a39daee580aef4986996620f365c4e0fda6a86100231d62f1bf69"}, + {file = "s3transfer-0.10.2.tar.gz", hash = "sha256:0711534e9356d3cc692fdde846b4a1e4b0cb6519971860796e6bc4c7aea00ef6"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "shellingham" version = "1.5.4" @@ -1134,6 +1414,17 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -1553,7 +1844,35 @@ files = [ {file = "websockets-13.0.1.tar.gz", hash = "sha256:4d6ece65099411cfd9a48d13701d7438d9c34f479046b34c50ff60bb8834e43e"}, ] +[[package]] +name = "werkzeug" +version = "3.0.4" +description = "The comprehensive WSGI web application library." +optional = false +python-versions = ">=3.8" +files = [ + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog (>=2.3)"] + +[[package]] +name = "xmltodict" +version = "0.13.0" +description = "Makes working with XML feel like you are working with JSON" +optional = false +python-versions = ">=3.4" +files = [ + {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"}, + {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"}, +] + [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "c81d62a4b37be18de0c3ce50c48dcd72b486284e456578ca9d6eb35514e55816" +content-hash = "f95a9065c433c889efe0d87b4446b0a5aac127a62b398712d4be4b037a4fd3b8" diff --git a/pyproject.toml b/pyproject.toml index e98a117..ee7f3fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ fastapi = {extras = ["standard"], version = "^0.114.2"} httpx = "^0.27.2" alembic = "^1.13.2" typer = "^0.12.5" +boto3 = "^1.35.28" [tool.poetry.group.dev.dependencies] @@ -27,6 +28,7 @@ ruff = "^0.6.6" responses = "^0.25.3" pytest-socket = "^0.7.0" pytest-mock = "^3.14.0" +moto = {extras = ["s3"], version = "^5.0.15"} [tool.pytest.ini_options] diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index d5eb1c6..2f1680c 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -60,3 +60,14 @@ def test_load_statuses(mocker): assert session_local_mock.call_count == 1 assert load_statuse_mock.call_count == 1 assert result.exit_code == 0 + + +def test_list_barcodes_in_input_bucket(mocker): + list_barcodes_mock = mocker.patch( + "aim.cli.digifeeds.list_barcodes_in_bucket", + return_value=["barcode1", "barcode2"], + ) + result = runner.invoke(app, ["digifeeds", "list-barcodes-in-input-bucket"]) + assert list_barcodes_mock.call_count == 1 + assert result.exit_code == 0 + assert "['barcode1', 'barcode2']" == result.stdout diff --git a/tests/digifeeds/test_list_barcodes_in_bucket.py b/tests/digifeeds/test_list_barcodes_in_bucket.py new file mode 100644 index 0000000..299fafb --- /dev/null +++ b/tests/digifeeds/test_list_barcodes_in_bucket.py @@ -0,0 +1,26 @@ +import boto3 +from moto import mock_aws +from aim.services import S +from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket + + +@mock_aws +def test_list_barcodes_in_bucket(): + conn = boto3.resource( + "s3", + aws_access_key_id=S.digifeeds_s3_access_key, + aws_secret_access_key=S.digifeeds_s3_secret_access_key, + ) + conn.create_bucket(Bucket=S.digifeeds_s3_bucket) + + barcode1 = conn.Object( + S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode1/some_file.txt" + ) + barcode1.put(Body="some text") + barcode2 = conn.Object( + S.digifeeds_s3_bucket, f"{S.digifeeds_s3_input_path}/barcode2/some_file.txt" + ) + barcode2.put(Body="some text") + + result = list_barcodes_in_bucket() + assert result == ["barcode1", "barcode2"] From 4c30c9bde9a7a5bb8d37a2dead7f626dface4e9a Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 27 Sep 2024 20:03:38 +0000 Subject: [PATCH 16/17] outputs properly formmated json dump --- aim/cli/digifeeds.py | 4 +++- tests/cli/test_digifeeds.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index 83b8049..ed7e9fc 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -2,6 +2,8 @@ from aim.digifeeds.add_to_db import add_to_db as add_to_digifeeds_db from aim.digifeeds.list_barcodes_in_bucket import list_barcodes_in_bucket from aim.digifeeds.database import models, main +import json +import sys app = typer.Typer() @@ -21,4 +23,4 @@ def load_statuses(): @app.command() def list_barcodes_in_input_bucket(): - print(list_barcodes_in_bucket(), end="") + json.dump(list_barcodes_in_bucket(), sys.stdout) diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 2f1680c..53ffd00 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -70,4 +70,4 @@ def test_list_barcodes_in_input_bucket(mocker): result = runner.invoke(app, ["digifeeds", "list-barcodes-in-input-bucket"]) assert list_barcodes_mock.call_count == 1 assert result.exit_code == 0 - assert "['barcode1', 'barcode2']" == result.stdout + assert '["barcode1", "barcode2"]' == result.stdout From 4967ae290802be5c520a02e07854407e9b2de5ef Mon Sep 17 00:00:00 2001 From: Monique Rio Date: Fri, 4 Oct 2024 19:14:50 +0000 Subject: [PATCH 17/17] fixes bug where same status applied twice causes an error --- aim/cli/digifeeds.py | 8 +++++++- aim/digifeeds/add_to_db.py | 14 +++++++++++--- aim/digifeeds/item.py | 4 ++++ tests/cli/test_digifeeds.py | 16 +++++++++++++++- tests/digifeeds/test_add_to_db.py | 25 +++++++++++++++---------- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/aim/cli/digifeeds.py b/aim/cli/digifeeds.py index ed7e9fc..1f6736d 100644 --- a/aim/cli/digifeeds.py +++ b/aim/cli/digifeeds.py @@ -11,8 +11,14 @@ @app.command() def add_to_db(barcode: str): - add_to_digifeeds_db(barcode) print(f'Adding barcode "{barcode}" to database') + item = add_to_digifeeds_db(barcode) + if item.has_status("not_found_in_alma"): + print("Item not found in alma.") + if item.has_status("added_to_digifeeds_set"): + print("Item added to digifeeds set") + else: + print("Item not added to digifeeds set") @app.command() diff --git a/aim/digifeeds/add_to_db.py b/aim/digifeeds/add_to_db.py index 3580bb2..fdb973e 100644 --- a/aim/digifeeds/add_to_db.py +++ b/aim/digifeeds/add_to_db.py @@ -12,8 +12,16 @@ def add_to_db(barcode: str): except HTTPError as ext_inst: errorList = ext_inst.response.json()["errorList"]["error"] if any(e["errorCode"] == "60120" for e in errorList): - DBClient().add_item_status(barcode=barcode, status="not_found_in_alma") - return None + if not item.has_status("not_found_in_alma"): + item = Item( + DBClient().add_item_status( + barcode=barcode, status="not_found_in_alma" + ) + ) + return item else: raise ext_inst - DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") + item = Item( + DBClient().add_item_status(barcode=barcode, status="added_to_digifeeds_set") + ) + return item diff --git a/aim/digifeeds/item.py b/aim/digifeeds/item.py index 2b019cf..ce9f79a 100644 --- a/aim/digifeeds/item.py +++ b/aim/digifeeds/item.py @@ -4,3 +4,7 @@ def __init__(self, data: dict) -> None: def has_status(self, status: str) -> bool: return any(s["name"] == status for s in self.data["statuses"]) + + @property + def barcode(self) -> str: + return self.data["barcode"] diff --git a/tests/cli/test_digifeeds.py b/tests/cli/test_digifeeds.py index 53ffd00..9f5a9db 100644 --- a/tests/cli/test_digifeeds.py +++ b/tests/cli/test_digifeeds.py @@ -5,6 +5,7 @@ from typer.testing import CliRunner from aim.cli.main import app from aim.services import S +from aim.digifeeds.item import Item runner = CliRunner() @@ -23,8 +24,10 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): get_item = responses.get(get_add_url, json={}, status=404) post_item = responses.post(get_add_url, json=item_data, status=200) + more_statuses = item_data.copy() + more_statuses["statuses"].append({"name": "added_to_digifeeds_set"}) add_item_status = responses.put( - f"{get_add_url}/status/added_to_digifeeds_set", json=item_data, status=200 + f"{get_add_url}/status/added_to_digifeeds_set", json=more_statuses, status=200 ) add_to_digifeeds_url = f"{S.alma_api_url}/conf/sets/{S.digifeeds_set_id}" @@ -51,6 +54,17 @@ def test_add_to_db_where_item_is_not_in_digifeeds_set(item_data): assert add_item_status.call_count == 1 assert result.exit_code == 0 assert 'Adding barcode "some_barcode" to database' in result.stdout + assert "Item added to digifeeds set" in result.stdout + + +def test_add_to_db_where_item_is_not_in_alma(item_data, mocker): + item_data["statuses"][0]["name"] = "not_found_in_alma" + item = Item(item_data) + mocker.patch("aim.cli.digifeeds.add_to_digifeeds_db", return_value=item) + + result = runner.invoke(app, ["digifeeds", "add-to-db", "some_barcode"]) + assert "Item not found in alma" in result.stdout + assert "Item not added to digifeeds set" in result.stdout def test_load_statuses(mocker): diff --git a/tests/digifeeds/test_add_to_db.py b/tests/digifeeds/test_add_to_db.py index f14d243..9b8b1e1 100644 --- a/tests/digifeeds/test_add_to_db.py +++ b/tests/digifeeds/test_add_to_db.py @@ -21,23 +21,26 @@ def test_add_to_db_barcode_thats_in_the_digifeeds_set(mocker, item_data): ) result = add_to_db("my_barcode") get_item_mock.assert_called_once() - assert result is None + assert result.barcode == "some_barcode" def test_add_to_db_barcode_thats_not_in_the_digifeeds_set(mocker, item_data): item_data["statuses"][0]["name"] = "some_other_status" get_item_mock = mocker.patch( - "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data + "aim.digifeeds.add_to_db.DBClient.get_or_add_item", return_value=item_data + ) + add_status_mock = mocker.patch( + "aim.digifeeds.add_to_db.DBClient.add_item_status", return_value=item_data ) - add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") add_to_digifeeds_set_mock = mocker.patch( - "aim.digifeeds.alma_client.AlmaClient.add_barcode_to_digifeeds_set" + "aim.digifeeds.add_to_db.AlmaClient.add_barcode_to_digifeeds_set", + return_value=item_data, ) - result = add_to_db("my_barcode") + result = add_to_db("some_barcode") get_item_mock.assert_called_once() add_status_mock.assert_called_once() add_to_digifeeds_set_mock.assert_called_once() - assert result is None + assert result.barcode == "some_barcode" @responses.activate @@ -46,7 +49,9 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): get_item_mock = mocker.patch( "aim.digifeeds.db_client.DBClient.get_or_add_item", return_value=item_data ) - add_status_mock = mocker.patch("aim.digifeeds.db_client.DBClient.add_item_status") + add_status_mock = mocker.patch( + "aim.digifeeds.db_client.DBClient.add_item_status", return_value=item_data + ) error_body = { "errorsExist": True, "errorList": { @@ -66,13 +71,13 @@ def test_add_to_db_barcode_that_is_not_in_alma(mocker, item_data): status=400, ) - result = add_to_db("my_barcode") + result = add_to_db("some_barcode") get_item_mock.assert_called_once() add_status_mock.assert_called_once_with( - barcode="my_barcode", status="not_found_in_alma" + barcode="some_barcode", status="not_found_in_alma" ) assert add_to_digifeeds_set.call_count == 1 - assert result is None + assert result.barcode == "some_barcode" @responses.activate