Add ecr auto push (#12)

* Add ecr auto push

* Fix tests

* Fix tests

---------

Co-authored-by: Nicolas Frank <[email protected]>

.github/workflows/ecr.yml

@@ -0,0 +1,46 @@
+name: Deploy to ECR
+
+on:
+  push:
+    tags:
+      - '*'
+
+permissions:
+  id-token: write   # This is required for requesting the JWT
+  contents: read    # This is required for actions/checkout
+
+jobs:
+
+  build:
+    name: Build and Push Docker Images
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Build Image
+      run: |
+        docker build -t temp:latest -f Dockerfile .
+
+    - name: Configure AWS credentials
+      uses: aws-actions/[email protected]
+
+      with:
+        role-to-assume: arn:aws:iam::671250183987:role/GitHub2ECR  # check terraform output
+
+        role-session-name: GitHub_to_AWS_via_FederatedOIDC
+        aws-region: us-east-1
+
+    - name: Login to Amazon ECR
+      id: login-ecr-management
+      uses: aws-actions/amazon-ecr-login@v2
+
+    - name: Tag and Push Image to Amazon ECR
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr-management.outputs.registry }}
+        ECR_REPOSITORY: ml-repository
+        IMAGE_TAG: scholarag-${{ github.ref_name }}
+      run: |
+        docker tag temp:latest $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+        docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG

CHANGELOG.md

@@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.0.7] - 30.10.2024
+
+### Added
+- Auto push to ecr.
 
 ## [0.0.6] - 19.09.2024
 

src/scholarag/__init__.py

@@ -1,3 +1,3 @@
 """Scholarag: A pipeline for RAG on scientific articles."""
 
-__version__ = "0.0.6"
+__version__ = "0.0.7"

tests/app/test_retrieval.py

@@ -190,10 +190,10 @@ async def test_article_count(
 
 
 @pytest.mark.asyncio
-async def test_article_listing(get_testing_async_ds_client, request):
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
+async def test_article_listing(get_testing_async_ds_client, mock_http_calls):
     ds_client, parameters = get_testing_async_ds_client
 
-    request.getfixturevalue("mock_http_calls")
     test_settings = Settings(
         db={
             "db_type": (

tests/conftest.py

@@ -187,7 +187,7 @@ def disable_sentry(monkeypatch):
 
 
 @pytest.fixture()
-def mock_http_calls(httpx_mock, non_mocked_hosts):
+def mock_http_calls(httpx_mock):
     httpx_mock.add_response(
         url="https://portal.issn.org/resource/ISSN/1234-5678",
         method="GET",
@@ -199,8 +199,6 @@ def mock_http_calls(httpx_mock, non_mocked_hosts):
         method="GET",
         json={"citationCount": 2},
     )
-    non_mocked_hosts.append("testserver")
-    non_mocked_hosts.append("test")
     with (
         patch("scholarag.retrieve_metadata.recreate_abstract") as abstract,
         patch("scholarag.retrieve_metadata.get_impact_factors") as impact,

tests/scripts/test_parse_and_upload.py

@@ -64,6 +64,7 @@ async def test_raise_error(tmp_path):
             )
 
 
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 @pytest.mark.asyncio
 async def test_run(tmp_path, httpx_mock):
     # Create two files
@@ -117,12 +118,6 @@ async def test_run(tmp_path, httpx_mock):
 
         # Call specifying the directory with two files
         # Reset mock calls
-        httpx_mock.reset(assert_all_responses_were_requested=False)
-        httpx_mock.add_response(
-            method="POST",
-            url="http://localhost/fake_parser",
-            json=content,
-        )
 
         # Launching the function
         await run(
@@ -137,14 +132,15 @@ async def test_run(tmp_path, httpx_mock):
             index="paragraphs",
         )
 
-    assert len(httpx_mock.get_requests()) == 2
+    assert len(httpx_mock.get_requests()) == 3
 
 
 async def fake_close(self):
     pass
 
 
 @pytest.mark.asyncio
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 @patch("scholarag.document_stores.AsyncOpenSearch.close", new=fake_close)
 async def test_run_with_es_instance(tmp_path, httpx_mock, get_testing_async_ds_client):
     ds_client, parameters = get_testing_async_ds_client

tests/scripts/test_pmc_parse_and_upload.py

@@ -10,6 +10,7 @@
 
 
 @pytest.mark.asyncio
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 async def test_pmc_parse_and_upload(httpx_mock, get_testing_async_ds_client):
     ds_client, parameters = get_testing_async_ds_client
     index = "paragraphs_parse_script_pytest"

tests/services/test_etl.py

@@ -20,6 +20,7 @@
         "pypdf_pdf",
     ],
 )
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 def test_run(parser, tmp_path, httpx_mock):
     """Test etl."""
     extension = "xml" if "xml" in parser else "pdf"
@@ -83,6 +84,7 @@ def test_run(parser, tmp_path, httpx_mock):
         "pypdf_pdf",
     ],
 )
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 @pytest.mark.asyncio
 async def test_arun(parser, tmp_path, httpx_mock):
     """Test etl."""

tests/test_retrieve_metadata.py

@@ -500,6 +500,7 @@ async def test_metadata_retriever_recreate_abstract_with_db(
 @pytest.mark.parametrize(
     ["issn", "name"], [("1234-5678", "great_journal"), ("3426-1936", "bad_journal")]
 )
+@pytest.mark.httpx_mock(can_send_already_matched_responses=True)
 async def test_get_journal_name(httpx_mock, issn, name):
     # If ISSN is None
     async with httpx.AsyncClient() as client:
@@ -556,8 +557,6 @@ async def test_get_journal_name(httpx_mock, issn, name):
         name_found = await get_journal_name(issn, client)
     assert name == name_found
 
-    httpx_mock.reset(assert_all_responses_were_requested=False)
-
     httpx_mock.add_response(
         url=f"https://portal.issn.org/resource/ISSN/{issn}",
         method="GET",
@@ -568,8 +567,6 @@ async def test_get_journal_name(httpx_mock, issn, name):
         name = await get_journal_name(issn, client)
     assert not name
 
-    httpx_mock.reset(assert_all_responses_were_requested=False)
-
     httpx_mock.add_response(
         url=f"https://portal.issn.org/resource/ISSN/{issn}",
         method="GET",