Add object creation api

- Ingest and plain object endpoint

api/routes/ingest.py

@@ -1,6 +1,8 @@
 import os
+from typing import Union
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, UploadFile
+from starlette.datastructures import UploadFile as StarletteUploadFile
 from sqlalchemy import insert, select, update, and_, delete
 from sqlalchemy.orm import selectinload, joinedload, defer
 import minio
@@ -13,6 +15,7 @@
 from api.routes.security import has_access
 import api.models.ingest as IngestProcessModel
 import api.models.object as Object
+import api.schemas as schemas
 from api.schemas import IngestProcess as IngestProcessSchema, ObjectGroup, Sources, IngestProcessTag
 from api.query_parser import get_filter_query_params, QueryParser
 
@@ -250,3 +253,54 @@ async def get_ingest_process_objects(id: int):
             status_code=500,
             detail=f"Failed to get secure url for object: {e}"
         )
+
+@router.post("/{id}/objects", response_model=Object.Get)
+async def create_object(id: int, object: UploadFile, user_has_access: bool = Depends(has_access)):
+    """Create/Register a new object"""
+
+    if not user_has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to create object")
+
+    engine = get_engine()
+    async_session = get_async_session(engine)
+
+    async with async_session() as session:
+
+        ingest_stmt = select(IngestProcessSchema).where(IngestProcessSchema.id == id)
+        ingest_process = await session.scalar(ingest_stmt)
+
+        # Upload the file to s3
+        m = minio.Minio(endpoint=os.environ['S3_HOST'], access_key=os.environ['access_key'],
+                        secret_key=os.environ['secret_key'], secure=True)
+
+        file_length = len(object.file.read())
+        object.file.seek(0)
+
+        object_file_name = f"{ingest_process.id}/{object.filename}"
+
+        m.put_object(
+            bucket_name=os.environ['S3_BUCKET'],
+            object_name=object_file_name,
+            data=object.file,
+            content_type=object.content_type,
+            length=file_length
+        )
+
+        # Upload this file pointer to postgres
+        object = Object.Post(
+            mime_type=object.content_type,
+            key=object_file_name,
+            bucket=os.environ['S3_BUCKET'],
+            host=os.environ['S3_HOST'],
+            scheme=schemas.SchemeEnum.http,
+            object_group_id=ingest_process.object_group_id
+        )
+
+        insert_stmt = insert(schemas.Object)\
+            .values(**object.model_dump())\
+            .returning(schemas.Object)
+        server_object = await session.scalar(insert_stmt)
+
+        response = Object.Get(**server_object.__dict__)
+        await session.commit()
+        return response

api/routes/object.py

@@ -1,7 +1,11 @@
 import datetime
+from typing import Union
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, File, UploadFile
+from starlette.datastructures import UploadFile as StarletteUploadFile
 from sqlalchemy import insert, select, update, and_
+import minio
+import os
 
 from api.database import (
     get_async_session,
@@ -23,7 +27,6 @@
     },
 )
 
-
 @router.get("", response_model=list[Object.Get])
 async def get_objects(page: int = 0, page_size: int = 50, filter_query_params=Depends(get_filter_query_params)):
     """Get all objects"""
@@ -76,12 +79,35 @@ async def get_object(id: int):
 
 
 @router.post("", response_model=Object.Get)
-async def create_object(object: Object.Post, user_has_access: bool = Depends(has_access)):
+async def create_object(object: Union[Object.Post, UploadFile], user_has_access: bool = Depends(has_access)):
     """Create/Register a new object"""
 
     if not user_has_access:
         raise HTTPException(status_code=403, detail="User does not have access to create object")
 
+    if isinstance(object, StarletteUploadFile):
+        m = minio.Minio(endpoint=os.environ['S3_HOST'], access_key=os.environ['access_key'],
+                        secret_key=os.environ['secret_key'], secure=True)
+
+        file_length = len(object.file.read())
+        object.file.seek(0)
+
+        m.put_object(
+            bucket_name=os.environ['S3_BUCKET'],
+            object_name=object.filename,
+            data=object.file,
+            content_type=object.content_type,
+            length=file_length
+        )
+
+        object = Object.Post(
+            mime_type=object.content_type,
+            key=object.filename,
+            bucket=os.environ['S3_BUCKET'],
+            host=os.environ['S3_HOST'],
+            scheme=schemas.SchemeEnum.http
+        )
+
     engine = get_engine()
     async_session = get_async_session(engine)
 

api/tests/data/test.txt

@@ -0,0 +1 @@
+Hello World!

api/tests/main.py

@@ -118,101 +118,3 @@ async def test_patch_sources_sub_table_set_columns_equal(self, engine: AsyncEngi
         assert all([x["descrip"] == test_value for x in result.to_dict()])
 
 
-class TestObjectCRUD:
-
-    def test_object_post(self, api_client):
-        """Test posting an object to the database"""
-
-        key = f"test-{random.randint(0, 10000000)}"
-
-        object_data = {
-            "scheme": "http",
-            "host": "test.com",
-            "bucket": "test",
-            "key": key,
-            "source": {
-                "test_key": "test_value"
-            },
-            "mime_type": "application/json",
-            "sha256_hash": hashlib.sha256(open(__file__, "rb").read()).hexdigest()
-        }
-
-        response = api_client.post(
-            "/object",
-            json=object_data,
-        )
-
-        assert response.status_code == 200
-
-    def test_get_objects(self, api_client):
-        response = api_client.get("/object")
-        assert response.status_code == 200
-
-        data = response.json()
-
-        assert len(data) > 0
-
-    def test_get_object(self, api_client):
-        response = api_client.get("/object")
-        assert response.status_code == 200
-
-        data = response.json()
-
-        assert len(data) > 0
-
-        response = api_client.get(f"/object/{data[0]['id']}")
-
-        assert response.status_code == 200
-
-        single_data = response.json()
-
-        assert single_data == data[0]
-
-    def test_patch_object(self, api_client):
-        # Get a object
-        response = api_client.get("/object")
-        assert response.status_code == 200
-        object_data = response.json()
-        assert len(object_data) > 0
-
-        # Patch Object
-        response = api_client.patch(
-            f"/object/{object_data[0]['id']}",
-            json={
-                "source": {
-                    "comments": "test"
-                }
-            }
-        )
-        assert response.status_code == 200
-        single_data = response.json()
-
-        assert single_data['source']['comments'] == "test"
-
-    def test_delete_object(self, api_client):
-        key = f"test-{random.randint(0, 10000000)}"
-
-        object_data = {
-            "scheme": "http",
-            "host": "test.com",
-            "bucket": "test",
-            "key": key,
-            "source": {
-                "test_key": "test_value"
-            },
-            "mime_type": "application/json",
-            "sha256_hash": hashlib.sha256(open(__file__, "rb").read()).hexdigest()
-        }
-
-        response = api_client.post("/object", json=object_data)
-        assert response.status_code == 200
-
-        data = response.json()
-
-        assert len(data) > 0
-
-        response = api_client.delete(f"/object/{data['id']}")
-        assert response.status_code == 200
-
-        response = api_client.get(f"/object/{data['id']}")
-        assert response.status_code == 404

api/tests/object.py

@@ -0,0 +1,124 @@
+import random
+import hashlib
+import os
+
+from .main import api_client
+
+class TestObjectCRUD:
+
+    def test_object_post(self, api_client):
+        """Test posting an object to the database"""
+
+        key = f"test-{random.randint(0, 10000000)}"
+
+        object_data = {
+            "scheme": "http",
+            "host": "test.com",
+            "bucket": "test",
+            "key": key,
+            "source": {
+                "test_key": "test_value"
+            },
+            "mime_type": "application/json",
+            "sha256_hash": hashlib.sha256(open(__file__, "rb").read()).hexdigest()
+        }
+
+        response = api_client.post(
+            "/object",
+            json=object_data,
+        )
+
+        assert response.status_code == 200
+
+    def test_object_file_post(self, api_client):
+        """Test posting an object to the database"""
+
+        response = api_client.post(
+            "/object",
+            files={"object": open("./tests/data/test.txt", "rb")}
+        )
+
+        assert response.status_code == 200
+
+    def test_object_post_to_ingest_process(self, api_client):
+        """Test posting an object to the database and associating it with an ingest process"""
+
+        response = api_client.post(
+            "/ingest-process/1/objects",
+            files={"object": open("./tests/data/test.txt", "rb")}
+        )
+
+        assert response.status_code == 200
+
+    def test_get_objects(self, api_client):
+        response = api_client.get("/object")
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert len(data) > 0
+
+    def test_get_object(self, api_client):
+        response = api_client.get("/object")
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert len(data) > 0
+
+        response = api_client.get(f"/object/{data[0]['id']}")
+
+        assert response.status_code == 200
+
+        single_data = response.json()
+
+        assert single_data == data[0]
+
+    def test_patch_object(self, api_client):
+        # Get a object
+        response = api_client.get("/object")
+        assert response.status_code == 200
+        object_data = response.json()
+        assert len(object_data) > 0
+
+        # Patch Object
+        response = api_client.patch(
+            f"/object/{object_data[0]['id']}",
+            json={
+                "source": {
+                    "comments": "test"
+                }
+            }
+        )
+        assert response.status_code == 200
+        single_data = response.json()
+
+        assert single_data['source']['comments'] == "test"
+
+    def test_delete_object(self, api_client):
+        key = f"test-{random.randint(0, 10000000)}"
+
+        object_data = {
+            "scheme": "http",
+            "host": "test.com",
+            "bucket": "test",
+            "key": key,
+            "source": {
+                "test_key": "test_value"
+            },
+            "mime_type": "application/json",
+            "sha256_hash": hashlib.sha256(open(__file__, "rb").read()).hexdigest()
+        }
+
+        response = api_client.post("/object", json=object_data)
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert len(data) > 0
+
+        response = api_client.delete(f"/object/{data['id']}")
+        assert response.status_code == 200
+
+        response = api_client.get(f"/object/{data['id']}")
+        assert response.status_code == 404