Merge branch 'main' into feat/mock-api-assistants

.github/workflows/backend-tests.yml

@@ -9,7 +9,9 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-
+    strategy:
+      matrix:
+        project: [quivr-api, quivr-worker]
     steps:
       - name: 👀 Checkout code
         uses: actions/checkout@v2
@@ -65,4 +67,4 @@ jobs:
           supabase start
           rye run python -c "from unstructured.nlp.tokenize import download_nltk_packages; download_nltk_packages()"
           rye run python -c "import nltk;nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger_eng')"
-          rye test -p quivr-api -p quivr-worker
+          rye test -p ${{ matrix.project }}

.pre-commit-config.yaml

@@ -32,13 +32,6 @@ repos:
       - id: mypy
         name: mypy
         additional_dependencies: ["types-aiofiles"]
-  - repo: https://github.com/python-poetry/poetry
-    rev: "1.8.0"
-    hooks:
-      - id: poetry-check
-        args: ["-C", "./backend/core"]
-      - id: poetry-lock
-        args: ["-C", "./backend/core"]
 ci:
   autofix_commit_msg: |
     [pre-commit.ci] auto fixes from pre-commit.com hooks

.release-please-manifest.json

@@ -1,4 +1,4 @@
 {
-    "backend/core": "0.0.14",
-    ".": "0.0.310"
+    "backend/core": "0.0.16",
+    ".": "0.0.315"
 }

CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## 0.0.315 (2024-09-17)
+
+## What's Changed
+* chore(main): release core 0.0.15 by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3203
+* fix: knowledge user_id fix by @AmineDiro in https://github.com/QuivrHQ/quivr/pull/3216
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.314...v0.0.315
+
+## 0.0.314 (2024-09-16)
+
+## What's Changed
+* feat:  CRUD KMS (no syncs)  by @AmineDiro in https://github.com/QuivrHQ/quivr/pull/3162
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.313...v0.0.314
+
+## 0.0.313 (2024-09-13)
+
+## What's Changed
+* feat: save and load brain by @AmineDiro in https://github.com/QuivrHQ/quivr/pull/3202
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.312...v0.0.313
+
+## 0.0.312 (2024-09-13)
+
+## What's Changed
+* fix: Update LLMEndpoint to include max_tokens parameter by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3201
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.311...v0.0.312
+
+## 0.0.311 (2024-09-12)
+
+## What's Changed
+* chore(embeddings): added tests for embeddings by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3183
+* feat(uptime): check if connection to db works by @StanGirard in https://github.com/QuivrHQ/quivr/pull/3199
+
+
+**Full Changelog**: https://github.com/QuivrHQ/quivr/compare/v0.0.310...v0.0.311
+
 ## 0.0.310 (2024-09-10)
 
 ## What's Changed

backend/api/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "markdownify>=0.13.1",
     "langchain-openai>=0.1.21",
     "resend>=2.4.0",
-    "langchain>=0.2.14",
+    "langchain>=0.2.14,<0.3.0",
     "litellm>=1.43.15",
     "openai>=1.40.8",
     "tiktoken>=0.7.0",

backend/api/quivr_api/middlewares/auth/auth_bearer.py

@@ -63,5 +63,8 @@ def get_test_user(self) -> UserIdentity:
         )  # replace with test user information
 
 
-def get_current_user(user: UserIdentity = Depends(AuthBearer())) -> UserIdentity:
+auth_bearer = AuthBearer()
+
+
+def get_current_user(user: UserIdentity = Depends(auth_bearer)) -> UserIdentity:
     return user

backend/api/quivr_api/modules/brain/entity/brain_entity.py

@@ -69,6 +69,7 @@ class Brain(AsyncAttrs, SQLModel, table=True):
         back_populates="brains", link_model=KnowledgeBrain
     )
 
+
     # TODO : add
     # "meaning" "public"."vector",
     # "tags" "public"."tags"[]

backend/api/quivr_api/modules/brain/service/brain_vector_service.py

@@ -2,7 +2,7 @@
 
 from quivr_api.logger import get_logger
 from quivr_api.modules.brain.repository.brains_vectors import BrainsVectors
-from quivr_api.modules.knowledge.repository.storage import Storage
+from quivr_api.modules.knowledge.repository.storage import SupabaseS3Storage
 
 logger = get_logger(__name__)
 
@@ -11,7 +11,7 @@ class BrainVectorService:
     def __init__(self, brain_id: UUID):
         self.repository = BrainsVectors()
         self.brain_id = brain_id
-        self.storage = Storage()
+        self.storage = SupabaseS3Storage()
 
     def create_brain_vector(self, vector_id: str, file_sha1: str):
         return self.repository.create_brain_vector(self.brain_id, vector_id, file_sha1)  # type: ignore
@@ -26,10 +26,10 @@ def update_brain_with_file(self, file_sha1: str):
         for vector_id in vector_ids:
             self.create_brain_vector(vector_id, file_sha1)
 
-    def delete_file_from_brain(self, file_name: str, only_vectors: bool = False):
+    async def delete_file_from_brain(self, file_name: str, only_vectors: bool = False):
         file_name_with_brain_id = f"{self.brain_id}/{file_name}"
         if not only_vectors:
-            self.storage.remove_file(file_name_with_brain_id)
+            await self.storage.remove_file(file_name_with_brain_id)
         return self.repository.delete_file_from_brain(self.brain_id, file_name)  # type: ignore
 
     def delete_file_url_from_brain(self, file_name: str):

backend/api/quivr_api/modules/conftest.py

@@ -24,9 +24,6 @@
     "postgresql+asyncpg://" + pg_database_base_url,
     echo=True if os.getenv("ORM_DEBUG") else False,
     future=True,
-    pool_pre_ping=True,
-    pool_size=10,
-    pool_recycle=0.1,
 )
 
 

backend/api/quivr_api/modules/knowledge/controller/knowledge_routes.py

@@ -1,8 +1,8 @@
 from http import HTTPStatus
-from typing import Annotated
+from typing import Annotated, List, Optional
 from uuid import UUID
 
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile, status
 
 from quivr_api.logger import get_logger
 from quivr_api.middlewares.auth import AuthBearer, get_current_user
@@ -12,6 +12,14 @@
     validate_brain_authorization,
 )
 from quivr_api.modules.dependencies import get_service
+from quivr_api.modules.knowledge.dto.inputs import AddKnowledge
+from quivr_api.modules.knowledge.entity.knowledge import Knowledge, KnowledgeUpdate
+from quivr_api.modules.knowledge.service.knowledge_exceptions import (
+    KnowledgeDeleteError,
+    KnowledgeForbiddenAccess,
+    KnowledgeNotFoundException,
+    UploadError,
+)
 from quivr_api.modules.knowledge.service.knowledge_service import KnowledgeService
 from quivr_api.modules.upload.service.generate_file_signed_url import (
     generate_file_signed_url,
@@ -21,9 +29,8 @@
 knowledge_router = APIRouter()
 logger = get_logger(__name__)
 
-KnowledgeServiceDep = Annotated[
-    KnowledgeService, Depends(get_service(KnowledgeService))
-]
+get_km_service = get_service(KnowledgeService)
+KnowledgeServiceDep = Annotated[KnowledgeService, Depends(get_km_service)]
 
 
 @knowledge_router.get(
@@ -53,7 +60,7 @@ async def list_knowledge_in_brain_endpoint(
     ],
     tags=["Knowledge"],
 )
-async def delete_endpoint(
+async def delete_knowledge_brain(
     knowledge_id: UUID,
     knowledge_service: KnowledgeServiceDep,
     current_user: UserIdentity = Depends(get_current_user),
@@ -65,7 +72,7 @@ async def delete_endpoint(
 
     knowledge = await knowledge_service.get_knowledge(knowledge_id)
     file_name = knowledge.file_name if knowledge.file_name else knowledge.url
-    await knowledge_service.remove_knowledge(brain_id, knowledge_id)
+    await knowledge_service.remove_knowledge_brain(brain_id, knowledge_id)
 
     return {
         "message": f"{file_name} of brain {brain_id} has been deleted by user {current_user.email}."
@@ -88,13 +95,13 @@ async def generate_signed_url_endpoint(
 
     knowledge = await knowledge_service.get_knowledge(knowledge_id)
 
-    if len(knowledge.brain_ids) == 0:
+    if len(knowledge.brains) == 0:
         raise HTTPException(
             status_code=HTTPStatus.NOT_FOUND,
             detail="knowledge not associated with brains yet.",
         )
 
-    brain_id = knowledge.brain_ids[0]
+    brain_id = knowledge.brains[0]["brain_id"]
 
     validate_brain_authorization(brain_id=brain_id, user_id=current_user.id)
 
@@ -108,3 +115,153 @@ async def generate_signed_url_endpoint(
     file_signed_url = generate_file_signed_url(file_path_in_storage)
 
     return file_signed_url
+
+
+@knowledge_router.post(
+    "/knowledge/",
+    tags=["Knowledge"],
+    response_model=Knowledge,
+)
+async def create_knowledge(
+    knowledge_data: str = File(...),
+    file: Optional[UploadFile] = None,
+    knowledge_service: KnowledgeService = Depends(get_km_service),
+    current_user: UserIdentity = Depends(get_current_user),
+):
+    knowledge = AddKnowledge.model_validate_json(knowledge_data)
+    if not knowledge.file_name and not knowledge.url:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Either file_name or url must be provided",
+        )
+    try:
+        km = await knowledge_service.create_knowledge(
+            knowledge_to_add=knowledge, upload_file=file, user_id=current_user.id
+        )
+        km_dto = await km.to_dto()
+        return km_dto
+    except ValueError:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail="Unprocessable knowledge ",
+        )
+    except FileExistsError:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT, detail="Existing knowledge"
+        )
+    except UploadError:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="Error occured uploading knowledge",
+        )
+    except Exception:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+@knowledge_router.get(
+    "/knowledge/children",
+    response_model=List[Knowledge] | None,
+    tags=["Knowledge"],
+)
+async def list_knowledge(
+    parent_id: UUID | None = None,
+    knowledge_service: KnowledgeService = Depends(get_km_service),
+    current_user: UserIdentity = Depends(get_current_user),
+):
+    try:
+        # TODO: Returns one level of children
+        children = await knowledge_service.list_knowledge(parent_id, current_user.id)
+        return [await c.to_dto(get_children=False) for c in children]
+    except KnowledgeNotFoundException as e:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN, detail=f"{e.message}"
+        )
+    except KnowledgeForbiddenAccess as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=f"{e.message}"
+        )
+    except Exception:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+@knowledge_router.get(
+    "/knowledge/{knowledge_id}",
+    response_model=Knowledge,
+    tags=["Knowledge"],
+)
+async def get_knowledge(
+    knowledge_id: UUID,
+    knowledge_service: KnowledgeService = Depends(get_km_service),
+    current_user: UserIdentity = Depends(get_current_user),
+):
+    try:
+        km = await knowledge_service.get_knowledge(knowledge_id)
+        if km.user_id != current_user.id:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="You do not have permission to access this knowledge.",
+            )
+        return await km.to_dto()
+    except KnowledgeNotFoundException as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=f"{e.message}"
+        )
+    except Exception:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+@knowledge_router.patch(
+    "/knowledge/{knowledge_id}",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=Knowledge,
+    tags=["Knowledge"],
+)
+async def update_knowledge(
+    knowledge_id: UUID,
+    payload: KnowledgeUpdate,
+    knowledge_service: KnowledgeService = Depends(get_km_service),
+    current_user: UserIdentity = Depends(get_current_user),
+):
+    try:
+        km = await knowledge_service.get_knowledge(knowledge_id)
+        if km.user_id != current_user.id:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="You do not have permission to access this knowledge.",
+            )
+        km = await knowledge_service.update_knowledge(km, payload)
+        return km
+    except KnowledgeNotFoundException as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=f"{e.message}"
+        )
+    except Exception:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
+
+
+@knowledge_router.delete(
+    "/knowledge/{knowledge_id}",
+    status_code=status.HTTP_202_ACCEPTED,
+    tags=["Knowledge"],
+)
+async def delete_knowledge(
+    knowledge_id: UUID,
+    knowledge_service: KnowledgeService = Depends(get_km_service),
+    current_user: UserIdentity = Depends(get_current_user),
+):
+    try:
+        km = await knowledge_service.get_knowledge(knowledge_id)
+
+        if km.user_id != current_user.id:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail="You do not have permission to remove this knowledge.",
+            )
+        delete_response = await knowledge_service.remove_knowledge(km)
+        return delete_response
+    except KnowledgeNotFoundException as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND, detail=f"{e.message}"
+        )
+    except KnowledgeDeleteError:
+        raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)