perf: bump to better indexing

.gitignore

@@ -54,4 +54,4 @@ docs/site
 # logs
 src/app/logs/
 
-src/submodules
+src/submodules

Dockerfile

@@ -1,7 +1,7 @@
 # ================================================================
 #  Docker image for fastapi-vss
 #  ================================================================
-FROM mbari/aidata:1.12.2
+FROM mbari/aidata:1.17.0
 
 LABEL vendor="MBARI"
 LABEL maintainer="[email protected]"

Dockerfile.cuda

@@ -1,7 +1,7 @@
 # ================================================================
 #  Docker image for fastapi-vss (CUDA enabled)
 #  ================================================================
-FROM mbari/aidata:1.12.2-cuda124
+FROM mbari/aidata:1.17.0-cuda124
 
 LABEL vendor="MBARI"
 LABEL maintainer="[email protected]"

justfile

@@ -12,14 +12,12 @@ list:
 # Setup the environment
 install:
     conda env create -f environment.yml
-    git submodule update --init --recursive
-    python -m pip install -r src/submodules/aidata/requirements.txt
+    git clone --branch v1.16.0 https://github.com/mbari-org/aidata ./src/aidata
+    python -m pip install -r src/aidata/requirements.txt
     python -m pip install https://github.com/redis/redis-py/archive/refs/tags/v5.0.9.zip
 
 # Update the conda environment. Run this command after checking out any code changes
 update:
-    git submodule update --init --recursive
-    python -m pip install -r src/submodules/aidata/requirements.txt
     conda env update --file environment.yml --prune
     python -m pip install https://github.com/redis/redis-py/archive/refs/tags/v5.0.9.zip
 
@@ -35,7 +33,7 @@ run-server: kill-uvicorn
     #!/usr/bin/env bash
     echo "FastAPI server running at http://localhost:8002"
     echo "FastAPI docs running at http://localhost:8002/docs"
-    export PYTHONPATH=$PWD/src:$PWD/src/submodules/aidata
+    export PYTHONPATH=$PWD/src:/Users/dcline/Dropbox/code/aidata
     cd src/app && conda run -n fastapi-vss --no-capture-output uvicorn main:app --port 8002 --reload
 
 run-server-prod: build-docker
@@ -50,6 +48,12 @@ build-docker:
     tag=$(git describe --tags --always)
     docker build -t mbari/fastapi-app:$tag .
 
+# Build the CUDA Docker image
+build-docker-cuda:
+    #!/usr/bin/env bash
+    tag=$(git describe --tags --always)
+    docker build -t mbari/fastapi-app:$tag -f Dockerfile.cuda .
+
 build-docker-no-cache:
     #!/usr/bin/env bash
     tag=$(git describe --tags --always)
@@ -62,3 +66,29 @@ run-docker:
 # Default recipe
 default:
     just run-server
+
+test_all_ids:
+    #!/usr/bin/env bash
+    curl -X 'POST' \
+      'http://localhost:8002/ids/i2map' \
+      -H 'accept: application/json' \
+      -d ''
+
+process_copepods:
+    #!/usr/bin/env bash
+    cd ./tests/images/copepod
+    curl -X 'POST' \
+     'http:/localhost:8002/knn/3/i2map' \
+     -H 'accept: application/json' \
+     -H 'Content-Type: multipart/form-data' \
+     -F '[email protected];type=image/png'
+    curl -X 'POST' \
+     'http:/localhost:8002/knn/3/i2map' \
+     -H 'accept: application/json' \
+     -H 'Content-Type: multipart/form-data' \
+     -F '[email protected];type=image/png'
+    curl -X 'POST' \
+     'http:/localhost:8002/knn/3/i2map' \
+     -H 'accept: application/json' \
+     -H 'Content-Type: multipart/form-data' \
+     -F '[email protected];type=image/png'

src/app/main.py

@@ -40,6 +40,18 @@ async def root():
 async def get_projects():
     return {"projects": list(global_config.keys())}
 
+@app.post("/ids/{project}", status_code=status.HTTP_200_OK)
+async def get_ids(project: str = DEFAULT_PROJECT):
+    # Check if the project name is in the config
+    if project not in global_config.keys():
+        return {"error": f"Invalid project name {project}"}
+
+    v = global_config[project]['v']
+    try:
+        classes, ids = v.get_ids()
+        return {"ids": ids, "classes": classes}
+    except Exception as e:
+        return {"error": f"Error getting ids: {e}"}
 
 @app.post("/knn/{top_n}/{project}", status_code=status.HTTP_200_OK)
 async def knn(files: List[UploadFile] = File(...), top_n: int = 1, project: str = DEFAULT_PROJECT):
@@ -56,5 +68,8 @@ async def knn(files: List[UploadFile] = File(...), top_n: int = 1, project: str
 
     v = global_config[project]['v']
     images = [f.file for f in files]
-    predictions, scores = v.predict(images, top_n)
-    return {"predictions": predictions, "scores": scores}
+    try:
+        predictions, scores, ids = v.predict(images, top_n)
+        return {"predictions": predictions, "scores": scores, "ids": ids}
+    except Exception as e:
+        return {"error": f"Error predicting images: {e}"}

src/app/project/config/config.yml

@@ -0,0 +1,56 @@
+# Config file for 902111-CFE project
+redis:
+  host: "doris.shore.mbari.org"
+  port: 6382
+
+data:
+  raw_data_path: "/mnt/CFElab/"
+  processed_path: "/mnt/ML_SCRATCH/902111-CFE"
+  version: "Baseline"
+  labels: "copepod"
+  download_args: ["--verified"]
+
+sdcat:
+  model: "hustvl/yolos-tiny"
+  clu_det_ini: "sdcat_clu_det.ini"
+  exemplar_ini: "sdcat_exemplar.ini"
+
+vss:
+  project: "cfe"
+  threshold: 0.4
+
+tator:
+  project: "902111-CFE"
+  host: "mantis.shore.mbari.org"
+  bulk_url: "http://mantis.shore.mbari.org:8001/label/id"
+
+  image:
+    attributes:
+      iso_datetime:
+        type: string
+      depth:
+        type: int
+
+  box:
+    attributes:
+      Label:
+        type: string
+      concept:
+        type: string
+      score:
+        type: float
+      cluster:
+        type: string
+      saliency:
+        type: float
+      group:
+        type: string
+      obs_uuid:
+        type: string
+      exemplar:
+        type: bool
+      verified:
+        type: bool
+
+
+ffmpeg_path: "/usr/bin/ffmpeg"