Save calibration data and show preliminary GUI (#1078)

* Serialize all calibration data

* Run lint

* typing nit

* fix code

* move these tables around some

* Add cool formatting

* add request to get snapshots by resolution and camera

* re-enable all resolutions

* add wip so i can change computers (SQUASH ME AND KILL ME AHHHH)

* Get everything working but viewing snapshots

* Update RequestHandler.java

* Update CameraCalibrationInfoCard.vue

* Update CameraCalibrationInfoCard.vue

* add observation viewer

* round

* fix illiegal import

* Swap to PNG and serialize insolution

* move import/export buttons TO THE TOP

* Update WebsocketDataTypes.ts

* Add snapshotname to observation

* Refactor to serialize snapshot image itself

* Run lint

* Use new base64 image data in info card

* Update SettingTypes.ts

* Create calibration json -> mrcal converter script

* Update calibrationUtils.py

* Fix calibrate NPEs in teest

* Run lint

* Always run cornersubpix

* Update CameraCalibrationInfoCard.vue

Update CameraCalibrationInfoCard.vue

* Update OpenCVHelp.java

* Update OpenCVHelp.java

* Replace test mode camera JSONs

* Run wpiformat

* Revert intrinsics but keep other data

* Remove misc comments

* Rename JsonMat->JsonImageMat and add calobject_warp

* Update Server.java

* Rename cameraExtrinsics to distCoeffs

* fix typing issues

* use util methods

* Formatting fixes

* fix styling

* move to devTools

* remove unneeded or unused imports

* Remove fixed-right css

If its really that big of a deal, we can add it back later, kind of a drag to fix rn.

* Create util method

* Remove extra legacy calibration things

---------

Co-authored-by: Sriman Achanta <[email protected]>

.gitignore

@@ -162,3 +162,5 @@ photonlib-cpp-examples/*/networktables.json.bck
 photonlib-java-examples/*/networktables.json.bck
 *.sqlite
 photon-server/src/main/resources/web/index.html
+
+venv

devTools/calibrationUtils.py

@@ -0,0 +1,134 @@
+import base64
+from dataclasses import dataclass
+import json
+import os
+import cv2
+import numpy as np
+
+
+@dataclass
+class Resolution:
+    width: int
+    height: int
+
+
+@dataclass
+class JsonMatOfDoubles:
+    rows: int
+    cols: int
+    type: int
+    data: list[float]
+
+
+@dataclass
+class JsonMat:
+    rows: int
+    cols: int
+    type: int
+    data: str  # Base64-encoded PNG data
+
+
+@dataclass
+class Point2:
+    x: float
+    y: float
+
+
+@dataclass
+class Translation3d:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Quaternion:
+    X: float
+    Y: float
+    Z: float
+    W: float
+
+
+@dataclass
+class Rotation3d:
+    quaternion: Quaternion
+
+
+@dataclass
+class Pose3d:
+    translation: Translation3d
+    rotation: Rotation3d
+
+
+@dataclass
+class Point3:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Observation:
+    # Expected feature 3d location in the camera frame
+    locationInObjectSpace: list[Point3]
+    # Observed location in pixel space
+    locationInImageSpace: list[Point2]
+    # (measured location in pixels) - (expected from FK)
+    reprojectionErrors: list[Point2]
+    # Solver optimized board poses
+    optimisedCameraToObject: Pose3d
+    # If we should use this observation when re-calculating camera calibration
+    includeObservationInCalibration: bool
+    snapshotName: str
+    # The actual image the snapshot is from
+    snapshotData: JsonMat
+
+
+@dataclass
+class CameraCalibration:
+    resolution: Resolution
+    cameraIntrinsics: JsonMatOfDoubles
+    distCoeffs: JsonMatOfDoubles
+    observations: list[Observation]
+
+
+def convert_photon_to_mrcal(photon_cal_json_path: str, output_folder: str):
+    """
+    Unpack a Photon calibration JSON (eg, photon_calibration_Microsoft_LifeCam_HD-3000_800x600.json) into
+    the output_folder directory with images and corners.vnl file for use with mrcal.
+    """
+    with open(photon_cal_json_path, "r") as cal_json:
+        # Convert to nested objects instead of nameddicts on json-loads
+        class Generic:
+            @classmethod
+            def from_dict(cls, dict):
+                obj = cls()
+                obj.__dict__.update(dict)
+                return obj
+
+        camera_cal_data: CameraCalibration = json.loads(
+            cal_json.read(), object_hook=Generic.from_dict
+        )
+
+        # Create output_folder if not exists
+        if not os.path.exists(output_folder):
+            os.makedirs(output_folder)
+
+        # Decode each image and save it as a png
+        for obs in camera_cal_data.observations:
+            image = obs.snapshotData.data
+            decoded_data = base64.b64decode(image)
+            np_data = np.frombuffer(decoded_data, np.uint8)
+            img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
+            cv2.imwrite(f"{output_folder}/{obs.snapshotName}", img)
+
+        # And create a VNL file for use with mrcal
+        with open(f"{output_folder}/corners.vnl", "w+") as vnl_file:
+            vnl_file.write("# filename x y level\n")
+
+            for obs in camera_cal_data.observations:
+                for corner in obs.locationInImageSpace:
+                    # Always level zero
+                    vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n")
+
+            vnl_file.flush()

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { computed, ref } from "vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { CalibrationBoardTypes, type Resolution, type VideoFormat } from "@/types/SettingTypes";
+import { CalibrationBoardTypes, type VideoFormat } from "@/types/SettingTypes";
 import JsPDF from "jspdf";
 import { font as PromptRegular } from "@/assets/fonts/PromptRegular";
 import MonoLogo from "@/assets/images/logoMono.png";
@@ -12,37 +12,40 @@ import PvSelect from "@/components/common/pv-select.vue";
 import PvNumberInput from "@/components/common/pv-number-input.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 import { getResolutionString, resolutionsAreEqual } from "@/lib/PhotonUtils";
+import CameraCalibrationInfoCard from "@/components/cameras/CameraCalibrationInfoCard.vue";
 
 const settingsValid = ref(true);
 
-const getCalibrationCoeffs = (resolution: Resolution) => {
-  return useCameraSettingsStore().currentCameraSettings.completeCalibrations.find((cal) =>
-    resolutionsAreEqual(cal.resolution, resolution)
-  );
-};
-const getUniqueVideoResolutions = (): VideoFormat[] => {
+const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
   const uniqueResolutions: VideoFormat[] = [];
   useCameraSettingsStore().currentCameraSettings.validVideoFormats.forEach((format, index) => {
     if (!uniqueResolutions.some((v) => resolutionsAreEqual(v.resolution, format.resolution))) {
       format.index = index;
 
-      const calib = getCalibrationCoeffs(format.resolution);
+      const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
       if (calib !== undefined) {
-        format.standardDeviation = calib.standardDeviation;
-        format.mean =
-          calib.perViewErrors === null
-            ? Number.NaN
-            : calib.perViewErrors.reduce((a, b) => a + b) / calib.perViewErrors.length;
-        format.horizontalFOV = 2 * Math.atan2(format.resolution.width / 2, calib.intrinsics[0]) * (180 / Math.PI);
-        format.verticalFOV = 2 * Math.atan2(format.resolution.height / 2, calib.intrinsics[4]) * (180 / Math.PI);
+        // Is this the right formula for RMS error? who knows! not me!
+        const perViewSumSquareReprojectionError = calib.observations.flatMap((it) =>
+          it.reprojectionErrors.flatMap((it2) => [it2.x, it2.y])
+        );
+        // For each error, square it, sum the squares, and divide by total points N
+        format.mean = Math.sqrt(
+          perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
+            perViewSumSquareReprojectionError.length
+        );
+
+        format.horizontalFOV =
+          2 * Math.atan2(format.resolution.width / 2, calib.cameraIntrinsics.data[0]) * (180 / Math.PI);
+        format.verticalFOV =
+          2 * Math.atan2(format.resolution.height / 2, calib.cameraIntrinsics.data[4]) * (180 / Math.PI);
         format.diagonalFOV =
           2 *
           Math.atan2(
             Math.sqrt(
               format.resolution.width ** 2 +
-                (format.resolution.height / (calib.intrinsics[4] / calib.intrinsics[0])) ** 2
+                (format.resolution.height / (calib.cameraIntrinsics.data[4] / calib.cameraIntrinsics.data[0])) ** 2
             ) / 2,
-            calib.intrinsics[0]
+            calib.cameraIntrinsics.data[0]
           ) *
           (180 / Math.PI);
       }
@@ -55,7 +58,7 @@ const getUniqueVideoResolutions = (): VideoFormat[] => {
   return uniqueResolutions;
 };
 const getUniqueVideoResolutionStrings = (): { name: string; value: number }[] =>
-  getUniqueVideoResolutions().map<{ name: string; value: number }>((f) => ({
+  getUniqueVideoFormatsByResolution().map<{ name: string; value: number }>((f) => ({
     name: `${getResolutionString(f.resolution)}`,
     // Index won't ever be undefined
     value: f.index || 0
@@ -150,7 +153,7 @@ const importCalibrationFromCalibDB = ref();
 const openCalibUploadPrompt = () => {
   importCalibrationFromCalibDB.value.click();
 };
-const readImportedCalibration = () => {
+const readImportedCalibrationFromCalibDB = () => {
   const files = importCalibrationFromCalibDB.value.files;
   if (files.length === 0) return;
 
@@ -214,6 +217,13 @@ const endCalibration = () => {
       isCalibrating.value = false;
     });
 };
+
+let showCalDialog = ref(false);
+let selectedVideoFormat = ref<VideoFormat | undefined>(undefined);
+const setSelectedVideoFormat = (format: VideoFormat) => {
+  selectedVideoFormat.value = format;
+  showCalDialog.value = true;
+};
 </script>
 
 <template>
@@ -234,7 +244,12 @@ const endCalibration = () => {
               </tr>
             </thead>
             <tbody>
-              <tr v-for="(value, index) in getUniqueVideoResolutions()" :key="index">
+              <tr
+                v-for="(value, index) in getUniqueVideoFormatsByResolution()"
+                :key="index"
+                title="Click to get calibration specific information"
+                @click="setSelectedVideoFormat(value)"
+              >
                 <td>{{ getResolutionString(value.resolution) }}</td>
                 <td>
                   {{ value.mean !== undefined ? (isNaN(value.mean) ? "NaN" : value.mean.toFixed(2) + "px") : "-" }}
@@ -429,7 +444,7 @@ const endCalibration = () => {
               type="file"
               accept=".json"
               style="display: none"
-              @change="readImportedCalibration"
+              @change="readImportedCalibrationFromCalibDB"
             />
           </v-col>
         </v-row>
@@ -478,6 +493,9 @@ const endCalibration = () => {
         </v-card-actions>
       </v-card>
     </v-dialog>
+    <v-dialog v-model="showCalDialog" width="80em">
+      <CameraCalibrationInfoCard v-if="selectedVideoFormat" :video-format="selectedVideoFormat" />
+    </v-dialog>
   </div>
 </template>
 
@@ -494,6 +512,7 @@ const endCalibration = () => {
 
   tbody :hover td {
     background-color: #005281 !important;
+    cursor: pointer;
   }
 
   ::-webkit-scrollbar {