Merge branch 'master' into csicamerastability

.github/workflows/build.yml

@@ -262,18 +262,23 @@ jobs:
     needs: [build-package]
 
     if: ${{ github.event_name != 'pull_request' }}
+
     strategy:
       fail-fast: false
       matrix:
         include:
           - os: ubuntu-latest
             artifact-name: LinuxArm64
             image_suffix: RaspberryPi
-            image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.1.3_arm64
+            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_raspi.img.xz
           - os: ubuntu-latest
             artifact-name: LinuxArm64
             image_suffix: limelight2
-            image_url: https://api.github.com/repos/photonvision/photon-pi-gen/releases/tags/v2023.2.2_limelight-arm64
+            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_limelight.img.xz
+          - os: ubuntu-latest
+            artifact-name: LinuxArm64
+            image_suffix: orangepi5
+            image_url: https://github.com/PhotonVision/photon-image-modifier/releases/download/v2024.0.4/photonvision_opi5.img.xz
 
     runs-on: ${{ matrix.os }}
     name: "Build image - ${{ matrix.image_url }}"
@@ -286,6 +291,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: jar-${{ matrix.artifact-name }}
+      # TODO- replace with the arm-runner action and run this inside of the chroot. but this works for now.
       - name: Generate image
         run: |
           chmod +x scripts/generatePiImage.sh

.gitignore

@@ -162,3 +162,5 @@ photonlib-cpp-examples/*/networktables.json.bck
 photonlib-java-examples/*/networktables.json.bck
 *.sqlite
 photon-server/src/main/resources/web/index.html
+
+venv

README.md

@@ -59,6 +59,16 @@ To run them, use the commands listed below. Photonlib must first be published to
 ~/photonvision/photonlib-cpp-examples$ ./gradlew <example-name>:simulateNative
 ```
 
+## Out-of-Source Dependencies
+
+PhotonVision uses the following additonal out-of-source repositories for building code.
+
+- Base system images for Raspberry Pi & Orange Pi: https://github.com/PhotonVision/photon-image-modifier
+- C++ driver for Raspberry Pi CSI cameras: https://github.com/PhotonVision/photon-libcamera-gl-driver
+- JNI code for [mrcal](https://mrcal.secretsauce.net/): https://github.com/PhotonVision/mrcal-java
+- Custom build of OpenCV with GStreamer/Protobuf/other custom flags: https://github.com/PhotonVision/thirdparty-opencv
+- JNI code for aruco-nano: https://github.com/PhotonVision/aruconano-jni
+
 
 ## Acknowledgments
 PhotonVision was forked from [Chameleon Vision](https://github.com/Chameleon-Vision/chameleon-vision/). Thank you to everyone who worked on the original project.

build.gradle

@@ -27,7 +27,7 @@ ext {
     openCVversion = "4.8.0-2"
     joglVersion = "2.4.0-rc-20200307"
     javalinVersion = "5.6.2"
-    photonGlDriverLibVersion = "dev-v2023.1.0-6-g5e6f7fa"
+    photonGlDriverLibVersion = "dev-v2023.1.0-8-g38bbe74"
     frcYear = "2024"
 
     pubVersion = versionString

devTools/calibrationUtils.py

@@ -0,0 +1,134 @@
+import base64
+from dataclasses import dataclass
+import json
+import os
+import cv2
+import numpy as np
+
+
+@dataclass
+class Resolution:
+    width: int
+    height: int
+
+
+@dataclass
+class JsonMatOfDoubles:
+    rows: int
+    cols: int
+    type: int
+    data: list[float]
+
+
+@dataclass
+class JsonMat:
+    rows: int
+    cols: int
+    type: int
+    data: str  # Base64-encoded PNG data
+
+
+@dataclass
+class Point2:
+    x: float
+    y: float
+
+
+@dataclass
+class Translation3d:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Quaternion:
+    X: float
+    Y: float
+    Z: float
+    W: float
+
+
+@dataclass
+class Rotation3d:
+    quaternion: Quaternion
+
+
+@dataclass
+class Pose3d:
+    translation: Translation3d
+    rotation: Rotation3d
+
+
+@dataclass
+class Point3:
+    x: float
+    y: float
+    z: float
+
+
+@dataclass
+class Observation:
+    # Expected feature 3d location in the camera frame
+    locationInObjectSpace: list[Point3]
+    # Observed location in pixel space
+    locationInImageSpace: list[Point2]
+    # (measured location in pixels) - (expected from FK)
+    reprojectionErrors: list[Point2]
+    # Solver optimized board poses
+    optimisedCameraToObject: Pose3d
+    # If we should use this observation when re-calculating camera calibration
+    includeObservationInCalibration: bool
+    snapshotName: str
+    # The actual image the snapshot is from
+    snapshotData: JsonMat
+
+
+@dataclass
+class CameraCalibration:
+    resolution: Resolution
+    cameraIntrinsics: JsonMatOfDoubles
+    distCoeffs: JsonMatOfDoubles
+    observations: list[Observation]
+
+
+def convert_photon_to_mrcal(photon_cal_json_path: str, output_folder: str):
+    """
+    Unpack a Photon calibration JSON (eg, photon_calibration_Microsoft_LifeCam_HD-3000_800x600.json) into
+    the output_folder directory with images and corners.vnl file for use with mrcal.
+    """
+    with open(photon_cal_json_path, "r") as cal_json:
+        # Convert to nested objects instead of nameddicts on json-loads
+        class Generic:
+            @classmethod
+            def from_dict(cls, dict):
+                obj = cls()
+                obj.__dict__.update(dict)
+                return obj
+
+        camera_cal_data: CameraCalibration = json.loads(
+            cal_json.read(), object_hook=Generic.from_dict
+        )
+
+        # Create output_folder if not exists
+        if not os.path.exists(output_folder):
+            os.makedirs(output_folder)
+
+        # Decode each image and save it as a png
+        for obs in camera_cal_data.observations:
+            image = obs.snapshotData.data
+            decoded_data = base64.b64decode(image)
+            np_data = np.frombuffer(decoded_data, np.uint8)
+            img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
+            cv2.imwrite(f"{output_folder}/{obs.snapshotName}", img)
+
+        # And create a VNL file for use with mrcal
+        with open(f"{output_folder}/corners.vnl", "w+") as vnl_file:
+            vnl_file.write("# filename x y level\n")
+
+            for obs in camera_cal_data.observations:
+                for corner in obs.locationInImageSpace:
+                    # Always level zero
+                    vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n")
+
+            vnl_file.flush()

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -1,7 +1,7 @@
 <script setup lang="ts">
 import { computed, ref } from "vue";
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { CalibrationBoardTypes, type Resolution, type VideoFormat } from "@/types/SettingTypes";
+import { CalibrationBoardTypes, type VideoFormat } from "@/types/SettingTypes";
 import JsPDF from "jspdf";
 import { font as PromptRegular } from "@/assets/fonts/PromptRegular";
 import MonoLogo from "@/assets/images/logoMono.png";
@@ -12,37 +12,40 @@ import PvSelect from "@/components/common/pv-select.vue";
 import PvNumberInput from "@/components/common/pv-number-input.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 import { getResolutionString, resolutionsAreEqual } from "@/lib/PhotonUtils";
+import CameraCalibrationInfoCard from "@/components/cameras/CameraCalibrationInfoCard.vue";
 
 const settingsValid = ref(true);
 
-const getCalibrationCoeffs = (resolution: Resolution) => {
-  return useCameraSettingsStore().currentCameraSettings.completeCalibrations.find((cal) =>
-    resolutionsAreEqual(cal.resolution, resolution)
-  );
-};
-const getUniqueVideoResolutions = (): VideoFormat[] => {
+const getUniqueVideoFormatsByResolution = (): VideoFormat[] => {
   const uniqueResolutions: VideoFormat[] = [];
   useCameraSettingsStore().currentCameraSettings.validVideoFormats.forEach((format, index) => {
     if (!uniqueResolutions.some((v) => resolutionsAreEqual(v.resolution, format.resolution))) {
       format.index = index;
 
-      const calib = getCalibrationCoeffs(format.resolution);
+      const calib = useCameraSettingsStore().getCalibrationCoeffs(format.resolution);
       if (calib !== undefined) {
-        format.standardDeviation = calib.standardDeviation;
-        format.mean =
-          calib.perViewErrors === null
-            ? Number.NaN
-            : calib.perViewErrors.reduce((a, b) => a + b) / calib.perViewErrors.length;
-        format.horizontalFOV = 2 * Math.atan2(format.resolution.width / 2, calib.intrinsics[0]) * (180 / Math.PI);
-        format.verticalFOV = 2 * Math.atan2(format.resolution.height / 2, calib.intrinsics[4]) * (180 / Math.PI);
+        // Is this the right formula for RMS error? who knows! not me!
+        const perViewSumSquareReprojectionError = calib.observations.flatMap((it) =>
+          it.reprojectionErrors.flatMap((it2) => [it2.x, it2.y])
+        );
+        // For each error, square it, sum the squares, and divide by total points N
+        format.mean = Math.sqrt(
+          perViewSumSquareReprojectionError.map((it) => Math.pow(it, 2)).reduce((a, b) => a + b, 0) /
+            perViewSumSquareReprojectionError.length
+        );
+
+        format.horizontalFOV =
+          2 * Math.atan2(format.resolution.width / 2, calib.cameraIntrinsics.data[0]) * (180 / Math.PI);
+        format.verticalFOV =
+          2 * Math.atan2(format.resolution.height / 2, calib.cameraIntrinsics.data[4]) * (180 / Math.PI);
         format.diagonalFOV =
           2 *
           Math.atan2(
             Math.sqrt(
               format.resolution.width ** 2 +
-                (format.resolution.height / (calib.intrinsics[4] / calib.intrinsics[0])) ** 2
+                (format.resolution.height / (calib.cameraIntrinsics.data[4] / calib.cameraIntrinsics.data[0])) ** 2
             ) / 2,
-            calib.intrinsics[0]
+            calib.cameraIntrinsics.data[0]
           ) *
           (180 / Math.PI);
       }
@@ -55,7 +58,7 @@ const getUniqueVideoResolutions = (): VideoFormat[] => {
   return uniqueResolutions;
 };
 const getUniqueVideoResolutionStrings = (): { name: string; value: number }[] =>
-  getUniqueVideoResolutions().map<{ name: string; value: number }>((f) => ({
+  getUniqueVideoFormatsByResolution().map<{ name: string; value: number }>((f) => ({
     name: `${getResolutionString(f.resolution)}`,
     // Index won't ever be undefined
     value: f.index || 0
@@ -150,7 +153,7 @@ const importCalibrationFromCalibDB = ref();
 const openCalibUploadPrompt = () => {
   importCalibrationFromCalibDB.value.click();
 };
-const readImportedCalibration = () => {
+const readImportedCalibrationFromCalibDB = () => {
   const files = importCalibrationFromCalibDB.value.files;
   if (files.length === 0) return;
 
@@ -214,6 +217,13 @@ const endCalibration = () => {
       isCalibrating.value = false;
     });
 };
+
+let showCalDialog = ref(false);
+let selectedVideoFormat = ref<VideoFormat | undefined>(undefined);
+const setSelectedVideoFormat = (format: VideoFormat) => {
+  selectedVideoFormat.value = format;
+  showCalDialog.value = true;
+};
 </script>
 
 <template>
@@ -234,7 +244,12 @@ const endCalibration = () => {
               </tr>
             </thead>
             <tbody>
-              <tr v-for="(value, index) in getUniqueVideoResolutions()" :key="index">
+              <tr
+                v-for="(value, index) in getUniqueVideoFormatsByResolution()"
+                :key="index"
+                title="Click to get calibration specific information"
+                @click="setSelectedVideoFormat(value)"
+              >
                 <td>{{ getResolutionString(value.resolution) }}</td>
                 <td>
                   {{ value.mean !== undefined ? (isNaN(value.mean) ? "NaN" : value.mean.toFixed(2) + "px") : "-" }}
@@ -429,7 +444,7 @@ const endCalibration = () => {
               type="file"
               accept=".json"
               style="display: none"
-              @change="readImportedCalibration"
+              @change="readImportedCalibrationFromCalibDB"
             />
           </v-col>
         </v-row>
@@ -478,6 +493,9 @@ const endCalibration = () => {
         </v-card-actions>
       </v-card>
     </v-dialog>
+    <v-dialog v-model="showCalDialog" width="80em">
+      <CameraCalibrationInfoCard v-if="selectedVideoFormat" :video-format="selectedVideoFormat" />
+    </v-dialog>
   </div>
 </template>
 
@@ -494,6 +512,7 @@ const endCalibration = () => {
 
   tbody :hover td {
     background-color: #005281 !important;
+    cursor: pointer;
   }
 
   ::-webkit-scrollbar {