Use mrcal for camera-calibration

README.md

@@ -82,3 +82,11 @@ Our meeting notes can be found in the wiki section of this repository.
 
 * [2020 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2020-Meeting-Notes)
 * [2021 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2021-Meeting-Notes)
+
+## Additional packages
+
+For now, using mrcal requires installing these additional packages on Linux systems:
+
+```
+sudo apt install libcholmod3 liblapack3 libsuitesparseconfig5
+```

build.gradle

@@ -29,18 +29,21 @@ ext {
     javalinVersion = "5.6.2"
     photonGlDriverLibVersion = "dev-v2023.1.0-8-g38bbe74"
     frcYear = "2024"
+    mrcalVersion = "dev-v2024.0.0-7-gc976aaa";
 
     pubVersion = versionString
     isDev = pubVersion.startsWith("dev")
 
     // A list, for legacy reasons, with only the current platform contained
-    String nativeName = wpilibTools.platformMapper.currentPlatform.platformName;
-    if (nativeName == "linuxx64") nativeName = "linuxx86-64";
-    if (nativeName == "winx64") nativeName = "windowsx86-64";
-    if (nativeName == "macx64") nativeName = "osxx86-64";
-    if (nativeName == "macarm64") nativeName = "osxarm64";
+    wpilibNativeName = wpilibTools.platformMapper.currentPlatform.platformName;
+    def nativeName = wpilibNativeName
+    if (wpilibNativeName == "linuxx64") nativeName = "linuxx86-64";
+    if (wpilibNativeName == "winx64") nativeName = "windowsx86-64";
+    if (wpilibNativeName == "macx64") nativeName = "osxx86-64";
+    if (wpilibNativeName == "macarm64") nativeName = "osxarm64";
     jniPlatform = nativeName
-    println("Building for platform: " + jniPlatform)
+
+    println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
     println("Using Wpilib: " + wpilibVersion)
     println("Using OpenCV: " + openCVversion)
 }

devTools/calibrationUtils.py

@@ -1,3 +1,4 @@
+import argparse
 import base64
 from dataclasses import dataclass
 import json
@@ -132,3 +133,23 @@ def from_dict(cls, dict):
                     vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n")
 
             vnl_file.flush()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert Photon calibration JSON for use with mrcal"
+    )
+    parser.add_argument(
+        "input", type=str, help="Path to Photon calibration JSON file"
+    )
+    parser.add_argument(
+        "output_folder", type=str, help="Output folder for mrcal VNL file + images"
+    )
+
+    args = parser.parse_args()
+
+    convert_photon_to_mrcal(args.input, args.output_folder)
+
+
+if __name__ == "__main__":
+    main()

photon-client/src/components/cameras/CameraCalibrationCard.vue

@@ -13,6 +13,7 @@ import PvNumberInput from "@/components/common/pv-number-input.vue";
 import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
 import { getResolutionString, resolutionsAreEqual } from "@/lib/PhotonUtils";
 import CameraCalibrationInfoCard from "@/components/cameras/CameraCalibrationInfoCard.vue";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 const settingsValid = ref(true);
 
@@ -74,6 +75,15 @@ const squareSizeIn = ref(1);
 const patternWidth = ref(8);
 const patternHeight = ref(8);
 const boardType = ref<CalibrationBoardTypes>(CalibrationBoardTypes.Chessboard);
+const useMrCalRef = ref(true);
+const useMrCal = computed<boolean>({
+  get() {
+    return useMrCalRef.value && useSettingsStore().general.mrCalWorking;
+  },
+  set(value) {
+    useMrCalRef.value = value && useSettingsStore().general.mrCalWorking;
+  }
+});
 
 const downloadCalibBoard = () => {
   const doc = new JsPDF({ unit: "in", format: "letter" });
@@ -188,7 +198,8 @@ const startCalibration = () => {
     squareSizeIn: squareSizeIn.value,
     patternHeight: patternHeight.value,
     patternWidth: patternWidth.value,
-    boardType: boardType.value
+    boardType: boardType.value,
+    useMrCal: useMrCal.value
   });
   // The Start PnP method already handles updating the backend so only a store update is required
   useCameraSettingsStore().currentCameraSettings.currentPipelineIndex = WebsocketPipelineType.Calib3d;
@@ -314,6 +325,23 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
               :rules="[(v) => v >= 4 || 'Height must be at least 4']"
               :label-cols="5"
             />
+            <pv-switch
+              v-model="useMrCal"
+              label="Try using MrCal over OpenCV"
+              :disabled="!useSettingsStore().general.mrCalWorking || isCalibrating"
+              tooltip="If enabled, Photon will (try to) use MrCal instead of OpenCV for camera calibration."
+              :label-cols="5"
+            />
+            <v-banner
+              v-show="!useSettingsStore().general.mrCalWorking"
+              rounded
+              color="red"
+              text-color="white"
+              style="margin: 10px 0"
+              icon="mdi-alert-circle-outline"
+            >
+              MrCal JNI could not be loaded! Consult journalctl logs for additional details.
+            </v-banner>
           </v-form>
           <v-row justify="center">
             <v-chip

photon-client/src/stores/settings/CameraSettingsStore.ts

@@ -317,6 +317,7 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
         patternWidth: number;
         patternHeight: number;
         boardType: CalibrationBoardTypes;
+        useMrCal: boolean;
       },
       cameraIndex: number = useStateStore().currentCameraIndex
     ) {

photon-client/src/stores/settings/GeneralSettingsStore.ts

@@ -26,7 +26,8 @@ export const useSettingsStore = defineStore("settings", {
       version: undefined,
       gpuAcceleration: undefined,
       hardwareModel: undefined,
-      hardwarePlatform: undefined
+      hardwarePlatform: undefined,
+      mrCalWorking: true
     },
     network: {
       ntServerAddress: "",
@@ -97,7 +98,8 @@ export const useSettingsStore = defineStore("settings", {
         version: data.general.version || undefined,
         hardwareModel: data.general.hardwareModel || undefined,
         hardwarePlatform: data.general.hardwarePlatform || undefined,
-        gpuAcceleration: data.general.gpuAcceleration || undefined
+        gpuAcceleration: data.general.gpuAcceleration || undefined,
+        mrCalWorking: data.general.mrCalWorking
       };
       this.lighting = data.lighting;
       this.network = data.networkSettings;

photon-client/src/types/SettingTypes.ts

@@ -6,6 +6,7 @@ export interface GeneralSettings {
   gpuAcceleration?: string;
   hardwareModel?: string;
   hardwarePlatform?: string;
+  mrCalWorking: boolean;
 }
 
 export interface MetricData {

photon-core/build.gradle

@@ -16,6 +16,11 @@ dependencies {
 
     implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"
     implementation "org.photonvision:photon-libcamera-gl-driver-java:$photonGlDriverLibVersion"
+
+    implementation "org.photonvision:photon-mrcal-java:$mrcalVersion"
+    implementation "org.photonvision:photon-mrcal-jni:$mrcalVersion:$wpilibNativeName"
+
+    testImplementation group: 'org.junit-pioneer' , name: 'junit-pioneer', version: '2.2.0'
 }
 
 task writeCurrentVersion {

photon-core/src/main/java/org/photonvision/common/configuration/PhotonConfiguration.java

@@ -27,6 +27,7 @@
 import org.photonvision.common.hardware.Platform;
 import org.photonvision.common.networking.NetworkUtils;
 import org.photonvision.common.util.SerializationUtils;
+import org.photonvision.mrcal.MrCal;
 import org.photonvision.raspi.LibCameraJNILoader;
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 import org.photonvision.vision.processes.VisionModule;
@@ -140,6 +141,7 @@ public Map<String, Object> toHashMap() {
                 LibCameraJNILoader.isSupported()
                         ? "Zerocopy Libcamera Working"
                         : ""); // TODO add support for other types of GPU accel
+        generalSubmap.put("mrCalWorking", MrCal.isWorking());
         generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
         generalSubmap.put("hardwarePlatform", Platform.getPlatformName());
         settingsSubmap.put("general", generalSubmap);

photon-core/src/main/java/org/photonvision/common/hardware/Platform.java

@@ -17,6 +17,7 @@
 
 package org.photonvision.common.hardware;
 
+import com.jogamp.common.os.Platform.OSType;
 import edu.wpi.first.util.RuntimeDetector;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -27,23 +28,32 @@
 @SuppressWarnings("unused")
 public enum Platform {
     // WPILib Supported (JNI)
-    WINDOWS_64("Windows x64", false, OSType.WINDOWS, true),
-    LINUX_32("Linux x86", false, OSType.LINUX, true),
-    LINUX_64("Linux x64", false, OSType.LINUX, true),
+    WINDOWS_64("Windows x64", "winx64", false, OSType.WINDOWS, true),
+    LINUX_32("Linux x86", "linuxx64", false, OSType.LINUX, true),
+    LINUX_64("Linux x64", "linuxx64", false, OSType.LINUX, true),
     LINUX_RASPBIAN32(
-            "Linux Raspbian 32-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 32-bit image
+            "Linux Raspbian 32-bit",
+            "linuxarm32",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 32-bit image
     LINUX_RASPBIAN64(
-            "Linux Raspbian 64-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 64-bit image
-    LINUX_AARCH64("Linux AARCH64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
+            "Linux Raspbian 64-bit",
+            "linuxarm64",
+            true,
+            OSType.LINUX,
+            true), // Raspberry Pi 3/4 with a 64-bit image
+    LINUX_AARCH64(
+            "Linux AARCH64", "linuxarm64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
 
     // PhotonVision Supported (Manual build/install)
-    LINUX_ARM32("Linux ARM32", false, OSType.LINUX, true), // ODROID XU4, C1+
-    LINUX_ARM64("Linux ARM64", false, OSType.LINUX, true), // ODROID C2, N2
+    LINUX_ARM32("Linux ARM32", "linuxarm32", false, OSType.LINUX, true), // ODROID XU4, C1+
+    LINUX_ARM64("Linux ARM64", "linuxarm64", false, OSType.LINUX, true), // ODROID C2, N2
 
     // Completely unsupported
-    WINDOWS_32("Windows x86", false, OSType.WINDOWS, false),
-    MACOS("Mac OS", false, OSType.MACOS, false),
-    UNKNOWN("Unsupported Platform", false, OSType.UNKNOWN, false);
+    WINDOWS_32("Windows x86", "windowsx64", false, OSType.WINDOWS, false),
+    MACOS("Mac OS", "osxuniversal", false, OSType.MACOS, false),
+    UNKNOWN("Unsupported Platform", "", false, OSType.UNKNOWN, false);
 
     private enum OSType {
         WINDOWS,
@@ -54,6 +64,7 @@ private enum OSType {
 
     private static final ShellExec shell = new ShellExec(true, false);
     public final String description;
+    public final String nativeLibraryFolderName;
     public final boolean isPi;
     public final OSType osType;
     public final boolean isSupported;
@@ -62,11 +73,17 @@ private enum OSType {
     private static final Platform currentPlatform = getCurrentPlatform();
     private static final boolean isRoot = checkForRoot();
 
-    Platform(String description, boolean isPi, OSType osType, boolean isSupported) {
+    Platform(
+            String description,
+            String nativeLibFolderName,
+            boolean isPi,
+            OSType osType,
+            boolean isSupported) {
         this.description = description;
         this.isPi = isPi;
         this.osType = osType;
         this.isSupported = isSupported;
+        this.nativeLibraryFolderName = nativeLibFolderName;
     }
 
     //////////////////////////////////////////////////////
@@ -89,6 +106,10 @@ public static String getPlatformName() {
         }
     }
 
+    public static final String getNativeLibraryFolderName() {
+        return currentPlatform.nativeLibraryFolderName;
+    }
+
     public static boolean isRoot() {
         return isRoot;
     }
@@ -212,4 +233,9 @@ private static boolean fileHasText(String filename, String text) {
             return false;
         }
     }
+
+    public static boolean isWindows() {
+        var p = getCurrentPlatform();
+        return (p == WINDOWS_32 || p == WINDOWS_64);
+    }
 }

photon-core/src/main/java/org/photonvision/common/util/TestUtils.java

@@ -39,7 +39,11 @@
 import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
 
 public class TestUtils {
+    private static boolean has_loaded = false;
+
     public static boolean loadLibraries() {
+        if (has_loaded) return true;
+
         NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
         WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
         WPIMathJNI.Helper.setExtractOnStaticLoad(false);
@@ -61,11 +65,13 @@ public static boolean loadLibraries() {
                     "cscorejni",
                     "apriltagjni");
 
-            return true;
+            has_loaded = true;
         } catch (IOException e) {
             e.printStackTrace();
-            return false;
+            has_loaded = false;
         }
+
+        return has_loaded;
     }
 
     @SuppressWarnings("unused")