Skip to content

Commit

Permalink
Add mrcal-jni for calibration
Browse files Browse the repository at this point in the history
Update Server.java

Pipe in use mr cal

Run lint

Implement calobject warp

Fix mrcal typos and use-after-free

Run lint

Cleanup calibration test
  • Loading branch information
mcm001 committed Jan 3, 2024
1 parent 7f09f9e commit ba190f9
Show file tree
Hide file tree
Showing 83 changed files with 3,896 additions and 240 deletions.
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,11 @@ Our meeting notes can be found in the wiki section of this repository.

* [2020 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2020-Meeting-Notes)
* [2021 Meeting Notes](https://github.com/PhotonVision/photonvision/wiki/2021-Meeting-Notes)

## Additional packages

For now, using mrcal requires installing these additional packages on Linux systems:

```
sudo apt install libcholmod3 liblapack3 libsuitesparseconfig5
```
15 changes: 9 additions & 6 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,21 @@ ext {
javalinVersion = "5.6.2"
photonGlDriverLibVersion = "dev-v2023.1.0-8-g38bbe74"
frcYear = "2024"
mrcalVersion = "dev-v2024.0.0-7-gc976aaa";

pubVersion = versionString
isDev = pubVersion.startsWith("dev")

// A list, for legacy reasons, with only the current platform contained
String nativeName = wpilibTools.platformMapper.currentPlatform.platformName;
if (nativeName == "linuxx64") nativeName = "linuxx86-64";
if (nativeName == "winx64") nativeName = "windowsx86-64";
if (nativeName == "macx64") nativeName = "osxx86-64";
if (nativeName == "macarm64") nativeName = "osxarm64";
wpilibNativeName = wpilibTools.platformMapper.currentPlatform.platformName;
def nativeName = wpilibNativeName
if (wpilibNativeName == "linuxx64") nativeName = "linuxx86-64";
if (wpilibNativeName == "winx64") nativeName = "windowsx86-64";
if (wpilibNativeName == "macx64") nativeName = "osxx86-64";
if (wpilibNativeName == "macarm64") nativeName = "osxarm64";
jniPlatform = nativeName
println("Building for platform: " + jniPlatform)

println("Building for platform " + jniPlatform + " wpilib: " + wpilibNativeName)
println("Using Wpilib: " + wpilibVersion)
println("Using OpenCV: " + openCVversion)
}
Expand Down
7 changes: 7 additions & 0 deletions devTools/calibrationUtils.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,10 @@ def from_dict(cls, dict):
vnl_file.write(f"{obs.snapshotName} {corner.x} {corner.y} 0\n")

vnl_file.flush()


if __name__ == "__main__":
convert_photon_to_mrcal(
"test-resources/calibration/lifecam480p.json",
"~/Downloads/2024-01-02_lifecam_480",
)
30 changes: 29 additions & 1 deletion photon-client/src/components/cameras/CameraCalibrationCard.vue
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import PvNumberInput from "@/components/common/pv-number-input.vue";
import { WebsocketPipelineType } from "@/types/WebsocketDataTypes";
import { getResolutionString, resolutionsAreEqual } from "@/lib/PhotonUtils";
import CameraCalibrationInfoCard from "@/components/cameras/CameraCalibrationInfoCard.vue";
import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
const settingsValid = ref(true);
Expand Down Expand Up @@ -74,6 +75,15 @@ const squareSizeIn = ref(1);
const patternWidth = ref(8);
const patternHeight = ref(8);
const boardType = ref<CalibrationBoardTypes>(CalibrationBoardTypes.Chessboard);
const useMrCalRef = ref(true);
const useMrCal = computed<boolean>({
get() {
return useMrCalRef.value && useSettingsStore().general.mrCalWorking;
},
set(value) {
useMrCalRef.value = value && useSettingsStore().general.mrCalWorking;
}
});
const downloadCalibBoard = () => {
const doc = new JsPDF({ unit: "in", format: "letter" });
Expand Down Expand Up @@ -188,7 +198,8 @@ const startCalibration = () => {
squareSizeIn: squareSizeIn.value,
patternHeight: patternHeight.value,
patternWidth: patternWidth.value,
boardType: boardType.value
boardType: boardType.value,
useMrCal: useMrCal.value
});
// The Start PnP method already handles updating the backend so only a store update is required
useCameraSettingsStore().currentCameraSettings.currentPipelineIndex = WebsocketPipelineType.Calib3d;
Expand Down Expand Up @@ -314,6 +325,23 @@ const setSelectedVideoFormat = (format: VideoFormat) => {
:rules="[(v) => v >= 4 || 'Height must be at least 4']"
:label-cols="5"
/>
<pv-switch
v-model="useMrCal"
label="Try using MrCal over OpenCV"
:disabled="!useSettingsStore().general.mrCalWorking || isCalibrating"
tooltip="If enabled, Photon will (try to) use MrCal instead of OpenCV for camera calibration."
:label-cols="5"
/>
<v-banner
v-show="!useSettingsStore().general.mrCalWorking"
rounded
color="red"
text-color="white"
style="margin: 10px 0"
icon="mdi-alert-circle-outline"
>
MrCal JNI could not be loaded! Consult journalctl logs for additional details.
</v-banner>
</v-form>
<v-row justify="center">
<v-chip
Expand Down
1 change: 1 addition & 0 deletions photon-client/src/stores/settings/CameraSettingsStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ export const useCameraSettingsStore = defineStore("cameraSettings", {
patternWidth: number;
patternHeight: number;
boardType: CalibrationBoardTypes;
useMrCal: boolean;
},
cameraIndex: number = useStateStore().currentCameraIndex
) {
Expand Down
6 changes: 4 additions & 2 deletions photon-client/src/stores/settings/GeneralSettingsStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ export const useSettingsStore = defineStore("settings", {
version: undefined,
gpuAcceleration: undefined,
hardwareModel: undefined,
hardwarePlatform: undefined
hardwarePlatform: undefined,
mrCalWorking: true
},
network: {
ntServerAddress: "",
Expand Down Expand Up @@ -97,7 +98,8 @@ export const useSettingsStore = defineStore("settings", {
version: data.general.version || undefined,
hardwareModel: data.general.hardwareModel || undefined,
hardwarePlatform: data.general.hardwarePlatform || undefined,
gpuAcceleration: data.general.gpuAcceleration || undefined
gpuAcceleration: data.general.gpuAcceleration || undefined,
mrCalWorking: data.general.mrCalWorking
};
this.lighting = data.lighting;
this.network = data.networkSettings;
Expand Down
1 change: 1 addition & 0 deletions photon-client/src/types/SettingTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export interface GeneralSettings {
gpuAcceleration?: string;
hardwareModel?: string;
hardwarePlatform?: string;
mrCalWorking: boolean;
}

export interface MetricData {
Expand Down
3 changes: 3 additions & 0 deletions photon-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ dependencies {

implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"
implementation "org.photonvision:photon-libcamera-gl-driver-java:$photonGlDriverLibVersion"

implementation "org.photonvision:photon-mrcal-java:$mrcalVersion"
implementation "org.photonvision:photon-mrcal-jni:$mrcalVersion:$wpilibNativeName"
}

task writeCurrentVersion {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.photonvision.common.hardware.Platform;
import org.photonvision.common.networking.NetworkUtils;
import org.photonvision.common.util.SerializationUtils;
import org.photonvision.mrcal.MrCal;
import org.photonvision.raspi.LibCameraJNILoader;
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;
import org.photonvision.vision.processes.VisionModule;
Expand Down Expand Up @@ -140,6 +141,7 @@ public Map<String, Object> toHashMap() {
LibCameraJNILoader.isSupported()
? "Zerocopy Libcamera Working"
: ""); // TODO add support for other types of GPU accel
generalSubmap.put("mrCalWorking", MrCal.isWorking());
generalSubmap.put("hardwareModel", hardwareConfig.deviceName);
generalSubmap.put("hardwarePlatform", Platform.getPlatformName());
settingsSubmap.put("general", generalSubmap);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.photonvision.common.hardware;

import com.jogamp.common.os.Platform.OSType;
import edu.wpi.first.util.RuntimeDetector;
import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -27,23 +28,32 @@
@SuppressWarnings("unused")
public enum Platform {
// WPILib Supported (JNI)
WINDOWS_64("Windows x64", false, OSType.WINDOWS, true),
LINUX_32("Linux x86", false, OSType.LINUX, true),
LINUX_64("Linux x64", false, OSType.LINUX, true),
WINDOWS_64("Windows x64", "winx64", false, OSType.WINDOWS, true),
LINUX_32("Linux x86", "linuxx64", false, OSType.LINUX, true),
LINUX_64("Linux x64", "linuxx64", false, OSType.LINUX, true),
LINUX_RASPBIAN32(
"Linux Raspbian 32-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 32-bit image
"Linux Raspbian 32-bit",
"linuxarm32",
true,
OSType.LINUX,
true), // Raspberry Pi 3/4 with a 32-bit image
LINUX_RASPBIAN64(
"Linux Raspbian 64-bit", true, OSType.LINUX, true), // Raspberry Pi 3/4 with a 64-bit image
LINUX_AARCH64("Linux AARCH64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2
"Linux Raspbian 64-bit",
"linuxarm64",
true,
OSType.LINUX,
true), // Raspberry Pi 3/4 with a 64-bit image
LINUX_AARCH64(
"Linux AARCH64", "linuxarm64", false, OSType.LINUX, true), // Jetson Nano, Jetson TX2

// PhotonVision Supported (Manual build/install)
LINUX_ARM32("Linux ARM32", false, OSType.LINUX, true), // ODROID XU4, C1+
LINUX_ARM64("Linux ARM64", false, OSType.LINUX, true), // ODROID C2, N2
LINUX_ARM32("Linux ARM32", "linuxarm32", false, OSType.LINUX, true), // ODROID XU4, C1+
LINUX_ARM64("Linux ARM64", "linuxarm64", false, OSType.LINUX, true), // ODROID C2, N2

// Completely unsupported
WINDOWS_32("Windows x86", false, OSType.WINDOWS, false),
MACOS("Mac OS", false, OSType.MACOS, false),
UNKNOWN("Unsupported Platform", false, OSType.UNKNOWN, false);
WINDOWS_32("Windows x86", "windowsx64", false, OSType.WINDOWS, false),
MACOS("Mac OS", "osxuniversal", false, OSType.MACOS, false),
UNKNOWN("Unsupported Platform", "", false, OSType.UNKNOWN, false);

private enum OSType {
WINDOWS,
Expand All @@ -54,6 +64,7 @@ private enum OSType {

private static final ShellExec shell = new ShellExec(true, false);
public final String description;
public final String nativeLibraryFolderName;
public final boolean isPi;
public final OSType osType;
public final boolean isSupported;
Expand All @@ -62,11 +73,17 @@ private enum OSType {
private static final Platform currentPlatform = getCurrentPlatform();
private static final boolean isRoot = checkForRoot();

Platform(String description, boolean isPi, OSType osType, boolean isSupported) {
Platform(
String description,
String nativeLibFolderName,
boolean isPi,
OSType osType,
boolean isSupported) {
this.description = description;
this.isPi = isPi;
this.osType = osType;
this.isSupported = isSupported;
this.nativeLibraryFolderName = nativeLibFolderName;
}

//////////////////////////////////////////////////////
Expand All @@ -89,6 +106,10 @@ public static String getPlatformName() {
}
}

public static final String getNativeLibraryFolderName() {
return currentPlatform.nativeLibraryFolderName;
}

public static boolean isRoot() {
return isRoot;
}
Expand Down Expand Up @@ -212,4 +233,9 @@ private static boolean fileHasText(String filename, String text) {
return false;
}
}

public static boolean isWindows() {
var p = getCurrentPlatform();
return (p == WINDOWS_32 || p == WINDOWS_64);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@
import org.photonvision.vision.calibration.CameraCalibrationCoefficients;

public class TestUtils {
private static boolean has_loaded = false;

public static boolean loadLibraries() {
if (has_loaded) return true;

NetworkTablesJNI.Helper.setExtractOnStaticLoad(false);
WPIUtilJNI.Helper.setExtractOnStaticLoad(false);
WPIMathJNI.Helper.setExtractOnStaticLoad(false);
Expand All @@ -61,11 +65,13 @@ public static boolean loadLibraries() {
"cscorejni",
"apriltagjni");

return true;
has_loaded = true;
} catch (IOException e) {
e.printStackTrace();
return false;
has_loaded = false;
}

return has_loaded;
}

@SuppressWarnings("unused")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

package org.photonvision.jni;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.photonvision.common.hardware.Platform;
import org.photonvision.common.logging.LogGroup;
import org.photonvision.common.logging.Logger;

public abstract class PhotonJniCommon {
static boolean libraryLoaded = false;
protected static Logger logger = null;

protected static synchronized void forceLoad(Class<?> clazz, List<String> libraries)
throws IOException {
if (libraryLoaded) return;
if (logger == null) logger = new Logger(clazz, LogGroup.Camera);

for (var libraryName : libraries) {
try {
// We always extract the shared object (we could hash each so, but that's a lot of work)
var arch_name = Platform.getNativeLibraryFolderName();
var nativeLibName = System.mapLibraryName(libraryName);
var in = clazz.getResourceAsStream("/nativelibraries/" + arch_name + "/" + nativeLibName);

if (in == null) {
libraryLoaded = false;
return;
}

// It's important that we don't mangle the names of these files on Windows at least
File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
FileOutputStream fos = new FileOutputStream(temp);

int read = -1;
byte[] buffer = new byte[1024];
while ((read = in.read(buffer)) != -1) {
fos.write(buffer, 0, read);
}
fos.close();
in.close();

System.load(temp.getAbsolutePath());

logger.info("Successfully loaded shared object " + temp.getName());

} catch (UnsatisfiedLinkError e) {
logger.error("Couldn't load shared object " + libraryName, e);
e.printStackTrace();
// logger.error(System.getProperty("java.library.path"));
break;
}
}
libraryLoaded = true;
}

protected static synchronized void forceLoad(Class<?> clazz, String libraryName)
throws IOException {
forceLoad(clazz, List.of(libraryName));
}

public static boolean isWorking() {
return libraryLoaded;
}
}
Loading

0 comments on commit ba190f9

Please sign in to comment.