Merge remote-tracking branch 'upstream/master' into nanobind

README.md

@@ -37,9 +37,11 @@ Note that these are case sensitive!
     * arm64
     * x86-64
     * x86
-- `-PtgtIp`: Specifies where `./gradlew deploy` should try to copy the fat JAR to
+- `-PtgtIP`: Specifies where `./gradlew deploy` should try to copy the fat JAR to
 - `-Pprofile`: enables JVM profiling
 
+If you're cross-compiling, you'll need the wpilib toolchain installed. This can be done via Gradle: for example `./gradlew installArm64Toolchain` or `./gradlew installRoboRioToolchain`
+
 ## Out-of-Source Dependencies
 
 PhotonVision uses the following additonal out-of-source repositories for building code.

build.gradle

@@ -1,12 +1,15 @@
 import edu.wpi.first.toolchain.*
 
 plugins {
+    id "java"
+    id "cpp"
     id "com.diffplug.spotless" version "6.24.0"
     id "edu.wpi.first.NativeUtils" version "2024.6.1" apply false
     id "edu.wpi.first.wpilib.repositories.WPILibRepositoriesPlugin" version "2020.2"
     id "edu.wpi.first.GradleRIO" version "2024.3.2"
     id 'edu.wpi.first.WpilibTools' version '1.3.0'
     id 'com.google.protobuf' version '0.9.4' apply false
+    id 'edu.wpi.first.GradleJni' version '1.1.0'
 }
 
 allprojects {

docs/source/docs/examples/poseest.md

@@ -1,6 +1,6 @@
 # Using WPILib Pose Estimation, Simulation, and PhotonVision Together
 
-The following example comes from the PhotonLib example repository ([Java](https://github.com/gerth2/photonvision/tree/master/photonlib-java-examples/poseest)/[C++](https://github.com/gerth2/photonvision/tree/master/photonlib-cpp-examples/poseest)/[Python](https://github.com/gerth2/photonvision/tree/master/photonlib-python-examples/poseest)).  Full code is available at that links.
+The following example comes from the PhotonLib example repository ([Java](https://github.com/PhotonVision/photonvision/tree/master/photonlib-java-examples/poseest)/[C++](https://github.com/PhotonVision/photonvision/tree/master/photonlib-cpp-examples/poseest)/[Python](https://github.com/PhotonVision/photonvision/tree/master/photonlib-python-examples/poseest)).  Full code is available at that links.
 
 ## Knowledge and Equipment Needed
 

photon-client/src/components/app/photon-camera-stream.vue

@@ -57,7 +57,7 @@ const handleFullscreenRequest = () => {
 const mjpgStream: any = ref(null);
 onBeforeUnmount(() => {
   if (!mjpgStream.value) return;
-  mjpgStream.value["src"] = null;
+  mjpgStream.value["src"] = "//:0";
 });
 </script>
 

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -132,7 +132,7 @@ const validNewPipelineTypes = computed(() => {
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
     { name: "Aruco", value: WebsocketPipelineType.Aruco }
   ];
-  if (useSettingsStore().general.rknnSupported) {
+  if (useSettingsStore().general.supportedBackends.length > 0) {
     pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
   }
   return pipelineTypes;
@@ -170,7 +170,7 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
     { name: "Aruco", value: WebsocketPipelineType.Aruco }
   ];
-  if (useSettingsStore().general.rknnSupported) {
+  if (useSettingsStore().general.supportedBackends.length > 0) {
     pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
   }
 

photon-client/src/components/dashboard/tabs/ObjectDetectionTab.vue

@@ -1,14 +1,15 @@
 <script setup lang="ts">
 import { useCameraSettingsStore } from "@/stores/settings/CameraSettingsStore";
-import { type ActivePipelineSettings, PipelineType } from "@/types/PipelineTypes";
+import { type ObjectDetectionPipelineSettings, PipelineType } from "@/types/PipelineTypes";
 import PvSlider from "@/components/common/pv-slider.vue";
 import { computed, getCurrentInstance } from "vue";
 import { useStateStore } from "@/stores/StateStore";
+import { useSettingsStore } from "@/stores/settings/GeneralSettingsStore";
 
 // TODO fix pipeline typing in order to fix this, the store settings call should be able to infer that only valid pipeline type settings are exposed based on pre-checks for the entire config section
 // Defer reference to store access method
-const currentPipelineSettings = computed<ActivePipelineSettings>(
-  () => useCameraSettingsStore().currentPipelineSettings
+const currentPipelineSettings = computed<ObjectDetectionPipelineSettings>(
+  () => useCameraSettingsStore().currentPipelineSettings as ObjectDetectionPipelineSettings
 );
 
 // TODO fix pv-range-slider so that store access doesn't need to be deferred
@@ -27,10 +28,30 @@ const interactiveCols = computed(() =>
     ? 9
     : 8
 );
+
+// Filters out models that are not supported by the current backend, and returns a flattened list.
+const supportedModels = computed(() => {
+  const { availableModels, supportedBackends } = useSettingsStore().general;
+  return supportedBackends.flatMap((backend) => availableModels[backend] || []);
+});
+
+const selectedModel = computed({
+  get: () => supportedModels.value.indexOf(currentPipelineSettings.value.model),
+  set: (v) => {
+    useCameraSettingsStore().changeCurrentPipelineSetting({ model: supportedModels.value[v] }, false);
+  }
+});
 </script>
 
 <template>
   <div v-if="currentPipelineSettings.pipelineType === PipelineType.ObjectDetection">
+    <pv-select
+      v-model="selectedModel"
+      label="Model"
+      tooltip="The model used to detect objects in the camera feed"
+      :select-cols="interactiveCols"
+      :items="supportedModels"
+    />
     <pv-slider
       v-model="currentPipelineSettings.confidence"
       class="pt-2"

photon-client/src/stores/settings/GeneralSettingsStore.ts

@@ -28,7 +28,8 @@ export const useSettingsStore = defineStore("settings", {
       hardwareModel: undefined,
       hardwarePlatform: undefined,
       mrCalWorking: true,
-      rknnSupported: false
+      availableModels: {},
+      supportedBackends: []
     },
     network: {
       ntServerAddress: "",
@@ -105,7 +106,8 @@ export const useSettingsStore = defineStore("settings", {
         hardwarePlatform: data.general.hardwarePlatform || undefined,
         gpuAcceleration: data.general.gpuAcceleration || undefined,
         mrCalWorking: data.general.mrCalWorking,
-        rknnSupported: data.general.rknnSupported
+        availableModels: data.general.availableModels || undefined,
+        supportedBackends: data.general.supportedBackends || []
       };
       this.lighting = data.lighting;
       this.network = data.networkSettings;

photon-client/src/types/PipelineTypes.ts

@@ -289,6 +289,7 @@ export interface ObjectDetectionPipelineSettings extends PipelineSettings {
   confidence: number;
   nms: number;
   box_thresh: number;
+  model: string;
 }
 export type ConfigurableObjectDetectionPipelineSettings = Partial<
   Omit<ObjectDetectionPipelineSettings, "pipelineType">
@@ -304,7 +305,8 @@ export const DefaultObjectDetectionPipelineSettings: ObjectDetectionPipelineSett
   cameraExposureRaw: 6,
   confidence: 0.9,
   nms: 0.45,
-  box_thresh: 0.25
+  box_thresh: 0.25,
+  model: ""
 };
 
 export type ActivePipelineSettings =

photon-client/src/types/SettingTypes.ts

@@ -7,7 +7,8 @@ export interface GeneralSettings {
   hardwareModel?: string;
   hardwarePlatform?: string;
   mrCalWorking: boolean;
-  rknnSupported: boolean;
+  availableModels: Record<string, string[]>;
+  supportedBackends: string[];
 }
 
 export interface MetricData {