draft

photon-client/src/App.vue

@@ -23,6 +23,7 @@ if (!is_demo) {
         useSettingsStore().updateGeneralSettingsFromWebsocket(data.settings);
       }
       if (data.cameraSettings !== undefined) {
+        console.log("cameras", data.cameraSettings);
         useCameraSettingsStore().updateCameraSettingsFromWebsocket(data.cameraSettings);
       }
       if (data.ntConnectionInfo !== undefined) {

photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue

@@ -170,7 +170,7 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
     { name: "AprilTag", value: WebsocketPipelineType.AprilTag },
     { name: "Aruco", value: WebsocketPipelineType.Aruco }
   ];
-  if (useSettingsStore().general.rknnSupported) {
+  if (useSettingsStore().general.supportedBackends.length > 0) {
     pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection });
   }
 

photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java

@@ -120,8 +120,8 @@ public List<String> getSupportedBackends() {
      * @return A map containing the available models, where the key is the backend and the value is a
      *     list of model names.
      */
-    public Map<String, ArrayList<String>> getModels() {
-        Map<String, ArrayList<String>> modelMap = new HashMap<>();
+    public HashMap<String, ArrayList<String>> getModels() {
+        HashMap<String, ArrayList<String>> modelMap = new HashMap<>();
         if (models == null) {
             return modelMap;
         }
@@ -177,14 +177,7 @@ public Optional<Model> getDefaultModel() {
         if (supportedBackends.isEmpty()) {
             return Optional.empty();
         }
-
-        if (supportedBackends.contains(NeuralNetworkBackend.RKNN)
-                && models.containsKey(NeuralNetworkBackend.RKNN)) {
-            return models.get(NeuralNetworkBackend.RKNN).stream()
-                    .filter(model -> model.getName().equals("note-640-640-yolov5s.rknn"))
-                    .findFirst();
-        }
-
+        logger.info("Default model: " + models.get(supportedBackends.get(0)).stream().findFirst().map(Model::getName).orElse("None"));
         return models.get(supportedBackends.get(0)).stream().findFirst();
     }
 
@@ -195,6 +188,10 @@ private void loadModel(File model) {
 
         // Get the model extension and check if it is supported
         String modelExtension = model.getName().substring(model.getName().lastIndexOf('.'));
+        if (modelExtension.equals(".txt")) {
+            return;
+        }
+
         Optional<NeuralNetworkBackend> backend =
                 Arrays.stream(NeuralNetworkBackend.values())
                         .filter(b -> b.format.equals(modelExtension))
@@ -206,12 +203,15 @@ private void loadModel(File model) {
         }
 
         String labels = model.getAbsolutePath().replace(backend.get().format, "-labels.txt");
-        ArrayList<Model> models = this.models.getOrDefault(backend.get(), new ArrayList<>());
+        if (!models.containsKey(backend.get())) {
+            models.put(backend.get(), new ArrayList<>());
+        }
 
         try {
             switch (backend.get()) {
                 case RKNN:
-                    models.add(new RknnModel(model, labels));
+                    models.get(backend.get()).add(new RknnModel(model, labels));     
+                    logger.info("Loaded model " + model.getName() + " for backend " + backend.get().toString());
                     break;
                 default:
                     break;
@@ -224,11 +224,11 @@ private void loadModel(File model) {
     }
 
     /**
-     * Loads models from the specified folder.
+     * Discovers DNN models from the specified folder.
      *
      * @param modelsFolder The folder where the models are stored
      */
-    public void loadModels(File modelsFolder) {
+    public void discoverModels(File modelsFolder) {
         logger.info("Supported backends: " + supportedBackends);
 
         if (!modelsFolder.exists()) {
@@ -245,17 +245,17 @@ public void loadModels(File modelsFolder) {
                     .filter(Files::isRegularFile)
                     .forEach(path -> loadModel(path.toFile()));
         } catch (IOException e) {
-            logger.error("Failed to load models from " + modelsFolder.getAbsolutePath(), e);
+            logger.error("Failed to discover models at " + modelsFolder.getAbsolutePath(), e);
         }
 
-        // After loading all of the models, sort them by name to ensure a consistent ordering each time
+        // After loading all of the models, sort them by name to ensure a consistent ordering
         models.forEach(
                 (backend, backendModels) ->
                         backendModels.sort((a, b) -> a.getName().compareTo(b.getName())));
 
-        // Log the loaded models
+        // Log
         StringBuilder sb = new StringBuilder();
-        sb.append("Loaded models: ");
+        sb.append("Discovered models: ");
         models.forEach(
                 (backend, backendModels) -> {
                     sb.append(backend).append(" [");
@@ -288,9 +288,8 @@ public void extractModels(File modelsDirectory) {
                     }
                     Path outputPath =
                             modelsDirectory.toPath().resolve(entry.getName().substring(resource.length() + 1));
-                    if (Files.exists(outputPath)) {
-                        continue;
-                    }
+
+                    logger.info("Extracting DNN resource: " + entry.getName());
                     Files.createDirectories(outputPath.getParent());
                     try (InputStream inputStream = jarFile.getInputStream(entry)) {
                         Files.copy(inputStream, outputPath, StandardCopyOption.REPLACE_EXISTING);

photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java

@@ -17,6 +17,9 @@
 
 package org.photonvision.vision.pipeline;
 
+import org.photonvision.common.configuration.NeuralNetworkModelManager;
+import org.photonvision.vision.objects.Model;
+
 public class ObjectDetectionPipelineSettings extends AdvancedPipelineSettings {
     public double confidence;
     public double nms; // non maximal suppression
@@ -31,6 +34,6 @@ public ObjectDetectionPipelineSettings() {
         this.ledMode = false;
         confidence = .9;
         nms = .45;
-        model = "";
+        model = NeuralNetworkModelManager.getInstance().getDefaultModel().map(Model::getName).orElse("");
     }
 }

photon-server/src/main/java/org/photonvision/Main.java

@@ -423,6 +423,11 @@ public static void main(String[] args) {
         ConfigManager.getInstance().load(); // init config manager
         ConfigManager.getInstance().requestSave();
 
+        logger.info("Loading ML models");
+        var modelManager = NeuralNetworkModelManager.getInstance();
+        modelManager.extractModels(ConfigManager.getInstance().getModelsDirectory());
+        modelManager.discoverModels(ConfigManager.getInstance().getModelsDirectory());
+
         logger.debug("Loading HardwareManager...");
         // Force load the hardware manager
         HardwareManager.getInstance();
@@ -434,11 +439,6 @@ public static void main(String[] args) {
         NetworkTablesManager.getInstance()
                 .setConfig(ConfigManager.getInstance().getConfig().getNetworkConfig());
 
-        logger.info("Loading ML models");
-        var modelManager = NeuralNetworkModelManager.getInstance();
-        modelManager.extractModels(ConfigManager.getInstance().getModelsDirectory());
-        modelManager.loadModels(ConfigManager.getInstance().getModelsDirectory());
-
         if (isSmoketest) {
             logger.info("PhotonVision base functionality loaded -- smoketest complete");
             System.exit(0);

photon-server/src/main/resources/web/index.html

@@ -5,7 +5,7 @@
     <link rel="icon" href="./favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Photon Client</title>
-    <script type="module" crossorigin src="./assets/index-50ab8a2c.js"></script>
+    <script type="module" crossorigin src="./assets/index-206c1fe7.js"></script>
     <link rel="stylesheet" href="./assets/index-69a5f457.css">
   </head>
   <body>