From 6554a426f11bb39c6a6a286ec08b989e6a7716e4 Mon Sep 17 00:00:00 2001 From: Alextopher Date: Wed, 11 Sep 2024 10:38:31 -0400 Subject: [PATCH] draft --- photon-client/src/App.vue | 1 + .../dashboard/CameraAndPipelineSelectCard.vue | 2 +- .../NeuralNetworkModelManager.java | 41 +++++++++---------- .../ObjectDetectionPipelineSettings.java | 5 ++- .../src/main/java/org/photonvision/Main.java | 10 ++--- .../src/main/resources/web/index.html | 2 +- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/photon-client/src/App.vue b/photon-client/src/App.vue index 518ad39833..d98300bd7f 100644 --- a/photon-client/src/App.vue +++ b/photon-client/src/App.vue @@ -23,6 +23,7 @@ if (!is_demo) { useSettingsStore().updateGeneralSettingsFromWebsocket(data.settings); } if (data.cameraSettings !== undefined) { + console.log("cameras", data.cameraSettings); useCameraSettingsStore().updateCameraSettingsFromWebsocket(data.cameraSettings); } if (data.ntConnectionInfo !== undefined) { diff --git a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue index 15511460b2..8f5f3b33e3 100644 --- a/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue +++ b/photon-client/src/components/dashboard/CameraAndPipelineSelectCard.vue @@ -170,7 +170,7 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => { { name: "AprilTag", value: WebsocketPipelineType.AprilTag }, { name: "Aruco", value: WebsocketPipelineType.Aruco } ]; - if (useSettingsStore().general.rknnSupported) { + if (useSettingsStore().general.supportedBackends.length > 0) { pipelineTypes.push({ name: "Object Detection", value: WebsocketPipelineType.ObjectDetection }); } diff --git a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java index 08f6866c1c..1133bfde1a 100644 --- a/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java +++ b/photon-core/src/main/java/org/photonvision/common/configuration/NeuralNetworkModelManager.java @@ -120,8 +120,8 @@ public List getSupportedBackends() { * @return A map containing the available models, where the key is the backend and the value is a * list of model names. */ - public Map> getModels() { - Map> modelMap = new HashMap<>(); + public HashMap> getModels() { + HashMap> modelMap = new HashMap<>(); if (models == null) { return modelMap; } @@ -177,14 +177,7 @@ public Optional getDefaultModel() { if (supportedBackends.isEmpty()) { return Optional.empty(); } - - if (supportedBackends.contains(NeuralNetworkBackend.RKNN) - && models.containsKey(NeuralNetworkBackend.RKNN)) { - return models.get(NeuralNetworkBackend.RKNN).stream() - .filter(model -> model.getName().equals("note-640-640-yolov5s.rknn")) - .findFirst(); - } - + logger.info("Default model: " + models.get(supportedBackends.get(0)).stream().findFirst().map(Model::getName).orElse("None")); return models.get(supportedBackends.get(0)).stream().findFirst(); } @@ -195,6 +188,10 @@ private void loadModel(File model) { // Get the model extension and check if it is supported String modelExtension = model.getName().substring(model.getName().lastIndexOf('.')); + if (modelExtension.equals(".txt")) { + return; + } + Optional backend = Arrays.stream(NeuralNetworkBackend.values()) .filter(b -> b.format.equals(modelExtension)) @@ -206,12 +203,15 @@ private void loadModel(File model) { } String labels = model.getAbsolutePath().replace(backend.get().format, "-labels.txt"); - ArrayList models = this.models.getOrDefault(backend.get(), new ArrayList<>()); + if (!models.containsKey(backend.get())) { + models.put(backend.get(), new ArrayList<>()); + } try { switch (backend.get()) { case RKNN: - models.add(new RknnModel(model, labels)); + models.get(backend.get()).add(new RknnModel(model, labels)); + logger.info("Loaded model " + model.getName() + " for backend " + backend.get().toString()); break; default: break; @@ -224,11 +224,11 @@ private void loadModel(File model) { } /** - * Loads models from the specified folder. + * Discovers DNN models from the specified folder. * * @param modelsFolder The folder where the models are stored */ - public void loadModels(File modelsFolder) { + public void discoverModels(File modelsFolder) { logger.info("Supported backends: " + supportedBackends); if (!modelsFolder.exists()) { @@ -245,17 +245,17 @@ public void loadModels(File modelsFolder) { .filter(Files::isRegularFile) .forEach(path -> loadModel(path.toFile())); } catch (IOException e) { - logger.error("Failed to load models from " + modelsFolder.getAbsolutePath(), e); + logger.error("Failed to discover models at " + modelsFolder.getAbsolutePath(), e); } - // After loading all of the models, sort them by name to ensure a consistent ordering each time + // After loading all of the models, sort them by name to ensure a consistent ordering models.forEach( (backend, backendModels) -> backendModels.sort((a, b) -> a.getName().compareTo(b.getName()))); - // Log the loaded models + // Log StringBuilder sb = new StringBuilder(); - sb.append("Loaded models: "); + sb.append("Discovered models: "); models.forEach( (backend, backendModels) -> { sb.append(backend).append(" ["); @@ -288,9 +288,8 @@ public void extractModels(File modelsDirectory) { } Path outputPath = modelsDirectory.toPath().resolve(entry.getName().substring(resource.length() + 1)); - if (Files.exists(outputPath)) { - continue; - } + + logger.info("Extracting DNN resource: " + entry.getName()); Files.createDirectories(outputPath.getParent()); try (InputStream inputStream = jarFile.getInputStream(entry)) { Files.copy(inputStream, outputPath, StandardCopyOption.REPLACE_EXISTING); diff --git a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java index f7f97dcc74..9c5ab0267e 100644 --- a/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java +++ b/photon-core/src/main/java/org/photonvision/vision/pipeline/ObjectDetectionPipelineSettings.java @@ -17,6 +17,9 @@ package org.photonvision.vision.pipeline; +import org.photonvision.common.configuration.NeuralNetworkModelManager; +import org.photonvision.vision.objects.Model; + public class ObjectDetectionPipelineSettings extends AdvancedPipelineSettings { public double confidence; public double nms; // non maximal suppression @@ -31,6 +34,6 @@ public ObjectDetectionPipelineSettings() { this.ledMode = false; confidence = .9; nms = .45; - model = ""; + model = NeuralNetworkModelManager.getInstance().getDefaultModel().map(Model::getName).orElse(""); } } diff --git a/photon-server/src/main/java/org/photonvision/Main.java b/photon-server/src/main/java/org/photonvision/Main.java index ab3b15aadc..889be73dbe 100644 --- a/photon-server/src/main/java/org/photonvision/Main.java +++ b/photon-server/src/main/java/org/photonvision/Main.java @@ -423,6 +423,11 @@ public static void main(String[] args) { ConfigManager.getInstance().load(); // init config manager ConfigManager.getInstance().requestSave(); + logger.info("Loading ML models"); + var modelManager = NeuralNetworkModelManager.getInstance(); + modelManager.extractModels(ConfigManager.getInstance().getModelsDirectory()); + modelManager.discoverModels(ConfigManager.getInstance().getModelsDirectory()); + logger.debug("Loading HardwareManager..."); // Force load the hardware manager HardwareManager.getInstance(); @@ -434,11 +439,6 @@ public static void main(String[] args) { NetworkTablesManager.getInstance() .setConfig(ConfigManager.getInstance().getConfig().getNetworkConfig()); - logger.info("Loading ML models"); - var modelManager = NeuralNetworkModelManager.getInstance(); - modelManager.extractModels(ConfigManager.getInstance().getModelsDirectory()); - modelManager.loadModels(ConfigManager.getInstance().getModelsDirectory()); - if (isSmoketest) { logger.info("PhotonVision base functionality loaded -- smoketest complete"); System.exit(0); diff --git a/photon-server/src/main/resources/web/index.html b/photon-server/src/main/resources/web/index.html index 5c33c978fc..3d1379c9fb 100644 --- a/photon-server/src/main/resources/web/index.html +++ b/photon-server/src/main/resources/web/index.html @@ -5,7 +5,7 @@ Photon Client - +