first pass at rknn

photon-client/src/components/dashboard/tabs/RknnTab.vue

@@ -18,24 +18,20 @@ const interactiveCols = computed(
 )
   ? 9
   : 8;
-
-const models = () => {
-  return ["a", "bcd"]
-}
 </script>
 
 <template>
   <div v-if="currentPipelineSettings.pipelineType === PipelineType.Rknn">
     <pv-slider
-      v-model="currentPipelineSettings.cameraBrightness"
+      v-model="currentPipelineSettings.confidence"
       class="pt-2"
       :slider-cols="interactiveCols"
       label="Confidence"
       tooltip="asdf"
       :min="0"
       :max="1"
-      :step="-1"
-      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ cameraExposure: value }, false)"
+      :step=".01"
+      @input="(value) => useCameraSettingsStore().changeCurrentPipelineSetting({ confidence: value }, false)"
     />
   </div>
 </template>

photon-client/src/types/PipelineTypes.ts

@@ -284,12 +284,14 @@ export const DefaultArucoPipelineSettings: ArucoPipelineSettings = {
 
 export interface RknnPipelineSettings extends PipelineSettings {
   pipelineType: PipelineType.Rknn;
+  confidence: number;
 }
 export type ConfigurableRknnPipelineSettings = Partial<Omit<RknnPipelineSettings, "pipelineType">> &
   ConfigurablePipelineSettings;
 export const DefaultRknnPipelineSettings: RknnPipelineSettings = {
   ...DefaultPipelineSettings,
   pipelineType: PipelineType.Rknn,
+  confidence: 90,
   cameraGain: 20,
   targetModel: TargetModel.InfiniteRechargeHighGoalOuter,
   ledMode: true,

photon-core/build.gradle

@@ -13,7 +13,7 @@ dependencies {
     implementation 'org.zeroturnaround:zt-zip:1.14'
 
     implementation "org.xerial:sqlite-jdbc:3.41.0.0"
-    def rknnjniversion = "dev-v2024.0.0-30-g001b5ec"
+    def rknnjniversion = "dev-v2024.0.0-32-g98b3886"
     implementation "org.photonvision:rknn_jni-jni:$rknnjniversion:linuxarm64"
     implementation "org.photonvision:rknn_jni-java:$rknnjniversion"
     implementation "org.photonvision:photon-libcamera-gl-driver-jni:$photonGlDriverLibVersion:linuxarm64"

photon-core/src/main/java/org/photonvision/jni/RknnDetector.java

@@ -6,19 +6,40 @@
 import java.util.stream.Collectors;
 
 import org.photonvision.rknn.RknnJNI;
+import org.photonvision.rknn.RknnJNI.RknnResult;
+import org.opencv.core.Rect2d;
 import org.photonvision.common.util.TestUtils;
-import org.photonvision.vision.aruco.ArucoDetectionResult;
 import org.photonvision.vision.opencv.CVMat;
+import org.photonvision.vision.pipe.impl.NeuralNetworkPipeResult;
 
 public class RknnDetector extends PhotonJNICommon {
+    static long objPointer = 0;
+    static boolean hasBeenDestroyed = false;
     public static synchronized void forceLoad() throws IOException {
-        forceLoad(RknnDetector.class, List.of("photonmiscjnijni"));
+        forceLoad(RknnDetector.class, List.of("rknn_jni", "rga", "rknnrt"));
     }
 
+    public static List<NeuralNetworkPipeResult> detect(CVMat in) {
+        RknnResult[] ret = RknnJNI.detect(objPointer, in.getMat().getNativeObjAddr());
 
-
+        return List.of(ret).stream().map(it->new NeuralNetworkPipeResult(
+            new Rect2d(it.left,it.top,it.right,it.bottom), it.class_id, it.conf
+        )).collect(Collectors.toList());
+    }
+    public static void release() {
+        if(!hasBeenDestroyed) {
+            RknnJNI.destroy(objPointer);
+            hasBeenDestroyed = true;
+        }
+        else {
+            System.out.print("aint no way bros tryna release the RKNN detector again skull");
+        }
+
+    }
+
     public static void main(String[] args) throws IOException {
         TestUtils.loadLibraries();
         forceLoad();
+        objPointer = RknnJNI.create("null");
     }
 }

photon-core/src/main/java/org/photonvision/vision/pipe/impl/NeuralNetworkPipeResult.java

@@ -3,10 +3,10 @@
 import org.opencv.core.Rect2d;
 
 public class NeuralNetworkPipeResult {
-    public NeuralNetworkPipeResult(Rect2d box2, Integer integer, Float float1) {
+    public NeuralNetworkPipeResult(Rect2d box2, Integer classIdx, Float confidence) {
         box = box2;
-        classIdx = integer;
-        confidence = float1;
+        this.classIdx = classIdx;
+        this.confidence = confidence;
     }
     public final int classIdx;
     public final Rect2d box;

photon-core/src/main/java/org/photonvision/vision/pipe/impl/RknnDetectionPipe.java

@@ -1,24 +1,124 @@
 package org.photonvision.vision.pipe.impl;
+import java.util.Iterator;
 import java.util.List;
 
 import org.photonvision.vision.opencv.CVMat;
 import org.photonvision.vision.opencv.Releasable;
 import org.photonvision.vision.pipe.CVPipe;
-import org.photonvision.vision.target.TrackedTarget;
-public class RknnDetectionPipe extends CVPipe<CVMat, List<TrackedTarget>, RknnDetectionPipe.RknnDetectionPipeParams> {
+import org.photonvision.jni.RknnDetector;
+public class RknnDetectionPipe extends CVPipe<CVMat, List<NeuralNetworkPipeResult>, RknnDetectionPipe.RknnDetectionPipeParams> {
 
     @Override
-    protected List<TrackedTarget> process(CVMat in) {
-        // TODO Auto-generated method stub
-        throw new UnsupportedOperationException("Unimplemented method 'process'");
+    protected List<NeuralNetworkPipeResult> process(CVMat in) {
+        var frame = in.getMat();
+        if (frame.empty()) {
+            return List.of();
+        }
+        double confThreshold = params.confidence;
+        List<NeuralNetworkPipeResult> result = RknnDetector.detect(in);
+        Iterator<NeuralNetworkPipeResult> itr = result.iterator();
+        while(itr.hasNext()) {
+            NeuralNetworkPipeResult res = itr.next();
+            if (res.confidence < confThreshold) {
+                itr.remove();
+            }
+        }
+        return result;
+
     }
 
     public static class RknnDetectionPipeParams implements Releasable{
-
+        public double confidence;
+        List<String> classNames;
+        public RknnDetectionPipeParams() {
+            this.classNames = List.of(
+                "person",
+                "bicycle",
+                "car",
+                "motorcycle",
+                "airplane",
+                "bus",
+                "train",
+                "truck",
+                "boat",
+                "traffic light",
+                "fire hydrant",
+                "stop sign",
+                "parking meter",
+                "bench",
+                "bird",
+                "cat",
+                "dog",
+                "horse",
+                "sheep",
+                "cow",
+                "elephant",
+                "bear",
+                "zebra",
+                "giraffe",
+                "backpack",
+                "umbrella",
+                "handbag",
+                "tie",
+                "suitcase",
+                "frisbee",
+                "skis",
+                "snowboard",
+                "sports ball",
+                "kite",
+                "baseball bat",
+                "baseball glove",
+                "skateboard",
+                "surfboard",
+                "tennis racket",
+                "bottle",
+                "wine glass",
+                "cup",
+                "fork",
+                "knife",
+                "spoon",
+                "bowl",
+                "banana",
+                "apple",
+                "sandwich",
+                "orange",
+                "broccoli",
+                "carrot",
+                "hot dog",
+                "pizza",
+                "donut",
+                "cake",
+                "chair",
+                "couch",
+                "potted plant",
+                "bed",
+                "dining table",
+                "toilet",
+                "tv",
+                "laptop",
+                "mouse",
+                "remote",
+                "keyboard",
+                "cell phone",
+                "microwave",
+                "oven",
+                "toaster",
+                "sink",
+                "refrigerator",
+                "book",
+                "clock",
+                "vase",
+                "scissors",
+                "teddy bear",
+                "hair drier",
+                "toothbrush");
+        }
         @Override
         public void release() {
-            // TODO Auto-generated method stub
-            throw new UnsupportedOperationException("Unimplemented method 'release'");
+            RknnDetector.release();
+        }
+        public List<String> getClassNames(){
+            return classNames;
         }
 
     }

photon-core/src/main/java/org/photonvision/vision/pipeline/RknnPipeline.java

@@ -1,5 +1,110 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 package org.photonvision.vision.pipeline;
 
-public class RknnPipeline {
-
-}
+import java.util.ArrayList;
+import java.util.List;
+import org.opencv.core.Core;
+import org.opencv.core.Mat;
+import org.opencv.core.MatOfFloat;
+import org.opencv.core.MatOfInt;
+import org.opencv.core.MatOfRect2d;
+import org.opencv.core.Point;
+import org.opencv.core.Rect2d;
+import org.opencv.core.Scalar;
+import org.opencv.core.Size;
+import org.opencv.dnn.Dnn;
+import org.opencv.dnn.Net;
+import org.opencv.imgproc.Imgproc;
+import org.opencv.utils.Converters;
+import org.photonvision.common.util.ColorHelper;
+import org.photonvision.vision.frame.Frame;
+import org.photonvision.vision.frame.FrameThresholdType;
+import org.photonvision.vision.pipe.CVPipe.CVPipeResult;
+import org.photonvision.vision.pipe.impl.*;
+import org.photonvision.vision.pipe.impl.RknnDetectionPipe.RknnDetectionPipeParams;
+import org.photonvision.vision.pipeline.result.CVPipelineResult;
+import org.photonvision.vision.target.TrackedTarget;
+import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;
+
+public class RknnPipeline extends CVPipeline<CVPipelineResult, RknnPipelineSettings> {
+    private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();
+    private final RknnDetectionPipe rknnPipe = new RknnDetectionPipe();
+
+    private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;
+
+    public RknnPipeline() {
+        super(PROCESSING_TYPE);
+        settings = new RknnPipelineSettings();
+    }
+
+    public RknnPipeline(RknnPipelineSettings settings) {
+        super(PROCESSING_TYPE);
+        this.settings = settings;
+    }
+
+    @Override
+    protected void setPipeParamsImpl() {
+        var params = new RknnDetectionPipeParams();
+        params.confidence = settings.confidence;
+        rknnPipe.setParams(params);
+    }
+
+    @Override
+    protected CVPipelineResult process(Frame input_frame, RknnPipelineSettings settings) {
+        long sumPipeNanosElapsed = 0;
+
+        CVPipeResult<List<NeuralNetworkPipeResult>> ret = rknnPipe.run(input_frame.colorImage);
+        sumPipeNanosElapsed += ret.nanosElapsed;
+        List<NeuralNetworkPipeResult> targetList;
+
+        targetList = ret.output;
+        var names = rknnPipe.getParams().getClassNames();
+
+        input_frame.colorImage.getMat().copyTo(input_frame.processedImage.getMat());
+
+        List<TrackedTarget> targets = new ArrayList<>();
+
+        // This belongs in a collect & draw pipe but I'm lazy
+        for (var t : targetList) {
+            Imgproc.rectangle(input_frame.processedImage.getMat(), t.box.tl(), t.box.br(), new Scalar(0, 0, 255), 2);
+
+            var name = String.format("%s (%f)", names.get(t.classIdx), t.confidence);
+
+            Imgproc.putText(
+                    input_frame.processedImage.getMat(),
+                    name,
+                    new Point(t.box.x + t.box.width / 2.0, t.box.y + t.box.height / 2.0),
+                    0,
+                    0.6,
+                    ColorHelper.colorToScalar(java.awt.Color.white),
+                    2);
+
+            targets.add(
+                    new TrackedTarget(
+                            t,
+                            new TargetCalculationParameters(
+                                    false, null, null, null, null, frameStaticProperties)));
+        }
+
+        var fpsResult = calculateFPSPipe.run(null);
+        var fps = fpsResult.output;
+
+        return new CVPipelineResult(sumPipeNanosElapsed, fps, targets, input_frame);
+    }
+}

photon-core/src/main/java/org/photonvision/vision/pipeline/RknnPipelineSettings.java

@@ -2,16 +2,15 @@
 package org.photonvision.vision.pipeline;
 
 public class RknnPipelineSettings extends AdvancedPipelineSettings {
-    double confidence;
+    public double confidence = 90.0;
 
     public RknnPipelineSettings() {
         super();
         this.pipelineType = PipelineType.Rknn;
-
-        // Sane defaults
         this.outputShowMultipleTargets = true;
         cameraExposure = 20;
         cameraAutoExposure = false;
         ledMode = false;
+        double confidence = 90.0;
     }
 }

photon-core/src/main/java/org/photonvision/vision/processes/PipelineManager.java

@@ -215,7 +215,7 @@ private void setPipelineInternal(int newIndex) {
                     break;
                 case Rknn:
                     logger.debug("Creating Rknn Pipeline");
-                    //currentUserPipeline = new RknnPipeline();
+                    currentUserPipeline = new RknnPipeline((RknnPipelineSettings) desiredPipelineSettings);
                 default:
                     // Can be calib3d or drivermode, both of which are special cases
                     break;