Skip to content

Commit

Permalink
Create opencv DNN pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
mcm001 committed Oct 15, 2023
1 parent 760de0f commit 80078db
Show file tree
Hide file tree
Showing 9 changed files with 370 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ const changeCurrentCameraIndex = (index: number) => {
case PipelineType.Aruco:
pipelineType.value = WebsocketPipelineType.Aruco;
break;
case PipelineType.Dnn:
pipelineType.value = WebsocketPipelineType.Dnn;
break;
}
};
Expand Down Expand Up @@ -151,8 +154,9 @@ const pipelineTypesWrapper = computed<{ name: string; value: number }[]>(() => {
const pipelineTypes = [
{ name: "Reflective", value: WebsocketPipelineType.Reflective },
{ name: "Colored Shape", value: WebsocketPipelineType.ColoredShape },
{ name: "AprilTag", value: WebsocketPipelineType.AprilTag }
{ name: "AprilTag", value: WebsocketPipelineType.AprilTag },
// { name: "Aruco", value: WebsocketPipelineType.Aruco }
{ name: "DNN", value: WebsocketPipelineType.Dnn }
];
if (useCameraSettingsStore().isDriverMode) {
Expand Down Expand Up @@ -203,6 +207,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
case PipelineType.Aruco:
pipelineType.value = WebsocketPipelineType.Aruco;
break;
case PipelineType.Dnn:
pipelineType.value = WebsocketPipelineType.Dnn;
break;
}
});
</script>
Expand Down Expand Up @@ -322,8 +329,9 @@ useCameraSettingsStore().$subscribe((mutation, state) => {
:items="[
{ name: 'Reflective', value: WebsocketPipelineType.Reflective },
{ name: 'Colored Shape', value: WebsocketPipelineType.ColoredShape },
{ name: 'AprilTag', value: WebsocketPipelineType.AprilTag }
{ name: 'AprilTag', value: WebsocketPipelineType.AprilTag },
// { name: 'Aruco', value: WebsocketPipelineType.Aruco }
{ name: 'Dnn', value: WebsocketPipelineType.Dnn }
]"
/>
</v-card-text>
Expand Down
19 changes: 16 additions & 3 deletions photon-client/src/types/PipelineTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ export enum PipelineType {
Reflective = 2,
ColoredShape = 3,
AprilTag = 4,
Aruco = 5
Aruco = 5,
Dnn = 6
}

export enum AprilTagFamily {
Expand Down Expand Up @@ -239,6 +240,16 @@ export const DefaultAprilTagPipelineSettings: AprilTagPipelineSettings = {
tagFamily: AprilTagFamily.Family16h5
};

export interface DnnPipelineSettings extends PipelineSettings {
pipelineType: PipelineType.Dnn;
}
export type ConfigurableDnnPipelineSettings = Partial<Omit<DnnPipelineSettings, "pipelineType">> &
ConfigurablePipelineSettings;
export const DefaultDnnPipelineSettings: DnnPipelineSettings = {
...DefaultPipelineSettings,
pipelineType: PipelineType.Dnn
};

export interface ArucoPipelineSettings extends PipelineSettings {
pipelineType: PipelineType.Aruco;
decimate: number;
Expand Down Expand Up @@ -269,9 +280,11 @@ export type ActivePipelineSettings =
| ReflectivePipelineSettings
| ColoredShapePipelineSettings
| AprilTagPipelineSettings
| ArucoPipelineSettings;
| ArucoPipelineSettings
| DnnPipelineSettings;
export type ActiveConfigurablePipelineSettings =
| ConfigurableReflectivePipelineSettings
| ConfigurableColoredShapePipelineSettings
| ConfigurableAprilTagPipelineSettings
| ConfigurableArucoPipelineSettings;
| ConfigurableArucoPipelineSettings
| ConfigurableDnnPipelineSettings;
3 changes: 2 additions & 1 deletion photon-client/src/types/WebsocketDataTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -119,5 +119,6 @@ export enum WebsocketPipelineType {
Reflective = 0,
ColoredShape = 1,
AprilTag = 2,
Aruco = 3
Aruco = 3,
Dnn = 4
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
@JsonSubTypes.Type(value = ReflectivePipelineSettings.class),
@JsonSubTypes.Type(value = DriverModePipelineSettings.class),
@JsonSubTypes.Type(value = AprilTagPipelineSettings.class),
@JsonSubTypes.Type(value = ArucoPipelineSettings.class)
@JsonSubTypes.Type(value = ArucoPipelineSettings.class),
@JsonSubTypes.Type(value = DnnPipelineSettings.class)
})
public class CVPipelineSettings implements Cloneable {
public int pipelineIndex = 0;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,273 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

package org.photonvision.vision.pipeline;

import java.util.ArrayList;
import java.util.List;
import org.opencv.core.Core;
import org.opencv.core.Mat;
import org.opencv.core.MatOfFloat;
import org.opencv.core.MatOfInt;
import org.opencv.core.MatOfRect2d;
import org.opencv.core.Point;
import org.opencv.core.Rect2d;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgproc.Imgproc;
import org.opencv.utils.Converters;
import org.photonvision.common.util.ColorHelper;
import org.photonvision.vision.frame.Frame;
import org.photonvision.vision.frame.FrameThresholdType;
import org.photonvision.vision.pipe.impl.*;
import org.photonvision.vision.pipeline.result.CVPipelineResult;
import org.photonvision.vision.target.TrackedTarget;
import org.photonvision.vision.target.TrackedTarget.TargetCalculationParameters;

public class DnnPipeline extends CVPipeline<CVPipelineResult, DnnPipelineSettings> {
private final CalculateFPSPipe calculateFPSPipe = new CalculateFPSPipe();

private static final FrameThresholdType PROCESSING_TYPE = FrameThresholdType.NONE;

public DnnPipeline() {
super(PROCESSING_TYPE);
settings = new DnnPipelineSettings();
}

Net net = null;
private List<String> outBlobNames = List.of();

private List<String> coco_names;

public DnnPipeline(DnnPipelineSettings settings) {
super(PROCESSING_TYPE);
this.settings = settings;

// Downloaded from https://dev.to/kojix2/yolov7-object-detection-in-ruby-in-10-minutes-5cjh
// https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/307_YOLOv7/with-postprocess/resources_post.tar.gz
try {
// this.net = Dnn.readNetFromONNX("/home/matt/Downloads/best_1.onnx");
// this.net = Dnn.readNet("/home/matt/Downloads/yolov7_post_640x640.onnx");
this.net =
Dnn.readNetFromDarknet(
"/home/matt/Downloads/yolov4-csp-swish.cfg",
"/home/matt/Downloads/yolov4-csp-swish.weights");
Core.setNumThreads(8);
} catch (Exception e) {
System.out.println(e);
}
this.outBlobNames = getOutputNames(net);

this.coco_names =
List.of(
"person",
"bicycle",
"car",
"motorcycle",
"airplane",
"bus",
"train",
"truck",
"boat",
"traffic light",
"fire hydrant",
"stop sign",
"parking meter",
"bench",
"bird",
"cat",
"dog",
"horse",
"sheep",
"cow",
"elephant",
"bear",
"zebra",
"giraffe",
"backpack",
"umbrella",
"handbag",
"tie",
"suitcase",
"frisbee",
"skis",
"snowboard",
"sports ball",
"kite",
"baseball bat",
"baseball glove",
"skateboard",
"surfboard",
"tennis racket",
"bottle",
"wine glass",
"cup",
"fork",
"knife",
"spoon",
"bowl",
"banana",
"apple",
"sandwich",
"orange",
"broccoli",
"carrot",
"hot dog",
"pizza",
"donut",
"cake",
"chair",
"couch",
"potted plant",
"bed",
"dining table",
"toilet",
"tv",
"laptop",
"mouse",
"remote",
"keyboard",
"cell phone",
"microwave",
"oven",
"toaster",
"sink",
"refrigerator",
"book",
"clock",
"vase",
"scissors",
"teddy bear",
"hair drier",
"toothbrush");
}

@Override
protected void setPipeParamsImpl() {}

private static List<String> getOutputNames(Net net) {
List<String> names = new ArrayList<>();

List<Integer> outLayers = net.getUnconnectedOutLayers().toList();
List<String> layersNames = net.getLayerNames();

outLayers.forEach(
(item) -> names.add(layersNames.get(item - 1))); // unfold and create R-CNN layers from the
// loaded YOLO model//
return names;
}

@Override
protected CVPipelineResult process(Frame input_frame, DnnPipelineSettings settings) {
long sumPipeNanosElapsed = 0L;

// ======================

var frame = input_frame.colorImage.getMat();

if (frame.empty()) {
return new CVPipelineResult(sumPipeNanosElapsed, 0, List.of(), input_frame);
}

var blob = Dnn.blobFromImage(frame, 1.0 / 255.0, new Size(640, 640));
net.setInput(blob);

List<Mat> result = new ArrayList<>();
net.forward(result, outBlobNames); // outputlayer : output1 and output2

// From https://github.com/suddh123/YOLO-object-detection-in-java/blob/code/yolo.java

float confThreshold = 0.3f; // Insert thresholding beyond which the model will detect objects//
List<Integer> clsIds = new ArrayList<>();
List<Float> confs = new ArrayList<>();
List<Rect2d> rects = new ArrayList<>();
for (int i = 0; i < result.size(); ++i) {
// each row is a candidate detection, the 1st 4 numbers are
// [center_x, center_y, width, height], followed by (N-4) class probabilities
Mat level = result.get(i);
for (int j = 0; j < level.rows(); ++j) {
Mat row = level.row(j);
Mat scores = row.colRange(5, level.cols());
Core.MinMaxLocResult mm = Core.minMaxLoc(scores);
float confidence = (float) mm.maxVal;
Point classIdPoint = mm.maxLoc;
if (confidence > confThreshold) {
// scaling for drawing the bounding boxes//
int centerX = (int) (row.get(0, 0)[0] * frame.cols());
int centerY = (int) (row.get(0, 1)[0] * frame.rows());
int width = (int) (row.get(0, 2)[0] * frame.cols());
int height = (int) (row.get(0, 3)[0] * frame.rows());
int left = centerX - width / 2;
int top = centerY - height / 2;

clsIds.add((int) classIdPoint.x);
confs.add((float) confidence);
rects.add(new Rect2d(left, top, width, height));
}
}
}
float nmsThresh = 0.5f;
MatOfFloat confidences = new MatOfFloat(Converters.vector_float_to_Mat(confs));
Rect2d[] boxesArray = rects.toArray(new Rect2d[0]);
MatOfRect2d boxes = new MatOfRect2d(boxesArray);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(
boxes,
confidences,
confThreshold,
nmsThresh,
indices); // We draw the bounding boxes for objects
// here//

List<TrackedTarget> targetList = new ArrayList<>();

int[] ind = indices.toArray();
for (int i = 0; i < ind.length; ++i) {
int idx = ind[i];
var box = boxesArray[idx];
Imgproc.rectangle(frame, box.tl(), box.br(), new Scalar(0, 0, 255), 2);

var name = String.format("%s (%f)", coco_names.get(clsIds.get(idx)), confs.get(idx));

Imgproc.putText(
frame,
name,
new Point(box.x + box.width / 2.0, box.y + box.height / 2.0),
0,
0.6,
ColorHelper.colorToScalar(java.awt.Color.white),
2);

targetList.add(
new TrackedTarget(
box,
clsIds.get(idx),
confs.get(idx),
new TargetCalculationParameters(
false, null, null, null, null, frameStaticProperties)));
}

// ======================

var fpsResult = calculateFPSPipe.run(null);
var fps = fpsResult.output;

return new CVPipelineResult(sumPipeNanosElapsed, fps, targetList, input_frame);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
* Copyright (C) Photon Vision.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

package org.photonvision.vision.pipeline;

public class DnnPipelineSettings extends CVPipelineSettings {
public DnnPipelineSettings() {
this.pipelineType = PipelineType.Dnn;
}
}
Loading

0 comments on commit 80078db

Please sign in to comment.