Hack in aruco-nano

photon-core/build.gradle

@@ -55,6 +55,10 @@ dependencies {
     }
 
     testImplementation group: 'org.junit-pioneer' , name: 'junit-pioneer', version: '2.2.0'
+
+    def arucoVer = "dev-v2024.0.1-15-g0d7210c"
+    implementation "org.photonvision:photonaruconano-jni:$arucoVer:$jniPlatform"
+    implementation "org.photonvision:photonaruconano-java:$arucoVer"
 }
 
 task writeCurrentVersion {

photon-core/src/main/java/org/photonvision/jni/ArucoNanoDetectorJNI.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) Photon Vision.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.photonvision.jni;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.photonvision.ArucoNanoV5Detector;
+import org.photonvision.ArucoNanoV5Detector.DetectionResult;
+import org.photonvision.common.util.TestUtils;
+import org.photonvision.vision.aruco.ArucoDetectionResult;
+import org.photonvision.vision.opencv.CVMat;
+
+public class ArucoNanoDetectorJNI extends PhotonJNICommon {
+    private boolean isLoaded;
+    private static ArucoNanoDetectorJNI instance = null;
+
+    private ArucoNanoDetectorJNI() {
+        isLoaded = false;
+    }
+
+    public static ArucoNanoDetectorJNI getInstance() {
+        if (instance == null) instance = new ArucoNanoDetectorJNI();
+
+        return instance;
+    }
+
+    public static synchronized void forceLoad() throws IOException {
+        var nativeLibName = System.mapLibraryName("photonaruconanojni");
+        extractAndLoad(ArucoNanoDetectorJNI.class, "/windows/x86-64/"+nativeLibName,nativeLibName);
+    }
+
+    @Override
+    public boolean isLoaded() {
+        return isLoaded;
+    }
+
+    @Override
+    public void setLoaded(boolean state) {
+        isLoaded = state;
+    }
+
+    public static List<ArucoDetectionResult> detect(CVMat in) {
+        DetectionResult[] ret = ArucoNanoV5Detector.detect(in.getMat().getNativeObjAddr(), 0);
+
+        return List.of(ret).stream()
+                .map(it -> new ArucoDetectionResult(it.xCorners, it.yCorners, it.id))
+                .collect(Collectors.toList());
+    }
+
+    public static void main(String[] args) throws IOException {
+        TestUtils.loadLibraries();
+        forceLoad();
+    }
+}

photon-core/src/main/java/org/photonvision/jni/PhotonJNICommon.java

@@ -42,39 +42,47 @@ protected static synchronized void forceLoad(
                 // We always extract the shared object (we could hash each so, but that's a lot of work)
                 var arch_name = Platform.getNativeLibraryFolderName();
                 var nativeLibName = System.mapLibraryName(libraryName);
-                var in = clazz.getResourceAsStream("/nativelibraries/" + arch_name + "/" + nativeLibName);
+                boolean success = extractAndLoad(clazz, "/nativelibraries/" + arch_name + "/" + nativeLibName, nativeLibName);
 
-                if (in == null) {
-                    instance.setLoaded(false);
-                    return;
+                if (!success) {
+                    logger.error("Could not load shared object " + libraryName);
+                    break;
                 }
-
-                // It's important that we don't mangle the names of these files on Windows at least
-                File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
-                FileOutputStream fos = new FileOutputStream(temp);
-
-                int read = -1;
-                byte[] buffer = new byte[1024];
-                while ((read = in.read(buffer)) != -1) {
-                    fos.write(buffer, 0, read);
-                }
-                fos.close();
-                in.close();
-
-                System.load(temp.getAbsolutePath());
-
-                logger.info("Successfully loaded shared object " + temp.getName());
-
-            } catch (UnsatisfiedLinkError e) {
+            } catch (UnsatisfiedLinkError | IOException e) {
                 logger.error("Couldn't load shared object " + libraryName, e);
                 e.printStackTrace();
-                // logger.error(System.getProperty("java.library.path"));
                 break;
             }
         }
         instance.setLoaded(true);
     }
 
+    protected static boolean extractAndLoad(Class clazz, String path, String nativeLibName) throws IOException {
+        if (logger == null) logger = new Logger(clazz, LogGroup.Camera);
+        var in = clazz.getResourceAsStream(path);
+
+        if (in == null) {
+            return false;
+        }
+
+        // It's important that we don't mangle the names of these files on Windows at least
+        File temp = new File(System.getProperty("java.io.tmpdir"), nativeLibName);
+        FileOutputStream fos = new FileOutputStream(temp);
+
+        int read = -1;
+        byte[] buffer = new byte[1024];
+        while ((read = in.read(buffer)) != -1) {
+            fos.write(buffer, 0, read);
+        }
+        fos.close();
+        in.close();
+
+        System.load(temp.getAbsolutePath());
+
+        logger.info("Successfully loaded shared object " + temp.getName());
+        return true;
+    }
+
     protected static synchronized void forceLoad(
             PhotonJNICommon instance, Class<?> clazz, String libraryName) throws IOException {
         forceLoad(instance, clazz, List.of(libraryName));

photon-core/src/main/java/org/photonvision/vision/pipe/impl/ArucoDetectionPipe.java

@@ -26,6 +26,7 @@
 import org.opencv.core.TermCriteria;
 import org.opencv.imgproc.Imgproc;
 import org.opencv.objdetect.Objdetect;
+import org.photonvision.jni.ArucoNanoDetectorJNI;
 import org.photonvision.vision.aruco.ArucoDetectionResult;
 import org.photonvision.vision.aruco.PhotonArucoDetector;
 import org.photonvision.vision.opencv.CVMat;
@@ -43,6 +44,8 @@ public class ArucoDetectionPipe
 
     @Override
     protected List<ArucoDetectionResult> process(CVMat in) {
+        if (in.getMat().empty()) return List.of();
+
         var imgMat = in.getMat();
 
         // Sanity check -- image should not be empty
@@ -51,8 +54,10 @@ protected List<ArucoDetectionResult> process(CVMat in) {
             return List.of();
         }
 
-        var detections = photonDetector.detect(imgMat);
-        // manually do corner refinement ourselves
+        // var detections = photonDetector.detect(imgMat);
+        var detections = ArucoNanoDetectorJNI.detect(in);
+
+        // manually do corner refinement ourselves (todo do we have to with aruco-nano?)
         if (params.useCornerRefinement) {
             for (var detection : detections) {
                 double[] xCorners = detection.getXCorners();
@@ -93,7 +98,9 @@ protected List<ArucoDetectionResult> process(CVMat in) {
                 }
             }
         }
-        return List.of(detections);
+
+        // return List.of(detections);
+        return (detections);
     }
 
     @Override

photon-core/src/test/java/org/photonvision/vision/pipeline/ArucoPipelineTest.java

@@ -18,12 +18,16 @@
 package org.photonvision.vision.pipeline;
 
 import edu.wpi.first.math.geometry.Translation3d;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.photonvision.common.configuration.ConfigManager;
 import org.photonvision.common.util.TestUtils;
+import org.photonvision.jni.ArucoNanoDetectorJNI;
 import org.photonvision.vision.apriltag.AprilTagFamily;
 import org.photonvision.vision.camera.QuirkyCamera;
 import org.photonvision.vision.frame.provider.FileFrameProvider;
@@ -36,6 +40,7 @@ public class ArucoPipelineTest {
     public void setup() {
         TestUtils.loadLibraries();
         ConfigManager.getInstance().load();
+        assertDoesNotThrow(ArucoNanoDetectorJNI::forceLoad);
     }
 
     @Test

photon-server/src/main/java/org/photonvision/Main.java

@@ -38,6 +38,7 @@
 import org.photonvision.common.networking.NetworkManager;
 import org.photonvision.common.util.TestUtils;
 import org.photonvision.common.util.numbers.IntegerCouple;
+import org.photonvision.jni.ArucoNanoDetectorJNI;
 import org.photonvision.jni.RknnDetectorJNI;
 import org.photonvision.mrcal.MrCalJNILoader;
 import org.photonvision.raspi.LibCameraJNILoader;
@@ -343,6 +344,13 @@ public static void main(String[] args) {
             logger.error("Failed to load native libraries!", e);
         }
 
+        try {
+            ArucoNanoDetectorJNI.forceLoad();
+            logger.info("Loaded aruco nano JNI");
+        } catch (Exception e) {
+            logger.error("Failed to load native libraries!", e);
+        }
+
         try {
             if (Platform.isRaspberryPi()) {
                 LibCameraJNILoader.forceLoad();