Merge pull request #1064 from luxonis/develop

DepthAI SDK 1.12.0

depthai_sdk/docs/source/conf.py

@@ -22,7 +22,7 @@
 author = 'Luxonis'
 
 # The full version, including alpha/beta/rc tags
-release = '1.11.0'
+release = '1.12.0'
 
 
 # -- General configuration ---------------------------------------------------

depthai_sdk/examples/CameraComponent/camera_control_with_nn.py

@@ -0,0 +1,10 @@
+from depthai_sdk import OakCamera
+
+with OakCamera() as oak:
+    color = oak.create_camera('color')
+    face_det = oak.create_nn('face-detection-retail-0004', color)
+    # Control the camera's exposure/focus based on the (largest) detected face
+    color.control_with_nn(face_det, auto_focus=True, auto_exposure=True, debug=False)
+
+    oak.visualize(face_det, fps=True)
+    oak.start(blocking=True)

depthai_sdk/examples/StereoComponent/stereo_encoded.py

@@ -6,7 +6,7 @@
     stereo = oak.create_stereo('800p', fps=30, encode='h264')
 
     # Set on-device output colorization, works only for encoded output
-    stereo.set_colormap(dai.Colormap.STEREO_JET)
+    stereo.set_colormap(dai.Colormap.JET)
 
     oak.visualize(stereo.out.encoded, fps=True)
     oak.start(blocking=True)

depthai_sdk/examples/trigger_action/custom_trigger.py

@@ -27,7 +27,7 @@ def my_condition(packet) -> bool:
 with OakCamera() as oak:
     color = oak.create_camera('color', fps=30)
     stereo = oak.create_stereo('800p')
-    stereo.config_stereo(align='color')
+    stereo.config_stereo(align=color)
 
     trigger = Trigger(input=stereo.out.depth, condition=my_condition, cooldown=30)
     action = RecordAction(

depthai_sdk/requirements.txt

@@ -4,7 +4,7 @@ opencv-contrib-python>4
 blobconverter>=1.4.1
 pytube>=12.1.0
 --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/
-depthai==2.21.2
+depthai==2.22.0
 PyTurboJPEG==1.6.4
 marshmallow==3.17.0
 xmltodict

depthai_sdk/sdk_tests/components/nn/test_nn_component.py

@@ -91,3 +91,15 @@ def callback(packet):
             if not oak_camera.poll():
                 raise RuntimeError('Polling failed')
             time.sleep(0.1)
+
+
+def test_encoded_output():
+    with OakCamera() as oak_camera:
+        camera = oak_camera.create_camera('color', '1080p', encode='h264')
+
+        oak_camera.callback(camera.out.encoded, lambda x: print(x))
+        oak_camera.start(blocking=False)
+
+        for i in range(10):
+            oak_camera.poll()
+            time.sleep(0.1)

depthai_sdk/sdk_tests/components/stereo/test_stereo_component.py

@@ -1,19 +1,21 @@
 import time
 
+import depthai as dai
 import pytest
+
 from depthai_sdk.oak_camera import OakCamera
-import depthai as dai
 
 
 def test_stereo_output():
     with OakCamera() as oak_camera:
         if dai.CameraBoardSocket.LEFT not in oak_camera.sensors:
             pytest.skip('Looks like camera does not have mono pair, skipping...')
         else:
-            stereo = oak_camera.create_stereo('400p')
+            stereo = oak_camera.create_stereo('800p', encode='h264')
 
             oak_camera.callback([stereo.out.depth, stereo.out.disparity,
-                                 stereo.out.rectified_left, stereo.out.rectified_right], callback=lambda x: None)
+                                 stereo.out.rectified_left, stereo.out.rectified_right,
+                                 stereo.out.encoded], callback=lambda x: None)
             oak_camera.start(blocking=False)
 
             for i in range(10):

depthai_sdk/sdk_tests/test_examples.py

@@ -1,27 +1,38 @@
+import os
 import subprocess
 import sys
 import time
 from pathlib import Path
 
 import cv2
+import pytest
 
-EXAMPLES_DIR = Path(__file__).parent.parent.parent / "examples"
+EXAMPLES_DIR = Path(__file__).parents[1] / 'examples'
 
+# Create a temporary directory for the tests
+Path('/tmp/depthai_sdk_tests').mkdir(exist_ok=True)
+os.chdir('/tmp/depthai_sdk_tests')
 
-def test_examples():
-    python_executable = Path(sys.executable)
-    for example in EXAMPLES_DIR.rglob("**/*.py"):
-        print(f"Running example: {example.name}")
-
-        result = subprocess.Popen(f"{python_executable} {example}", stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                  env={"DISPLAY": ""}, shell=True)
-
-        time.sleep(5)
-        result.kill()
-        time.sleep(5)
-        print('Stderr: ', result.stderr.read().decode())
 
-        # if result.returncode and result.returncode != 0:
-        #     assert False, f"{example} raised an exception: {result.stderr}"
-
-        cv2.destroyAllWindows()
+@pytest.mark.parametrize('example', list(EXAMPLES_DIR.rglob("**/*.py")))
+def test_examples(example):
+    print(f"Running {example}")
+    python_executable = Path(sys.executable)
+    result = subprocess.Popen(f"{python_executable} {example}",
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE,
+                              env={
+                                  'DISPLAY': '',
+                                  'PYTHONPATH': f'{os.environ["PYTHONPATH"]}:{EXAMPLES_DIR.parent}'
+                              },
+                              shell=True)
+
+    time.sleep(5)
+    result.kill()
+    time.sleep(5)
+    print('Stderr: ', result.stderr.read().decode())
+
+    if result.returncode and result.returncode != 0:
+        assert False, f"{example} raised an exception: {result.stderr}"
+
+    cv2.destroyAllWindows()

depthai_sdk/setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name='depthai-sdk',
-    version='1.11.0',
+    version='1.12.0',
     description='This package provides an abstraction of the DepthAI API library.',
     long_description=io.open("README.md", encoding="utf-8").read(),
     long_description_content_type="text/markdown",
@@ -30,7 +30,8 @@
         "replay": ['mcap>=0.0.10',
                    'mcap-ros1-support==0.0.8',
                    'rosbags==0.9.11'],
-        "record": ['av']
+        "record": ['av'],
+        "test": ['pytest']
     },
     project_urls={
         "Bug Tracker": "https://github.com/luxonis/depthai/issues",

depthai_sdk/src/depthai_sdk/__init__.py

@@ -1,5 +1,6 @@
 from depthai_sdk.args_parser import ArgsParser
 from depthai_sdk.classes.enum import ResizeMode
+from depthai_sdk.constants import CV2_HAS_GUI_SUPPORT
 from depthai_sdk.logger import set_logging_level
 from depthai_sdk.oak_camera import OakCamera
 from depthai_sdk.oak_device import OakDevice
@@ -10,7 +11,7 @@
 from depthai_sdk.utils import _create_config, get_config_field
 from depthai_sdk.visualize import *
 
-__version__ = '1.11.0'
+__version__ = '1.12.0'
 
 
 def __import_sentry(sentry_dsn: str) -> None:

depthai_sdk/src/depthai_sdk/classes/output_config.py

@@ -16,10 +16,21 @@
 from depthai_sdk.trigger_action.triggers.abstract_trigger import Trigger
 from depthai_sdk.visualize.visualizer import Visualizer
 
+def find_new_name(name: str, names: List[str]):
+    while True:
+        arr = name.split(' ')
+        num = arr[-1]
+        if num.isnumeric():
+            arr[-1] = str(int(num) + 1)
+            name = " ".join(arr)
+        else:
+            name = f"{name} 2"
+        if name not in names:
+            return name
 
 class BaseConfig:
     @abstractmethod
-    def setup(self, pipeline: dai.Pipeline, device, names: List[str]) -> List[XoutBase]:
+    def setup(self, pipeline: dai.Pipeline, device: dai.Device, names: List[str]) -> List[XoutBase]:
         raise NotImplementedError()
 
 
@@ -39,24 +50,12 @@ def __init__(self, output: Callable,
         self.visualizer_enabled = visualizer_enabled
         self.record_path = record_path
 
-    def find_new_name(self, name: str, names: List[str]):
-        while True:
-            arr = name.split(' ')
-            num = arr[-1]
-            if num.isnumeric():
-                arr[-1] = str(int(num) + 1)
-                name = " ".join(arr)
-            else:
-                name = f"{name} 2"
-            if name not in names:
-                return name
-
     def setup(self, pipeline: dai.Pipeline, device, names: List[str]) -> List[XoutBase]:
         xoutbase: XoutBase = self.output(pipeline, device)
         xoutbase.setup_base(self.callback)
 
         if xoutbase.name in names:  # Stream name already exist, append a number to it
-            xoutbase.name = self.find_new_name(xoutbase.name, names)
+            xoutbase.name = find_new_name(xoutbase.name, names)
         names.append(xoutbase.name)
 
         recorder = None

depthai_sdk/src/depthai_sdk/classes/packets.py

@@ -39,6 +39,7 @@ class _TrackingDetection(_Detection):
 class _TwoStageDetection(_Detection):
     nn_data: dai.NNData
 
+
 class NNDataPacket:
     """
     Contains only dai.NNData message
@@ -50,6 +51,7 @@ def __init__(self, name: str, nn_data: dai.NNData):
         self.name = name
         self.msg = nn_data
 
+
 class FramePacket:
     """
     Contains only dai.ImgFrame message and cv2 frame, which is used by visualization logic.
@@ -80,6 +82,7 @@ def __init__(self,
         self.color_frame = color_frame
         self.visualizer = visualizer
 
+
 class DepthPacket(FramePacket):
     mono_frame: dai.ImgFrame
 

depthai_sdk/src/depthai_sdk/components/camera_component.py

@@ -46,6 +46,7 @@ def __init__(self,
         """
         super().__init__()
         self.out = self.Out(self)
+        self._pipeline = pipeline
 
         self.node: Optional[Union[dai.node.ColorCamera, dai.node.MonoCamera, dai.node.XLinkIn]] = None
         self.encoder: Optional[dai.node.VideoEncoder] = None
@@ -279,6 +280,40 @@ def _config_camera_args(self, args: Dict):
         else:  # Replay
             self.config_camera(fps=args.get('fps', None))
 
+    def control_with_nn(self, detection_component: 'NNComponent', auto_focus=True, auto_exposure=True, debug=False):
+        """
+        Control the camera AF/AE/AWB based on the object detection results.
+
+        :param detection_component: NNComponent that will be used to control the camera
+        :param auto_focus: Enable auto focus to the object
+        :param auto_exposure: Enable auto exposure to the object
+        :param auto_white_balance: auto white balance to the object
+        """
+
+        if not auto_focus and not auto_exposure:
+            logging.error(
+                'Attempted to control camera with NN, but both Auto-Focus and Auto-Exposure were disabled! Attempt ignored.'
+            )
+            return
+        if 'NNComponent' not in str(type(detection_component)):
+            raise ValueError('nn_component must be an instance of NNComponent!')
+        if not detection_component._is_detector():
+            raise ValueError('nn_component must be a object detection model (YOLO/MobileNetSSD based)!')
+
+        from depthai_sdk.components.control_camera_with_nn import control_camera_with_nn
+
+        control_camera_with_nn(
+            pipeline=self._pipeline,
+            camera_control=self.node.inputControl,
+            nn_output=detection_component.node.out,
+            resize_mode=detection_component._ar_resize_mode,
+            resolution=self.node.getResolution(),
+            nn_size = detection_component._size,
+            af=auto_focus,
+            ae=auto_exposure,
+            debug=debug
+        )
+
     def config_color_camera(self,
                             interleaved: Optional[bool] = None,
                             color_order: Union[None, dai.ColorCameraProperties.ColorOrder, str] = None,

depthai_sdk/src/depthai_sdk/components/camera_helper.py

@@ -179,6 +179,18 @@ def get_res(resolutions: Dict[Any, Tuple[int,int]]):
     else:
         raise Exception('Camera sensor type unknown!', type)
 
+def get_resolution_size(
+        resolution: Union[
+            dai.ColorCameraProperties.SensorResolution,
+            dai.MonoCameraProperties.SensorResolution
+            ]) -> Tuple[int,int]:
+    if resolution in colorResolutions:
+        return colorResolutions[resolution]
+    elif resolution in monoResolutions:
+        return monoResolutions[resolution]
+    else:
+        raise Exception('Camera sensor resolution unknown!', resolution)
+
 def getClosesResolution(sensor: dai.CameraFeatures,
                         type: dai.CameraSensorType,
                         width: Optional[int] = None,

depthai_sdk/src/depthai_sdk/components/control_camera_with_nn.py

@@ -0,0 +1,69 @@
+import depthai as dai
+from depthai_sdk.classes.enum import ResizeMode
+from typing import Union, Tuple
+from depthai_sdk.components.camera_helper import get_resolution_size
+from pathlib import Path
+from string import Template
+
+def control_camera_with_nn(
+        pipeline: dai.Pipeline,
+        camera_control: dai.Node.Input,
+        nn_output: dai.Node.Output,
+        resize_mode: ResizeMode,
+        resolution: Union[dai.ColorCameraProperties.SensorResolution, dai.MonoCameraProperties.SensorResolution],
+        nn_size: Tuple[int, int],
+        af: bool,
+        ae: bool,
+        debug: bool
+        ):
+    sensor_resolution = get_resolution_size(resolution)
+    # width / height (old ar)
+    sensor_ar = sensor_resolution[0] / sensor_resolution[1]
+    # NN ar (new ar)
+    nn_ar = nn_size[0] / nn_size[1]
+
+    if resize_mode == ResizeMode.LETTERBOX:
+        padding = (sensor_ar - nn_ar) / 2
+        if padding > 0:
+            init = f"xmin = 0; ymin = {-padding}; xmax = 1; ymax = {1 + padding}"
+        else:
+            init = f"xmin = {padding}; ymin = 0; xmax = {1 - padding}; ymax = 1"
+    elif resize_mode in [ResizeMode.CROP, ResizeMode.FULL_CROP]:
+        cropping = (1 - (nn_ar / sensor_ar)) / 2
+        if cropping < 0:
+            init = f"xmin = 0; ymin = {-cropping}; xmax = 1; ymax = {1 + cropping}"
+        else:
+            init = f"xmin = {cropping}; ymin = 0; xmax = {1 - cropping}; ymax = 1"
+    else: # Stretch
+        init = f"xmin=0; ymin=0; xmax=1; ymax=1"
+
+
+    resize_str = f"new_xmin=xmin+width*det.xmin; new_ymin=ymin+height*det.ymin; new_xmax=xmin+width*det.xmax; new_ymax=ymin+height*det.ymax;"
+    denormalize = f"startx=int(new_xmin*{sensor_resolution[0]}); starty=int(new_ymin*{sensor_resolution[1]}); new_width=int((new_xmax-new_xmin)*{sensor_resolution[0]}); new_height=int((new_ymax-new_ymin)*{sensor_resolution[1]});"
+    control_str = ''
+    if ae:
+        control_str += f"control.setAutoExposureRegion(startx, starty, new_width, new_height);"
+    if af:
+        control_str += f"control.setAutoFocusRegion(startx, starty, new_width, new_height);"
+
+
+    script_node = pipeline.create(dai.node.Script)
+    script_node.setProcessor(dai.ProcessorType.LEON_CSS)  # More stable
+
+    with open(Path(__file__).parent / 'template_control_cam_with_nn.py', 'r') as file:
+        code = Template(file.read()).substitute(
+            DEBUG='' if debug else '#',
+            INIT=init,
+            RESIZE=resize_str,
+            DENORMALIZE=denormalize,
+            CONTROL=control_str
+        )
+        script_node.setScript(code)
+
+        if debug:
+            print(code)
+
+    # Node linking:
+    # NN output -> Script -> Camera input
+    nn_output.link(script_node.inputs['detections'])
+    script_node.outputs['control'].link(camera_control)