OAK ColorCamera ISP vs Video vs Preview - For Full FOV tutorial

isp_video_preview.py

#!/usr/bin/env python3

import cv2
import depthai as dai
import math

class BoundingBox:
    xmin: float
    ymin: float
    xmax: float
    ymax: float

    def __init__(self, bb = None):
        if bb is not None:
            if isinstance(bb, tuple):
                self.xmin = bb[0]
                self.ymin = bb[1]
                self.xmax = bb[2]
                self.ymax = bb[3]
            else: # object
                self.xmin = bb.xmin
                self.ymin = bb.ymin
                self.xmax = bb.xmax
                self.ymax = bb.ymax

            self.xdelta = self.xmax - self.xmin
            self.ydelta = self.ymax - self.ymin
    
    def calculate_new_bb(self, bb):
        bb = BoundingBox(bb)
        return BoundingBox((
            self.xmin + self.xdelta * bb.xmin,
            self.ymin + self.ydelta * bb.ymin,
            self.xmin + self.xdelta * bb.xmax,
            self.ymin + self.ydelta * bb.ymax,
        ))

    def normalize_to_frame(self, frame):
        return (
            (int(frame.shape[1] * self.xmin), int(frame.shape[0] * self.ymin)),
            (int(frame.shape[1] * self.xmax), int(frame.shape[0] * self.ymax))
            )


# We send ISP frame (4032x3040) to the device, but use 4k video frame (3840x2160) in the pipeline (crop/detection)
isp_to_video = BoundingBox((0.0238, 0.145, 0.9762, 0.855))

# 4k (16:9) to 400x400 (1:1)
video_to_preview = isp_to_video.calculate_new_bb((0.21875, 0, 0.78125, 1))

# Create pipeline
pipeline = dai.Pipeline()

# Define source and output
camRgb = pipeline.createColorCamera()

# Properties
camRgb.setBoardSocket(dai.CameraBoardSocket.RGB)
camRgb.setResolution(dai.ColorCameraProperties.SensorResolution.THE_12_MP)

camRgb.setPreviewSize(400,400) # 1:1 aspect ratio

# Linking
xoutIsp = pipeline.createXLinkOut()
xoutIsp.setStreamName("isp")
camRgb.isp.link(xoutIsp.input)

xoutVideo = pipeline.createXLinkOut()
xoutVideo.setStreamName('video')
camRgb.video.link(xoutVideo.input)

xoutPreview = pipeline.createXLinkOut()
xoutPreview.setStreamName('preview')
camRgb.preview.link(xoutPreview.input)

# Connect to device and start pipeline
with dai.Device(pipeline) as device:

    queues = {}

    qIsp = device.getOutputQueue('isp')
    qVideo = device.getOutputQueue('video')
    qPreview = device.getOutputQueue('preview')

    while True:
        if qIsp.has():
            isp = qIsp.get().getCvFrame()
            # 4x downscale so it fits the 1080P screen
            isp = cv2.pyrDown(isp)
            isp = cv2.pyrDown(isp)
            
            topLeft, bottomRight = isp_to_video.normalize_to_frame(isp)
            isp = cv2.rectangle(isp, topLeft, bottomRight, (200,100,0), 3)
            
            topLeft, bottomRight = video_to_preview.normalize_to_frame(isp)
            isp = cv2.rectangle(isp, topLeft, bottomRight, (0,200,200), 3)
            
            cv2.imshow('ISP - 12MP IMX378', isp)

        if qVideo.has():
            video = qVideo.get().getCvFrame()
            # 4x downscale so it fits the 1080P screen
            video = cv2.pyrDown(video)
            video = cv2.pyrDown(video)
            cv2.imshow('Video (cropped to 4k)', video)

        if qPreview.has():
            preview = qPreview.get().getCvFrame()
            cv2.imshow('Preview (cropped to fit 1:1)', preview)
        
                

        if cv2.waitKey(1) == ord('q'):
            break