Hardware components | ||||||
 |
| × | 1 | |||
| × | 1 | ||||
The goal of this project is to demonstrate that a single programmable-zoom camera can be utilized to generate a 3D depth map of a scene for use in applications such as augmented reality. It is not necessary to utilize a stereo camera system or structure from motion.
This project leverages an Arducam IMX477 motorized focus camera attached to a Jetson Orin Nano to generate a 3D depth map. The focus is adjusted from the minimum (0) to the maximum (1000), and a focus metric is computed in a neighborhood around each pixel for each captured image.
Image at focus level 50
Focus metric at focus level 50
Image at focus level 1000
Focus metric at focus level 1000
For each pixel, the image index with the highest focus metric is selected as the proper depth for that pixel. In the resulting depth map image, the higher the value the closer the object is to the camera.
Depth map image
# encoding: UTF-8
'''
Author: Douglas Turner
Date: 4/1/24
'''
import cv2 #sudo apt-get install python-opencv
import numpy as py
import argparse
from JetsonCamera import Camera
from Focuser import Focuser
def laplacian(img):
img_gray = cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)
img_sobel = cv2.Laplacian(img_gray,cv2.CV_16U)
return img_sobel
def parse_cmdline():
parser = argparse.ArgumentParser(description='Arducam Controller.')
parser.add_argument('-i', '--i2c-bus', type=int, nargs=None, required=True,
help='Set i2c bus, for A02 is 6, for B01 is 7 or 8, for Jetson Xavier NX it is 9 and 10.')
return parser.parse_args()
def calcDepthMap(camera, i2c_bus):
focuser = Focuser(i2c_bus)
focus = 0
maxValues = py.zeros((360,640))
maxIndex = py.zeros((360,640))
focusIndex = 0
while (focus <= 1000):
# Set focus
focuser.set(Focuser.OPT_FOCUS,focus)
# Get image and smooth image
currImg = camera.getFrame()
currImgBlur = cv2.GaussianBlur(currImg,(5,5),0)
shp = currImg.shape
# Calculate laplacian
sImg = laplacian(currImgBlur)
# Calculate average value over neighborhood for each pixel
winSize=5
minFocusValue=1
values = py.zeros(shp)
for i in range(winSize, shp[0]-winSize):
for j in range(winSize, shp[1]-winSize):
value = cv2.mean(sImg[i-winSize:i+winSize+1,j-winSize:j+winSize+1])[0]
values[i,j] = value
if value > minFocusValue and value > maxValues[i,j]:
maxValues[i,j] = value
maxIndex[i,j] = focusIndex
cv2.imwrite("focusImage{}.jpg".format(focus), 10*values)
cv2.imwrite("currImage{}.jpg".format(focus), currImg)
focus = focus + 50
focusIndex = focusIndex + 1
# Smooth max index image
maxIndexBlur = cv2.GaussianBlur(maxIndex,(5,5),0)
cv2.imwrite("maxIndex.jpg", 10*maxIndexBlur)
def main():
args = parse_cmdline()
#open camera
camera = Camera()
#open camera preview
camera.start_preview()
print(args.i2c_bus)
calcDepthMap(camera, args.i2c_bus)
camera.stop_preview()
camera.close()
if __name__ == "__main__":
main()
# MIT License
# Copyright (c) 2019 JetsonHacks
# See license
# Using a CSI camera (such as the Raspberry Pi Version 2) connected to a
# NVIDIA Jetson Nano Developer Kit using OpenCV
# Drivers for the camera and OpenCV are included in the base image
import cv2
import time
try:
from Queue import Queue
except ModuleNotFoundError:
from queue import Queue
import threading
import signal
import sys
# def signal_handler(sig, frame):
# print('You pressed Ctrl+C!')
# sys.exit(0)
# signal.signal(signal.SIGINT, signal_handler)
def gstreamer_pipeline(
capture_width=1280,
capture_height=720,
display_width=640,
display_height=360,
framerate=60,
flip_method=0,
):
return (
"nvarguscamerasrc ! "
"video/x-raw(memory:NVMM), "
"width=(int)%d, height=(int)%d, "
"format=(string)NV12, framerate=(fraction)%d/1 ! "
"nvvidconv flip-method=%d ! "
"video/x-raw, width=(int)%d, height=(int)%d, format=(string)BGRx ! "
"videoconvert ! "
"video/x-raw, format=(string)BGR ! appsink"
% (
capture_width,
capture_height,
framerate,
flip_method,
display_width,
display_height,
)
)
class FrameReader(threading.Thread):
queues = []
_running = True
camera = None
def __init__(self, camera, name):
threading.Thread.__init__(self)
self.name = name
self.camera = camera
def run(self):
while self._running:
_, frame = self.camera.read()
while self.queues:
queue = self.queues.pop()
queue.put(frame)
def addQueue(self, queue):
self.queues.append(queue)
def getFrame(self, timeout = None):
queue = Queue(1)
self.addQueue(queue)
return queue.get(timeout = timeout)
def stop(self):
self._running = False
class Previewer(threading.Thread):
window_name = "Arducam"
_running = True
camera = None
def __init__(self, camera, name):
threading.Thread.__init__(self)
self.name = name
self.camera = camera
def run(self):
self._running = True
while self._running:
cv2.imshow(self.window_name, self.camera.getFrame(2000))
keyCode = cv2.waitKey(16) & 0xFF
cv2.destroyWindow(self.window_name)
def start_preview(self):
self.start()
def stop_preview(self):
self._running = False
class Camera(object):
frame_reader = None
cap = None
previewer = None
def __init__(self):
self.open_camera()
def open_camera(self):
self.cap = cv2.VideoCapture(gstreamer_pipeline(flip_method=0), cv2.CAP_GSTREAMER)
if not self.cap.isOpened():
raise RuntimeError("Failed to open camera!")
if self.frame_reader == None:
self.frame_reader = FrameReader(self.cap, "")
self.frame_reader.daemon = True
self.frame_reader.start()
self.previewer = Previewer(self.frame_reader, "")
def getFrame(self):
return self.frame_reader.getFrame()
def start_preview(self):
self.previewer.daemon = True
self.previewer.start_preview()
def stop_preview(self):
self.previewer.stop_preview()
self.previewer.join()
def close(self):
self.frame_reader.stop()
self.cap.release()
if __name__ == "__main__":
camera = Camera()
camera.start_preview()
time.sleep(10)
camera.stop_preview()
camera.close()
'''
Arducam programable zoom-lens control component.
Copyright (c) 2019-4 Arducam <http://www.arducam.com>.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
OR OTHER DEALINGS IN THE SOFTWARE.
'''
import sys
import time
import os
class Focuser:
bus = None
CHIP_I2C_ADDR = 0x0C
def __init__(self, bus):
self.focus_value = 0
self.bus = bus
pass
def read(self):
return self.focus_value
def write(self, chip_addr, value):
if value < 0:
value = 0
self.focus_value = value
value = (value << 4) & 0x3ff0
data1 = (value >> 8) & 0x3f
data2 = value & 0xf0
os.system("i2cset -y {} 0x{:02X} {} {}".format(self.bus, chip_addr, data1, data2))
OPT_BASE = 0x1000
OPT_FOCUS = OPT_BASE | 0x01
OPT_ZOOM = OPT_BASE | 0x02
OPT_MOTOR_X = OPT_BASE | 0x03
OPT_MOTOR_Y = OPT_BASE | 0x04
OPT_IRCUT = OPT_BASE | 0x05
opts = {
OPT_FOCUS : {
"MIN_VALUE": 0,
"MAX_VALUE": 1000,
"DEF_VALUE": 0,
},
}
def reset(self,opt,flag = 1):
info = self.opts[opt]
if info == None or info["DEF_VALUE"] == None:
return
self.set(opt,info["DEF_VALUE"])
def get(self,opt,flag = 0):
info = self.opts[opt]
return self.read()
def set(self,opt,value,flag = 1):
info = self.opts[opt]
if value > info["MAX_VALUE"]:
value = info["MAX_VALUE"]
elif value < info["MIN_VALUE"]:
value = info["MIN_VALUE"]
self.write(self.CHIP_I2C_ADDR, value)
print("write: {}".format(value))
pass
def test():
focuser = Focuser(7)
focuser.set(Focuser.OPT_FOCUS, 0)
time.sleep(3)
focuser.set(Focuser.OPT_FOCUS, 1000)
time.sleep(3)
focuser.reset(Focuser.OPT_FOCUS)
if __name__ == "__main__":
test()
Thanks to Arducam.
Comments