ME 461 Final Project: Robot Car with Sound/Color Detection

Robot car that uses a microphone array and a camera to detect sound direction, distinguish between sounds and track colors

IntermediateShowcase (no instructions)186

ME 461 Final Project: Robot Car with Sound/Color Detection

Things used in this project

Hardware components

Texas Instruments LAUNCHXL-F28379D C2000 Delfino LaunchPad

Raspberry Pi 4 Model B

ReSpeaker USB Mic Array

ELP 720P CMOS OV9712 SENSOR MJPEG YUY2 DUAL LENS STEREO USB CAMERA

Story

Our project utilizes the TI F28379D Launchpad as well as a Raspberry Pi to control a robot car using sound and color recognition. The Raspberry Pi connects to a ReSpeaker microphone array and a stereo camera by USB. The information captured by these devices are read by the Raspberry Pi and then sent to the F28479D through UART.

A microphone on the circuit board the F28379D is connected to listens for sound and uses an fft to find the frequency of the sound. Using the frequency of the sound, different cases are accessed in the code. These cases utilize the information sent from the Raspberry Pi to determine the behavior of the car. Cases include follow sound, go away from sound, spin in place right, spin in place left, detect green, detect, orange, and detect red.

The cases for color detection will also send a float over UART back to the Raspberry Pi to change the values the camera detects.

#!/usr/bin/env python3

import numpy as np
import matplotlib.pyplot as plt
import serial
from time import sleep
import struct
import cv2
from tuning import Tuning
import usb.core
import usb.util
import time
h_min = 45
h_max = 106
s_min = 52
s_max = 255
v_min = 47
v_max = 255 #this is the default values for finding green

# initialize the serial port
ser = serial.Serial ("/dev/ttyAMA1", 115200)    #Open port with baud rate
dev = usb.core.find(idVendor=0x2886, idProduct=0x0018)
x = 0
y = 0
case = 0
trash = 0
buffer = [0.0,0.0,0.0,0.0]
def empty(a):
    pass

cap = cv2.VideoCapture(0)

params = cv2.SimpleBlobDetector_Params()
# Change thresholds
# params.minThreshold = 10
# params.maxThreshold = 200

# Filter by Area.
params.filterByArea = True
params.minArea = 100
params.maxArea = 20000

# Filter by Circularity
params.filterByCircularity = False
params.minCircularity = 0.1

# Filter by Convexity
params.filterByConvexity = False
params.minConvexity = 0.87

# Filter by Inertia
params.filterByInertia = False
params.minInertiaRatio = 0.01

# Create a detector with the parameters
ver = (cv2.__version__).split('.')
if int(ver[0]) < 3 :
    detector = cv2.SimpleBlobDetector(params)
else:
    detector = cv2.SimpleBlobDetector_create(params)

while(1):
    ret, frame = cap.read()
    imgHSV = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
    if case == (0.0,): # case for looing for green
        h_min = 45
        h_max = 106
        s_min = 52
        s_max = 255
        v_min = 47
        v_max = 255
    if case == (1.0,): #case for red
        h_min = 0
        h_max = 8
        s_min = 150
        s_max = 255
        v_min = 43
        v_max = 255
    if case ==(2.0,): #case for orange
        h_min = 10
        h_max = 25
        s_min = 100
        s_max = 255
        v_min = 100
        v_max = 255
    # print(h_min,h_max,s_min,s_max,v_min,v_max)
    lower = np.array([h_min,s_min,v_min])
    upper = np.array([h_max,s_max,v_max])
    mask = cv2.inRange(imgHSV,lower,upper)
    # cv2.imshow("Original",frame)
    # cv2.imshow("Mask", mask)
    # cv2.waitKey(1)
    # Detect blobs.
    keypoints = detector.detect(mask)
    #i = 0
    if dev:
        Mic_tuning = Tuning(dev)
    for point in keypoints:
        x = point.pt[0]
        y = point.pt[1]
        size = point.size
        print(x,y,size)
    ser.write(str.encode('*'))
    ser.write(str.encode('*'))
    ser.write(struct.pack('fff',x,y,float(Mic_tuning.direction))) # sending info
    ser.write(str.encode('!'))
    ser.write(str.encode('!')) # start to recieve info
    case = struct.unpack('f',ser.read(4))



# # Detect blobs.
# keypoints = detector.detect(im)
# print(keypoints)
# im_with_keypoints = cv2.drawKeypoints(im, keypoints, np.array([]), (0,0,255), cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
# cv2.imwrite("result.png",im_with_keypoints)
# plt.imshow(im_with_keypoints)
# plt.show()