This project is built for a my buddy Owen who is quadriplegic and the end goal is to help him control a computer with his eyes. This project is currently in progress and hopes to have a fully functional product in 6 months. As the project progresses, I will be updated story with trials tested on Owen and updated code and hardware.
Blue Target Picture
THIS IS NOT CIRCUIT DIAGRAM. I could not find a place to upload pictures so I decided to upload it here. This picture is needed for the guiAPP.py
main3.py
PythonThis is the main function that runs the calibration function and the gaze tracking function. This is the only function that should be run! OpenCv2, pyautogui, and numpy are also required to run this code.
import time
import cv2
import numpy as np
from eyeFunction import *
import pyautogui as pag
# First, run the GUI for Calibration
print("Running GUI for Calibration...")
time.sleep(1.0)
import guiAPP
# Second, run the calculation to run find the eye Center
print("Calibrating...")
time.sleep(1.0)
import generateSphere3
# Loading pupil loci...
importd = fromCSV('pupilCenter.csv')
dict_Calib_AB = fromCSV('calibrateAB.txt')
screenDimens = fromCSV('screenLocation.txt')
MiddleP = fromCSV('Middle.txt')
# set up the camera and call it eyeCam or worldCam
eyeCam = cv2.VideoCapture(2)
worldCam = cv2.VideoCapture(0)
# give the camera time to warm up
print("Setting up Cameras for use...")
time.sleep(2.0)
# Load the eyeBall center
eyeCenter = fromCSV('eyeCenter.txt')
# begin loop to read in from both cameras and make calculations
while True:
# read in from the cameras and save the frames with variable names 'eye' and 'world'
_, eye = eyeCam.read()
_, world = worldCam.read()
#resizes the frames by some factor because the original is too damn big
scalingFactor = 0.5
eye = cv2.resize(eye,None,fx=scalingFactor, fy=scalingFactor, interpolation = cv2.INTER_LINEAR)
world = cv2.resize(world,None,fx=scalingFactor, fy=scalingFactor, interpolation = cv2.INTER_LINEAR)
Alpha =0
Beta = 0
# Find the pupil and draw the angle on the image
eye, Beta, Alpha = findPupil_drawAngle(eye, eyeCenter,importd)
# Find the location of the realtionship of the pupil to the screen
# rewrite this section with no rotate_points
#print("beta: {} Alpha: {}".format(Beta,Alpha))
newPupx,newPupy = projectiveTransToScreen(Beta-MiddleP[0],Alpha-MiddleP[1],dict(dict_Calib_AB),dict(screenDimens))
# draw calibration points on world image
cv2.circle(world,tuple(screenDimens['topRight']),3,(255,255,0),-1)
cv2.circle(world,tuple(screenDimens['topLeft']),3,(255,255,0),-1)
cv2.circle(world,tuple(screenDimens['bottomLeft']),3,(255,255,0),-1)
cv2.circle(world,tuple(screenDimens['bottomRight']),3,(255,255,0),-1)
# Draw a circle where the pupil is looking at on the world image
# a weird occurence happens at the beginning where alpha and beta are Nan
# to check if nan we will test if it equals it self. Nan != Nan return true
if newPupx != newPupx or newPupy != newPupy:
continue
cv2.circle(world,(int(newPupx),int(newPupy)),3,(0,0,255),-1)
#print("X: {} Y: {}".format(newPupx,newPupy))
# # Find the screen
# world, screenDict, flag = findScreen(world)
# # draw screen locs
# cv2.circle(world,tuple(screenDict['topRight']),2,(255,0,0),-1)
# cv2.circle(world,tuple(screenDict['topLeft']),2,(255,0,0),-1)
# cv2.circle(world,tuple(screenDict['bottomLeft']),2,(255,0,0),-1)
# cv2.circle(world,tuple(screenDict['bottomRight']),2,(255,0,0),-1)
# move the mouse
# eyeLoc = [newPupx,newPupy]
# moveMouseRelative(eyeLoc,screenDimens)
# Display the Alpha and beta angles on the image
# text = 'Beta: %.2f Alpha: %.2f' % (Beta, Alpha)
# font = cv2.FONT_HERSHEY_SIMPLEX
# cv2.putText(eye,text,(10,50), font, 0.5, (200,255,155), 1, cv2.LINE_AA)
# show the eye and world images on the screen
cv2.imshow("Eye Camera", eye)
cv2.imshow("World Camera", world)
# if 'q' is pressed then break out of the loop
if cv2.waitKey(10) & 0xFF == ord('q'):
break
# safely destroy all windows and camera connections before ending the program
cv2.destroyAllWindows()
eyeCam.release()
worldCam.release()
guiAPP.py
PythonThis code runs the calibration step of the program. You may run this code separately from main3.py but the original purpose of this code is not to be run by itself. To run this code, you need to download PIL and tkinter
# This file is the combination of guiTut3 and guiTut4
from tkinter import *
import cv2
import PIL.Image, PIL.ImageTk
import time
import pyautogui as pag
from eyeFunction import *
class App:
def __init__(self, window, window_title, video_source = 2, video_source2 = 0): # will need to add a second video_source for world Cam
# create dictionary to hold calibration values. This dictionary will be placed into a folder called Dict.csv
self.diction = {'Middle': 1, 'topRight': 1, 'topLeft' : 1, 'bottomLeft' : 1, 'bottomRight' : 1, 'pupilRadius' : 1}
# create diction to hold screen location. This will be placed into a folder called screenLocation.txt
self.screenLocs = {'topLeft': 1, 'bottomLeft' : 1, 'topRight' : 1, 'bottomRight' : 1}
self.screenPoints = {'topLeft': 1, 'bottomLeft' : 1, 'topRight' : 1, 'bottomRight' : 1}
# create radius and pupil Location attributes
self.radius= 0
self.pupilLocation = 0
self.frame = []
self.window = window
self.window.title(window_title)
# open the video sources
print("warming Up...")
time.sleep(2.0)
self.vid = MyVideoCaptureEye(video_source)
self.vidWorld = MyVideoCaptureWorld(video_source2)
print("Loading Cameras...")
time.sleep(2.0)
# display the camera on the window using the canvas display function
x, y = pag.size()
self.canvas = Canvas(window,width = x, height = y)
self.canvas.pack()
# Take in the length and height of the screem in pixel units. make the window full screen of that size
length, height = pag.size()
self.window.attributes('-fullscreen', True)
# create path to the two target pictures
path = "blueTargetShrink.png"
path2 = "redTargetShrink.png"
# create a tkinter-compatible photo image
img1 = PIL.ImageTk.PhotoImage(file = path)
img2 = PIL.ImageTk.PhotoImage(file = path2)
# create a label widget to display the image or text
labelText = Label(window,text = 'LOOK AT THE RED TARGET\nKEEP YOUR HEAD STILL')
labelText.place(x = length/3, y= 0,width= 500, height= 200)
labelText.config(font = ('times', 20, 'bold'))
# arbitrary location FIX LATER
imgSize = 100
labelTL = self.createLabel(window, img1, int(0.1*x)-imgSize/2 , int(0.1*y)-imgSize/2)
labelTR = self.createLabel(window, img1, int(0.9*x)-imgSize/2, int(0.1*y)-imgSize/2)
labelBL = self.createLabel(window, img1, int(0.1*x)-imgSize/2, int(0.9*y)-imgSize/2)
labelBR = self.createLabel(window, img1, int(0.9*x)-imgSize/2, int(0.9*y)-imgSize/2)
labelMi = self.createLabel(window, img1, length/2 - 50, height/2 - 50)
self.labelList = [labelMi, labelTR, labelTL, labelBL, labelBR]
# after X time interval switch the location of the greenTarget and record the pupil Location
intervalT = 800
self.labelCounter = 0
self.window.after(intervalT*6, lambda: labelMi.configure(image = img2) )
#COLLECT PUPIL READING
self.window.after(intervalT*8, lambda: self.collectPupilLocation() )
self.window.after(intervalT*9, lambda: labelMi.configure(image = img1) )
self.window.after(intervalT*9, lambda: labelTR.configure(image = img2) )
#COLLECT PUPIL READING
self.window.after(intervalT*11, lambda: self.collectPupilLocation() )
self.window.after(intervalT*12, lambda: labelTR.configure(image = img1) )
self.window.after(intervalT*12, lambda: labelTL.configure(image = img2) )
#COLLECT PUPIL READING
self.window.after(intervalT*14, lambda: self.collectPupilLocation() )
self.window.after(intervalT*15, lambda: labelTL.configure(image = img1) )
self.window.after(intervalT*15, lambda: labelBL.configure(image = img2) )
#COLLECT PUPIL READING
self.window.after(intervalT*17, lambda: self.collectPupilLocation() )
self.window.after(intervalT*18, lambda: labelBL.configure(image = img1) )
self.window.after(intervalT*18, lambda: labelBR.configure(image = img2) )
#COLLECT PUPIL READING
self.window.after(intervalT*20, lambda: self.collectPupilLocation() )
self.window.after(intervalT*21, lambda: labelBR.configure(image = img1) )
# after all of the reading have been recorded save them to the filename
self.window.after(intervalT*22, lambda: self.saveLocation('pupilCenter.csv',self.diction) )
self.window.after(intervalT*22, lambda: self.saveLocation('screenLocation.txt',self.screenLocation) )
# after X seconds the window will close
self.window.after(intervalT*23, lambda: self.window.destroy())
self.delay = 5
self.update()
self.window.mainloop()
def saveLocation(self, filename,dicty):
toCSV(filename, dicty)
def collectPupilLocation(self):
if self.labelCounter == 0:
self.diction['pupilRadius'] = self.radius
self.diction['Middle'] = self.pupilLocation
# This change in points is because we are looking at the targets and not the screen dimesions
tL = self.screenPoints['topLeft']
bR = self.screenPoints['bottomRight']
lenX = bR[0] - tL[0]
lenY = bR[1] - tL[1]
# loop over values and change it to the correct ratio
# change this in the future
self.screenPoints['topLeft'] = [ int(self.screenPoints['topLeft'][0] + lenX*0.1),int(self.screenPoints['topLeft'][1] + lenY*0.1) ]
self.screenPoints['bottomLeft'] = [ int(self.screenPoints['topLeft'][0]),int(self.screenPoints['topLeft'][1] + lenY*0.8) ]
self.screenPoints['topRight'] = [ int(self.screenPoints['topLeft'][0] + lenX*0.8),int(self.screenPoints['topLeft'][1]) ]
self.screenPoints['bottomRight'] = [ int(self.screenPoints['topLeft'][0] + lenX*0.8),int(self.screenPoints['topLeft'][1] + lenY*0.8) ]
# assign the screenLocation
self.screenLocation = self.screenPoints
elif self.labelCounter == 1:
self.diction['topRight'] = self.pupilLocation
elif self.labelCounter == 2:
self.diction['topLeft'] = self.pupilLocation
elif self.labelCounter == 3:
self.diction['bottomLeft'] = self.pupilLocation
elif self.labelCounter == 4:
self.diction['bottomRight'] = self.pupilLocation
self.labelCounter += 1
def createLabel(self, window, img, xpos=0,ypos=0,w=100,h=100):
tempLabel = Label(window,image = img)
tempLabel.place(x = xpos, y= ypos,width= w, height= h)
return tempLabel
def update(self):
# get the frame from the video source
ret, frame, self.radius, eyeCenter = self.vid.get_frame()
# if the pupil is found then assign it to self.pupiLocation. if it is not then self.pupilLocation will be the previous value
if eyeCenter:
self.pupilLocation = eyeCenter
ret2, frame2, self.screenPoints = self.vidWorld.get_frame()
x, y = pag.size()
if ret:
self.photo = PIL.ImageTk.PhotoImage(image = PIL.Image.fromarray(frame))
self.canvas.create_image(x/4, y/2, image = self.photo, anchor = CENTER)
if ret2:
self.photo2 = PIL.ImageTk.PhotoImage(image = PIL.Image.fromarray(frame2))
self.canvas.create_image(3*x/4, y/2, image = self.photo2, anchor = CENTER)
self.window.after(self.delay, self.update)
class MyVideoCaptureWorld:
def __init__(self, video_source2):
# open the video source
self.vidWorld = cv2.VideoCapture(video_source2)
if not self.vidWorld.isOpened():
raise ValueError("Unable to open video source", video_source2)
# Get video source width and height
self.width = self.vidWorld.get(cv2.CAP_PROP_FRAME_WIDTH)
self.height = self.vidWorld.get(cv2.CAP_PROP_FRAME_HEIGHT)
def get_frame(self):
if self.vidWorld.isOpened():
ret2, frame2 = self.vidWorld.read()
frame2 = cv2.resize(frame2,None,fx=0.5, fy=0.5, interpolation = cv2.INTER_LINEAR)
if ret2:
# return success flag along with frame and the screenLocs
frame2, screenLocs, flag = findScreen(frame2)
return (ret2, cv2.cvtColor(frame2, cv2.COLOR_RGB2BGR), screenLocs)
else:
return(ret2,None)
else:
return (ret2,None)
def __del__(self):
if self.vidWorld.isOpened():
self.vidWorld.release()
class MyVideoCaptureEye:
def __init__(self, video_source):
# open the video source
self.vid = cv2.VideoCapture(video_source)
if not self.vid.isOpened():
raise ValueError("Unable to open video source", video_source)
# Get video source width and height
self.width = self.vid.get(cv2.CAP_PROP_FRAME_WIDTH)
self.height = self.vid.get(cv2.CAP_PROP_FRAME_HEIGHT)
def get_frame(self):
if self.vid.isOpened():
ret, frame = self.vid.read()
frame = cv2.resize(frame,None,fx=0.5, fy=0.5, interpolation = cv2.INTER_LINEAR)
if ret:
# return a success flag and the current frame converted to RGB
frame, pupilCords, radius = findPupil(frame)
return (ret, cv2.cvtColor(frame, cv2.COLOR_RGB2BGR), radius, pupilCords)
else:
return (ret,None)
else:
return (ret, None)
def __del__(self):
if self.vid.isOpened():
self.vid.release()
#Create the window and pass it to the Application Object
App(Tk(),"tkinter")
eyeFunction.py
PythonThis code serves a package for the main3.py, generateSphere3.py and guiAPP.py files.
import cv2
import numpy as np
import imutils
import json
from eyeCalibration import predictZpoint
import math
import time
import pyautogui as pag
# ADD DESCRIPTION OF FUNTION, INPUTS, OUTPUTS, DATE MADE AND FURTHER REVISION
def moveMouseRelative(eye,screenDict):
# this function will move the mouse by joystick configuration. It will move if look outside the calibration points
x = eye[0]
y = eye[1]
tl = screenDict['topLeft']
br = screenDict['bottomRight']
pix = 50
if x < tl[0]:
pag.moveRel(-pix,0)
elif x > br[0]:
pag.moveRel(pix,0)
if y < tl[1]:
pag.moveRel(0,-pix)
elif y > br[1]:
pag.moveRel(0,pix)
def rotate_points(dict_Calib_AB,pupilCenter_B, pupilCenter_A):
# dict_Calib_AB is a dictionary of calibrated angles in radians found in file calibrateAB.txt
# dict_Calib_Norm is a dictionary of the calibrated points in 'pixel' units
# pupilCenter_B is x coord of pupil center
# pupilCenter_A is y coord of pupil center
# First we want to rotate the coordinate so that it is approximately flat with respect to the horizon
BL = dict_Calib_AB['bottomLeft']
BR = dict_Calib_AB['bottomRight']
[_,m,_] = createLine(BL,BR)
angle = math.atan(m)
rotate_B, rotate_A = rotatePoints(pupilCenter_B,pupilCenter_A, m)
# rotate the calibration points
# Separate the dictionary into the Keys and the values
rotate_calib = []
for key, value in dict_Calib_AB.items():
rotate_calib.append(value)
rotate_calib.pop(0)
for i in list(range(0,4)):
rotate_calib[i] = rotatePoints(rotate_calib[i][0],rotate_calib[i][1], m)
# Turn the list of values back to a dictionary
keys = ['topRight', 'topLeft', 'bottomLeft', 'bottomRight']
rotate_calib_dict = {}
for i in range(0,4):
rotate_calib_dict[keys[i]] = rotate_calib[i]
return rotate_B, rotate_A, rotate_calib_dict
def fitToRect(arr0,arrX,arrY):
x=0
y=0
if arr0[0] > 0:
x = max(arr0[0],arrX[0])
else:
x = -max(-arr0[0],-arrX[0])
if arr0[1] > 0:
y = max(arr0[1],arrY[1])
else:
y = -max(-arr0[1],-arrY[1])
return [x,y]
# if the function is successful then we can delete stretchToTect and scale to screen
def projectiveTransToScreen(pX,pY,calibrationPt, screenPts):
# Step 0: We first need to scale pX,Py and calibrationPt so that the center is in the center
left = calibrationPt['topLeft'][0]
right = calibrationPt['bottomRight'][0]
up = calibrationPt['topLeft'][1]
down = calibrationPt['bottomRight'][1]
#ERRRORRRRR HERE FIX THIS LATER!
leftBool = abs(left) > abs(right)
ratX = abs(right) / abs(left)
upBool = abs(up) > abs(down)
ratY = abs(down) / abs(up)
# # Change pX, pY
# if pX < 0:
# pX = pX *ratX
# else:
# pX = pX*(1/ratX)
# if pY > 0:
# pY=pY*ratY
# else:
# pY=pY*(1/ratY)
# Change Calibration Pts
if leftBool:
if pX < 0:
pX = pX * ratX
# can delete this after troublshoot
calibrationPt['topLeft'][0] = calibrationPt['topLeft'][0]*ratX
calibrationPt['bottomLeft'][0] = calibrationPt['bottomLeft'][0]*ratX
else:
if pX > 0:
pX = pX * (1/ratX)
# can delete this after troublshoot
calibrationPt['topRight'][0] = calibrationPt['topRight'][0]*(1/ratX)
calibrationPt['bottomRight'][0] = calibrationPt['bottomRight'][0]*(1/ratX)
if upBool:
if pY > 0:
pY = pY * ratY
# can delete this after troublshoot
calibrationPt['topLeft'][1] = calibrationPt['topLeft'][1]*ratY
calibrationPt['topRight'][1] = calibrationPt['topRight'][1]*ratY
else:
if pY < 0:
pY = pY * (1/ratY)
# can delete this after troublshoot
calibrationPt['topLeft'][1] = calibrationPt['topLeft'][1]*ratY
calibrationPt['topRight'][1] = calibrationPt['topRight'][1]*ratY
# toCSV('rect.txt',calibrationPt)
# print(calibrationPt)
# print("PX: {} Py: {}".format(pX,pY))
# Step 1: find the inv(A) matrix
k = np.array( [[calibrationPt['topRight'][0],calibrationPt['topLeft'][0],calibrationPt['bottomLeft'][0]],\
[calibrationPt['topRight'][1],calibrationPt['topLeft'][1],calibrationPt['bottomLeft'][1]],\
[1,1,1]])
l = np.array([calibrationPt['bottomRight'][0],calibrationPt['bottomRight'][1],1])
[lam,u,tau] = np.linalg.solve(k, l)
A = np.array( [[lam*calibrationPt['topRight'][0],u*calibrationPt['topLeft'][0],tau*calibrationPt['bottomLeft'][0]],\
[lam*calibrationPt['topRight'][1],u*calibrationPt['topLeft'][1],tau*calibrationPt['bottomLeft'][1]],\
[lam,u,tau]])
A_inv = np.linalg.inv(A)
#print(A_inv)
# Step 2: find the B matrix
k = np.array( [[screenPts['topRight'][0],screenPts['topLeft'][0],screenPts['bottomLeft'][0]],\
[screenPts['topRight'][1],screenPts['topLeft'][1],screenPts['bottomLeft'][1]],\
[1,1,1]])
l = np.array([screenPts['bottomRight'][0],screenPts['bottomRight'][1],1])
[lam,u,tau] = np.linalg.solve(k, l)
B = np.array( [[lam*screenPts['topRight'][0],u*screenPts['topLeft'][0],tau*screenPts['bottomLeft'][0]],\
[lam*screenPts['topRight'][1],u*screenPts['topLeft'][1],tau*screenPts['bottomLeft'][1]],\
[lam,u,tau]])
#print(B)
# Step 3: find C
C = np.matmul(B,A_inv)
#print(C)
(xdot,ydot,zdot) = np.matmul(C,[pX,pY,1])
# print(xdot/zdot)
# print(ydot/zdot)
return xdot/zdot, ydot/zdot
# print("X :{} Y: {}".format(centerScreenX,centerScreenY))
def stretchToRect(rotate_B,rotate_A,rotate_calib):
# Second we want to stretch the coordinates to fit a perfect rectangle
# we do this by calculating a ratio that will stretch this field propotionally
rightLine = createLine(rotate_calib['bottomRight'],rotate_calib['topRight'])
leftLine = createLine(rotate_calib['bottomLeft'],rotate_calib['topLeft'])
topLine = createLine(rotate_calib['topLeft'],rotate_calib['topRight'])
bottomLine = createLine(rotate_calib['bottomLeft'],rotate_calib['bottomRight'])
# Find the ratios to stretch the point based on its quadrant
# Change pupilPoints by find the ratio to stretch the point
if rotate_B > 0:
x_pos = (rotate_A-rightLine[2]) / rightLine[1]
pos_X_ratio = max(rotate_calib['bottomRight'][0],rotate_calib['topRight'][0]) / x_pos
rotate_B = pos_X_ratio * rotate_B
elif rotate_B < 0:
x_pos = (rotate_A-leftLine[2]) / leftLine[1]
neg_X_ratio = min(rotate_calib['bottomLeft'][0],rotate_calib['topLeft'][0]) / x_pos
rotate_B = neg_X_ratio * rotate_B
if rotate_A > 0:
y_pos = topLine[1]*rotate_B+topLine[2]
pos_y_ratio = max(rotate_calib['topLeft'][1],rotate_calib['topRight'][1]) / y_pos
rotate_A = rotate_A * pos_y_ratio
elif rotate_A < 0:
y_pos = bottomLine[1]*rotate_B+bottomLine[2]
neg_y_ratio = min(rotate_calib['bottomLeft'][1],rotate_calib['bottomRight'][1]) / y_pos
rotate_A = rotate_A * neg_y_ratio
# IF the code fails change this part!!!!!!
# Change calibration points to fit a perfect rectangle
# for the future this code can be done outside of this function to save time
rotate_calib['topRight'] = fitToRect(rotate_calib['topRight'],rotate_calib['bottomRight'],rotate_calib['topLeft'])
rotate_calib['topLeft'] = fitToRect(rotate_calib['topLeft'],rotate_calib['bottomLeft'],rotate_calib['topRight'])
rotate_calib['bottomLeft'] = fitToRect(rotate_calib['bottomLeft'],rotate_calib['topLeft'],rotate_calib['bottomRight'])
rotate_calib['bottomRight'] = fitToRect(rotate_calib['bottomRight'],rotate_calib['topRight'],rotate_calib['bottomLeft'])
# Third we need to scale the points so that both sides of rectangle are equal in area
left = rotate_calib['topLeft'][0]
right = rotate_calib['bottomRight'][0]
up = rotate_calib['topLeft'][1]
down = rotate_calib['bottomRight'][1]
leftBool = abs(left) > abs(right)
ratX = abs(right) / abs(left)
upBool = abs(up) > abs(down)
ratY = abs(down) / abs(up)
if leftBool:
rotate_B = rotate_B*ratX
# can delete this after troublshoot
rotate_calib['topLeft'][0] = rotate_calib['topLeft'][0]*ratX
rotate_calib['bottomLeft'][0] = rotate_calib['bottomLeft'][0]*ratX
else:
rotate_B = rotate_B*(1/ratX)
# can delete this after troublshoot
rotate_calib['topRight'][0] = rotate_calib['topRight'][0]*(1/ratX)
rotate_calib['bottomRight'][0] = rotate_calib['bottomRight'][0]*(1/ratX)
if upBool:
rotate_A=rotate_A*ratY
# can delete this after troublshoot
rotate_calib['topLeft'][1] = rotate_calib['topLeft'][1]*ratY
rotate_calib['topRight'][1] = rotate_calib['topRight'][1]*ratY
else:
rotate_A=rotate_A*(1/ratY)
# can delete this after troublshoot
rotate_calib['topLeft'][1] = rotate_calib['topLeft'][1]*ratY
rotate_calib['topRight'][1] = rotate_calib['topRight'][1]*ratY
# Fourth we set the origin point to be the top left points that way we can scale it to Screen size
TL = rotate_calib['topLeft']
TL = [TL[0],TL[1]]
rotate_B = rotate_B - TL[0]
rotate_A = rotate_A - TL[1]
# do this for the calibration points as well
for key,value in rotate_calib.items():
rotate_calib[key][0] = rotate_calib[key][0]-TL[0]
rotate_calib[key][1] = rotate_calib[key][1]-TL[1]
# Because the screen has positive x to the right and positive y down. we have to flip our convention
# Our previous convention: positive x to right and positive y is up
for key,value in rotate_calib.items():
rotate_calib[key][1] = -value[1]
rotate_A = -rotate_A
# Return the altered values, we need to also return the altered rotate_calib points for scaleToScreen()
return rotate_B, rotate_A, rotate_calib
def scaleToScreen(rotate_B, rotate_A, rotate_calib_dict,screenDimens):
# First we have to set the origin (TL) to TL of ScreenDimension, since the rotate points were manipulated to be a perfect rectangle, we just need to add a constant to each coordinate to fit screen rectangel dimensions
new_TL = screenDimens['topLeft']
ratio_x = (screenDimens['bottomRight'][0]-screenDimens['topLeft'][0])/rotate_calib_dict['bottomRight'][0]
ratio_y = (screenDimens['bottomRight'][1]-screenDimens['topLeft'][1])/rotate_calib_dict['bottomRight'][1]
# scale the calibration points by the ratios
for key,value in rotate_calib_dict.items():
rotate_calib_dict[key][0] = value[0] * ratio_x + new_TL[0]
rotate_calib_dict[key][1] = value[1] * ratio_y + new_TL[1]
# scale the pupilPoint
scale_B = rotate_B * ratio_x + new_TL[0]
scale_A = rotate_A * ratio_y + new_TL[1]
# print("XPixel: {} YPixel: {}".format(scale_B,scale_A))
return scale_B, scale_A, rotate_calib_dict
def rotatePoints(x,y, RadOfRot):
c,s = np.cos(RadOfRot), np.sin(RadOfRot)
j = np.matrix( [ [c,s],[-s,c] ] )
m = np.dot(j, [x, y])
return [float(m.T[0]), float(m.T[1])]
def createLine(p1, p2):
slope = ( p2[1] - p1[1] ) / ( p2[0] - p1[0] )
x = slope
y = 1
b = p1[1] - slope*p1[0]
return [y,x,b]
def findAlphaBeta(center, originPt, rat):
# points shouuld be in mm units NOT PIXEL units
ratio = rat
# Return values in radians
alpha = math.atan( (center[1]-originPt[1]) / center[2] )
betas = math.atan( (center[0]-originPt[0]) / center[2] )
alpha = math.tan(alpha)
betas = math.tan(betas)
return -round(betas,8), round(alpha,8)
def toCSV(filename, diction):
with open(filename, 'w') as f:
json.dump(diction, f)
f.close()
def fromCSV(filename = 'pupilCenter.csv'):
with open(filename) as f:
loadedDictionary = json.load(f)
f.close()
return loadedDictionary
def findPupil_drawAngle(image, eyeCenter, importd):
#function takes in a image and return the pupil located in the image
center = []
foundPupil = 0
radius = 0
alpha = 0
betas = 0
a=0
b=0
#lets create our 'sliders' aka the kernels
# kernel2 = np.ones( (3,3), np.uint8 )
# gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
# _, thresholded = cv2.threshold(gray, 50, 255, cv2.THRESH_BINARY_INV)
# opening2 = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, kernel2)
# im2, contours, hierarchy = cv2.findContours(opening2, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
# height, width = opening2.shape
# contours = sorted(contours,key = cv2.contourArea,reverse = True)[:6]
# Lets create our 'sliders' aka the kernels
kernel2 = np.ones( (3,3), np.uint8)
# crop the image to only look at the ROI and scale image back to original size
height = image.shape[0]
width = image.shape[1]
cropped = image[int(0.4*height):int(1*height),int(0*width):int(0.6*width)]
resized = cv2.resize(cropped,(width,height), interpolation = cv2.INTER_AREA)
# turn the resized photo into a grayscale image, threshold filter, and find contours
gray = cv2.cvtColor(resized,cv2.COLOR_BGR2GRAY)
_, thresholded = cv2.threshold(gray, 75, 255, cv2.THRESH_BINARY_INV)
opening2 = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, kernel2)
im2, contours, hierarchy = cv2.findContours(opening2, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
#sort the contours from biggest to smallest then take only the X biggest values
contours = sorted(contours,key = cv2.contourArea,reverse = True)[:6]
for contour in contours:
#finds the area of the contour
area = cv2.contourArea(contour)
if area < 75:
continue
#create a bounding circle around the contour return center and radius
(x,y), radius = cv2.minEnclosingCircle(contour)
radius = int(radius)
extend = area / (3.14152*radius**2)
if extend < 0.3:
continue
# fit and draw ellipse to the contour image
ellipse = cv2.fitEllipse(contour)
center = (int(ellipse[0][0]),int(ellipse[0][1]))
radius = int( (ellipse[1][0]+ellipse[1][1] )/2 )
# Draw pupil
cv2.ellipse(resized,ellipse,(0,255,0),2)
cv2.circle(resized,tuple(center),2,(0,255,0),-1)
# Draw eyeCenter
cv2.circle(resized,(eyeCenter[0],eyeCenter[1]),1,(0,255,255),-1)
# Draw line connecting eyeCenter and pupilCenter
cv2.line(resized, center, (eyeCenter[0],eyeCenter[1]), (0,255,0), 2)
# #Draw the cailbration points on the eye image
# cv2.circle(resized,tuple(importd['Middle']),1,(255,255,0),-1)
# cv2.circle(resized,tuple(importd['topRight']),1,(255,255,0),-1)
# cv2.circle(resized,tuple(importd['topLeft']),1,(255,255,0),-1)
# cv2.circle(resized,tuple(importd['bottomLeft']),1,(255,255,0),-1)
# cv2.circle(resized,tuple(importd['bottomRight']),1,(255,255,0),-1)
originPt = [eyeCenter[0],eyeCenter[1],12.2]
center = [center[0], center[1]]
#changed the originPt and center in to mm
# Take the values of pupilocations and change them from "pixel" units to mm. average pupil radius is 3mm
ratio = 3 / importd['pupilRadius']
ratio = 2.0 / 25
for i in range(0,2):
center[i] = center[i]*ratio
for p in range(0,2):
originPt[p] = originPt[p]*ratio
center = predictZpoint(center,originPt)
alpha = math.degrees( math.atan( (center[1]-originPt[1]) / center[2] ) )
betas = math.degrees( math.atan( (center[0]-originPt[0]) / center[2] ) )
b,a = findAlphaBeta(center, originPt, ratio)
# IF we have made it this far then we have selected our pupil and we 'break' so we only draw one pupil
break
return resized, b, a
def findPupil(image):
# Function takes in a image and return the pupil located in the image
center = []
foundPupil = 0
radius = 0
# Lets create our 'sliders' aka the kernels
kernel2 = np.ones( (3,3), np.uint8)
# crop the image to only look at the ROI and scale image back to original size
height = image.shape[0]
width = image.shape[1]
cropped = image[int(0.4*height):int(1*height),int(0*width):int(0.6*width)]
resized = cv2.resize(cropped,(width,height), interpolation = cv2.INTER_AREA)
# turn the resized photo into a grayscale image, threshold filter, and find contours
gray = cv2.cvtColor(resized,cv2.COLOR_BGR2GRAY)
_, thresholded = cv2.threshold(gray, 75, 255, cv2.THRESH_BINARY_INV)
opening2 = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, kernel2)
im2, contours, hierarchy = cv2.findContours(opening2, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
#sort the contours from biggest to smallest then take only the X biggest values
contours = sorted(contours,key = cv2.contourArea,reverse = True)[:6]
for contour in contours:
# MAY NOT NEED TO DO THE AREA THING BECAUSE WE SORTED CONTOURS FROM BIGGEST TO SMALLEST
#finds the area of the contour
area = cv2.contourArea(contour)
# if area < 75:
# continue
#create a bounding circle around the contour return center and radius
(x,y), radius = cv2.minEnclosingCircle(contour)
radius = int(radius)
# sometimes the camera picks up a small radius by accident, if that is the case just pass
if radius == 0:
break
extend = area / (3.14152*radius**2)
if extend < 0.3:
continue
# fit and draw ellipse to the contour image
# if it cannt fit the ellipse then go on to the next loop
try:
ellipse = cv2.fitEllipse(contour)
center = [int(ellipse[0][0]),int(ellipse[0][1])]
radius = int( (ellipse[1][0]+ellipse[1][1] )/2 )
cv2.ellipse(resized,ellipse,(0,255,0),2)
cv2.circle(resized,tuple(center),5,(0,255,0),-1)
except:
continue
# IF we have made it this far then we have selected our pupil and we 'break' so we only draw one pupil
break
return resized, center, radius
####Experimating with new findPupil function
# def findPupil(image):
# #function takes in a image and return the pupil located in the image
# center = []
# foundPupil = 0
# radius = 0
# #lets create our 'sliders' aka the kernels
# kernel2 = np.ones( (3,3), np.uint8 )
# gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
# _, thresholded = cv2.threshold(gray, 50, 255, cv2.THRESH_BINARY_INV)
# opening2 = cv2.morphologyEx(thresholded, cv2.MORPH_OPEN, kernel2)
# im2, contours, hierarchy = cv2.findContours(opening2, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
# contours = sorted(contours,key = cv2.contourArea,reverse = True)[:6]
# height, width = opening2.shape
# for contour in contours:
# #finds the area of the contour
# area = cv2.contourArea(contour)
# if area < 75 or area > 1000:
# continue
# #create a bounding circle around the contour return center and radius
# (x,y), radius = cv2.minEnclosingCircle(contour)
# radius = int(radius)
# extend = area / (3.14152*radius**2)
# if extend < 0.3:
# continue
# if not (0.25*height) < int(y) < (0.75 * height):
# continue
# if not (0.25*width) < int(x) < (0.75 * width):
# continue
# center = ( int(x),int(y) )
# print("center: {} area: {} extend: {}".format(center,area,extend))
# cv2.circle(image,center,radius,(0,0,255),2)
# cv2.circle(image,center,2,(0,255,255),2)
# return image, center, radius
# ADD DESCRIPTION OF FUNTION, INPUTS, OUTPUTS, DATE MADE AND FURTHER REVISION
def auto_canny(image, sigma=0.33):
# compute the median of the single channel pizel intensities
v = np.median(image)
# apply automatic Canny edge detection using the computed median
lower = int(max(0, (1.0 - sigma) * v))
upper = int(min(255, (1.0 + sigma) * v))
edged = cv2.Canny(image, lower, upper)
#return the edged image
return edged
def findScreen(image):
# Convert the image to grayscale, blur it, and find the edges
gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
blur = cv2.bilateralFilter(gray, 11, 17, 17)
edged = auto_canny(blur)
# find the contours of in the edge image and keep only the 3 largest
_,cnts, _ = cv2.findContours(edged.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key = cv2.contourArea, reverse = True)[:3]
screenCnt = None
flag = False
screenPts = None
screenDict = {'topLeft': [0,0], 'bottomLeft' : [0,0], 'topRight' : [0,0], 'bottomRight' : [0,0]}
for c in cnts:
#aprroximate the contour
peri = cv2.arcLength(c,True)
approx = cv2.approxPolyDP(c, 0.02*peri, True)
# if the approximated contour has four points, then we have our screen
if len(approx) == 4:
screenCnt = approx
flag = True
break
if flag:
screenPts = [ [screenCnt[0][0][0].item(), screenCnt[0][0][1].item()] , \
[screenCnt[1][0][0].item(), screenCnt[1][0][1].item()], \
[screenCnt[2][0][0].item(), screenCnt[2][0][1].item()], \
[screenCnt[3][0][0].item(), screenCnt[3][0][1].item()] ]
screenPts = sorted(screenPts)
screenPts[0:2] = sortPoints(screenPts[0:2])
screenPts[2:4] = sortPoints(screenPts[2:4])
screenDict['topLeft'] = screenPts[0]
screenDict['bottomLeft'] = screenPts[1]
screenDict['topRight'] = screenPts[2]
screenDict['bottomRight'] = screenPts[3]
# draw the contours on the image
cv2.drawContours(image, screenCnt,-1,(0,255,0),3)
return image, screenDict, flag
#return frame, screenDiction
def sortPoints(arr):
x1 = arr[0][0]
y1 = arr[0][1]
x2 = arr[1][0]
y2 = arr[1][1]
if y1 < y2:
return [ [x1,y1],[x2,y2] ]
else:
return [ [x2,y2],[x1,y1] ]
import numpy as np
import json
def toCSV(filename, diction):
with open(filename, 'w') as f:
json.dump(diction, f)
f.close()
def fromCSV(filename = 'pupilCenter.csv'):
with open(filename) as f:
loadedDictionary = json.load(f)
f.close()
return loadedDictionary
def createMatEqn(p1,p4,side = 'left'):
if side == 'left':
return [ -2*(p1[0]-p4[0]), -2*(p1[1]-p4[1]), -2*(p1[2]-p4[2]) ]
elif side == 'right':
return -p1[0]**2-p1[1]**2-p1[2]**2+p4[0]**2+p4[1]**2+p4[2]**2
else:
print("invalid option")
def findCenter3D(p1,p2,p3,p4):
A = np.array( [createMatEqn(p1,p4),createMatEqn(p2,p4),createMatEqn(p3,p4)] )
B = np.array( [createMatEqn(p1,p4,'right'),createMatEqn(p2,p4,'right'),createMatEqn(p3,p4,'right')] )
return np.linalg.solve(A,B)
def findCenter2D(origin, p1, p2):
midP1 = findMidpoint2D(origin, p1)
slope1 = findSlope2D(origin, p1)
line1 = createLine2D(midP1,-1/slope1)
midP2 = findMidpoint2D(origin, p2)
slope2 = findSlope2D(origin, p2)
line2 = createLine2D(midP2,-1/slope2)
A = np.array([ [ line1[0],line1[1] ],[ line2[0],line2[1] ] ])
B = np.array([ line1[2],line2[2] ])
return np.linalg.solve(A,B)
def createLine2D(p, slope):
x = -slope
y = 1
b = p[1] - slope*p[0]
return [x,y,b]
def findSlope2D(p1,p2):
return ( p2[1] - p1[1] ) / ( p2[0] - p1[0] )
def findMidpoint2D(p1,p2):
return [ (p1[0]+p2[0])/2 , (p1[1]+p2[1])/2 ]
# no way to convert pixels to mm so we might have to approximate the z difference for the calibration points
def predictZpoint(p2, origin2D):
oriX = origin2D[0]
oriY = origin2D[1]
q = ( abs(oriX - p2[0])**2 + abs(oriY - p2[1])**2 )**0.5
# quadratic equation with r = 12.4mm
r = 12.4
B = -2*r
C = q**2
z = (-B-np.sqrt(B**2-4*C ) ) / 2
return [p2[0],p2[1], r - z]
generateSphere3.py
PythonFrom the calibration points, this file generates a sphere that models the eye. This allows us to look at the angles the pupil makes relative to the "center" position. This is code is not meant to be run separately. It is meant to be ran only by main3.py
from eyeCalibration import *
from eyeFunction import findAlphaBeta
import math
# Import the dictionary of pupil locations for the eye
importD = fromCSV('pupilCenter.csv')
keys = []
values = []
# Separate the dictionary into the Keys and the values
for key, value in importD.items():
keys.append(key)
values.append(value)
# Remove the last value 'radius' because we do not need to do any calculations on it
values.pop()
# the center is (BRx - 11, BRy+20, 0)
center3D = [ values[4][0]-11, values[4][1]+20,0 ]
# Take the values of pupilocations and change them from "pixel" units to mm. average pupil radius is 3mm
ratio = 2.0 / 25
for i in list(range(0,5)):
for p in list(range(0,2)):
values[i][p] = values[i][p] * ratio
# Do the same for the center point
center3D[0] = center3D[0]*ratio
center3D[1] = center3D[1]*ratio
# Take the values of pupilLocations and predict the Z height which should range from 0 < z < 12.4mm
# We set the reference point to the be the at the center point with a z hieght of 12.4mm
refPoint = [ center3D[0], center3D[1], 12.2]
for i in list(range(0,5)):
values[i]= predictZpoint(values[i],refPoint)
# change the numbers back to 'pixel units'
for i in range(0,3):
center3D[i] = center3D[i]*1/ratio
center3D = [ int(center3D[0]), int(center3D[1]), int(center3D[2]) ]
# Save the center3D
toCSV('eyeCenter.txt',center3D)
# Next, calculate the alphas and betas of the calibration points
# create dictionary to hold the Alphas and Betas. This dictionary will be placed into a folder called calibrateAB.txt
diction = {'Middle': [], 'topRight': [], 'topLeft' : [], 'bottomLeft' : [], 'bottomRight' : []}
bMid , aMid = findAlphaBeta(values[0], refPoint, ratio)
for i in list(range(0,5)):
b,a = findAlphaBeta(values[i], refPoint, ratio)
if i == 0:
diction['Middle'] = [b-bMid,a-aMid]
elif i == 1:
diction['topRight'] = [b-bMid,a-aMid]
elif i == 2:
diction['topLeft'] = [b-bMid,a-aMid]
elif i == 3:
diction['bottomLeft'] = [b-bMid,a-aMid]
elif i == 4:
diction['bottomRight'] = [b-bMid,a-aMid]
# Save the center3D
toCSV('calibrateAB.txt',diction)
# Save the Middle Point
MiddleP = [bMid,aMid]
toCSV('Middle.txt', MiddleP)
9 projects • 5 followers
Hi, my name is Kyelo and I am a 4th-year mechanical engineer at UC Berkeley. My class, club and personal projects will be posted here!
Thanks to Pupil Labs.
Comments
Please log in or sign up to comment.