Welcome to Hackster!
Hackster is a community dedicated to learning hardware, from beginner to pro. Join us, it's free!
v1ckery
Published © GPL3+

Game reactive LEDs with Python, OpenCV and Arduino

Using OpenCV screen detection, trigger LEDs based on something on your screen

IntermediateFull instructions provided416
Game reactive LEDs with Python, OpenCV and Arduino

Things used in this project

Hardware components

Arduino UNO
Arduino UNO
×1
Breadboard (generic)
Breadboard (generic)
×1
RGB Diffused Common Cathode
RGB Diffused Common Cathode
×1
Jumper wires (generic)
Jumper wires (generic)
×1
Male/Female Jumper Wires
Male/Female Jumper Wires
×1
Resistor 220 ohm
Resistor 220 ohm
×1

Story

Read more

Schematics

Wiring diagram

Simple RGB setup using the 6 available PWM pins

Code

mainDetection

Python
The object detection loops that utilize openCV and sends the commands to arduino
import cv2 as cv
import numpy as np
import os
import time
import serial
import keyboard

arduinoData = serial.Serial('com3',115200)

# imports screenshot function from other file
from windowCapture import WindowCapture
    
wincap = WindowCapture(None)

#Starts a timer to check for fps of the program
loop_time = time.time()

while(True):
    screenshot= wincap.get_screenshot()
    
    #can be adjusted to needs
    threshold = 0.9
    
    #renamed the max variable based on the ult name that is being detected
    RCneedle_img = cv.imread('Raincutter.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,RCneedle_img, cv.TM_CCOEFF_NORMED)
    min_val, RCmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    DBneedle_img = cv.imread('DandelionBreeze.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,DBneedle_img, cv.TM_CCOEFF_NORMED)
    min_val, DBmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    SSneedle_img = cv.imread('SakuraSwirl.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,SSneedle_img, cv.TM_CCOEFF_NORMED)
    min_val, SSmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    ROTneedle_img = cv.imread('ROTermination.png',cv.IMREAD_UNCHANGED)
    result = cv.matchTemplate(screenshot,ROTneedle_img, cv.TM_CCOEFF_NORMED)
    min_val, ROTmax_val, min_loc, max_loc = cv.minMaxLoc(result)

    if (RCmax_val>=threshold):
        cmd = "RC" + '\r'

    
    elif(DBmax_val>=threshold):
        cmd = "DB" + '\r'
        
    
    elif(ROTmax_val>=threshold):
        cmd = "ROT" + '\r'
        

    elif(SSmax_val>=threshold):
        cmd = "SS" + '\r'
         

    else:
        cmd = "No ult" + '\r'
        

                



    #after cmd is assigned a value, sent to arduino
    arduinoData.write(cmd.encode())
    
    #This prints the fps 
    print(f"FPS {1 / (time.time() - loop_time)} ")
    
    loop_time=time.time()
    
  
    # prints the command sent to the arduino for debugging purposes
    print(cmd)
    
    #killswitch for program, use any key
    if keyboard.is_pressed('u'):
        break



print('Done')

windowCapture

Python
Uses the win32api to take rapid screenshots and return them
import numpy as np
import os
import time

import win32gui, win32ui, win32con

class WindowCapture:
    
    #leave w,h as 0 for init
    #hwnd can be changed to your window name
    w = 0
    h = 0
    hwnd = None

    def __init__(self,window_name):

        self.hwnd = win32gui.FindWindow(None, window_name)
        if window_name is None:
            self.hwnd = win32gui.GetDesktopWindow()  
        if not self.hwnd:
            raise Exception(f'Window not found:{window_name}')
        
        # Size of the section of the screen that is being captured
        # Put monitor resolution for whole screen 
        self.h = 80
        self.w = 80


    def get_screenshot(self):
        

        
        
        wDC = win32gui.GetWindowDC(self.hwnd)
        dcObj=win32ui.CreateDCFromHandle(wDC)
        cDC=dcObj.CreateCompatibleDC()
        dataBitMap = win32ui.CreateBitmap()
        dataBitMap.CreateCompatibleBitmap(dcObj, self.w, self.h)
        cDC.SelectObject(dataBitMap)
        #Coordinates of the upper left corner of the section captured
        # Put 0,0 for whole screen
        cDC.BitBlt((0,0),(self.w, self.h) , dcObj, (1779,932), win32con.SRCCOPY)
    
        #Screenshot is unrecogonizable but this block of code converts it
        signedIntsArray = dataBitMap.GetBitmapBits(True)
        img = np.fromstring(signedIntsArray, dtype='uint8')
        img.shape = (self.h ,self.w ,4)


        # Frees resources by deleting frame data after each loop
        dcObj.DeleteDC()
        cDC.DeleteDC()
        win32gui.ReleaseDC(self.hwnd, wDC)
        win32gui.DeleteObject(dataBitMap.GetHandle())

        return img

templateMatching

Python
Used to get an idea of the needle location and confidence values
import cv2 as cv 

#reads the images
haystack_img = cv.imread('haystack.png',cv.IMREAD_UNCHANGED)
needle_img = cv.imread('needle.png',cv.IMREAD_UNCHANGED)

#assigns the results to variables
result = cv.matchTemplate(haystack_img,needle_img, cv.TM_CCOEFF_NORMED)

min_val, max_val, min_loc, max_loc = cv.minMaxLoc(result)


threshold = 0.9

if max_val >= threshold:
    print('Found needle')
    print(f'Best match top left position: {max_loc}')
    print(f'Best match confidence: {max_val}')

    #gets dimensions of needle image 
    needle_w = needle_img.shape[1]
    needle_h = needle_img.shape[0]
    
    #dimensions of rectangle
    top_left = max_loc
    bottom_right = (top_left[0] + needle_w, top_left[1] + needle_h )
    
    #draws rectangle over where the image is detected
    cv.rectangle(haystack_img, top_left, bottom_right,
    color=(0,255,0), thickness=2, lineType=cv.LINE_4)
    
    #displays result until a key is pressed
    cv.imshow('Result', haystack_img)
    cv.waitKey()

  
else:
    print('Needle not found')

Arduino File

Arduino
The code that allows the Arduino to read the commands from the serial port and triggers the LEDs
int rPin=6;
int gPin=3;
int bPin=5;

int rPin2=11;
int gPin2=10;
int bPin2=9;

int rVal;
int gVal;
int bVal;

int rVal2;
int gVal2;
int bVal2;


String mode;

void setup() {
  // put your setup code here, to run once:
Serial.begin(115200);
pinMode(rPin,OUTPUT);
pinMode(gPin,OUTPUT);
pinMode(bPin,OUTPUT);
pinMode(rPin2,OUTPUT);
pinMode(gPin2,OUTPUT);
pinMode(bPin2,OUTPUT);
}

void loop() {
  // put your main code here, to run repeatedly:

// reads the command from the serial
mode=Serial.readStringUntil('\r');


if (mode=="RC"){
analogWrite(rPin,0);
analogWrite(gPin,0);
analogWrite(bPin,70);
analogWrite(rPin2,0);
analogWrite(gPin2,0);
analogWrite(bPin2,70);
}

else if(mode== "SS"){
analogWrite(rPin,10);
analogWrite(gPin,20);
analogWrite(bPin,100);
analogWrite(rPin2,10);
analogWrite(gPin2,20);
analogWrite(bPin2,100);

}
else if (mode=="ROT"){
analogWrite(rPin,10);
analogWrite(gPin,20);
analogWrite(bPin,100);
analogWrite(rPin2,10);
analogWrite(gPin2,20);
analogWrite(bPin2,100);
}

else if (mode=="DB"){
analogWrite(rPin,10);
analogWrite(gPin,70);
analogWrite(bPin,0);
analogWrite(rPin2,10);
analogWrite(gPin2,70);
analogWrite(bPin2,0);
}

else{
analogWrite(rPin,0);
analogWrite(gPin,0);
analogWrite(bPin,0);
analogWrite(rPin2,0);
analogWrite(gPin2,0);
analogWrite(bPin2,0);
}

}

Credits

v1ckery
2 projects • 3 followers
Contact

Comments

Please log in or sign up to comment.