Welcome to Hackster!
Hackster is a community dedicated to learning hardware, from beginner to pro. Join us, it's free!
Brendon Hales
Published

Finger-based Servo Control

Using a mixture of computer vision, embedded systems, and servo motors, you can move a servo using only the number of fingers up on a hand.

IntermediateFull instructions provided2 hours2,898
Finger-based Servo Control

Things used in this project

Hardware components

Adafruit Mini Pan-Tilt Kit - Assembled with Micro Servos
×1
Arduino Uno Rev3
×1
Solderless Breadboard 830 (Hobby Line)
×1
Jumper Wires, Male/Male, 40 Conductors x 8″ (20cm)
×1

Software apps and online services

Arduino IDE
Microsoft Visual Studio Code

Story

Read more

Schematics

Wire layout between servos and Arduino

Utilized pins 9 and 10 for the X and Y data connection on the servos, respectively, and the 5V and the GND on the Arduino to power the servos. Arduino is powered via USB connection to the host computer (not shown)

Code

Arduino Code

Arduino
This is the code that will be ran on the Arduino board. In this case, it is ran on an Arduino Uno R3. Change pin layouts according to the board you are using.
*Note* The servos that are being used have a limit in the x-direction of 0-180 degrees, and the y direction of 90-180 degrees.
/*
  Servo Motion Based on Finger Tracking
*/
#include <Servo.h>

// Servos
Servo xServo;
Servo yServo;

// Global variables
int xPos = 0; // Variable for the x servo position
int yPos = 90;  // Variable for the y servo position
String dataFromPython;  // Variable to hold data transmitted from Python

void servoMove(String data)
{
  // Horizontal
  // Left
  // Moves the x servo left by 10 degrees until the limit of 180 degrees is reached
  if (data == "Left 10")
  {
    if ((xPos + 10) <= 180)
    {
      xPos = xPos + 10;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(180);
      delay(15);
    }
  }
  // Moves the x servo left by 20 degrees until the limit of 180 degrees is reached
  else if (data == "Left 20")
  {
    if ((xPos + 20) <= 180)
    {
      xPos = xPos + 20;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(180);
      delay(15);
    }
  }
  // Moves the x servo left by 30 degrees until the limit of 180 degrees is reached
  else if (data == "Left 30")
  {
    if ((xPos + 30) <= 180)
    {
      xPos = xPos + 30;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(180);
      delay(15);
    }
  }
  // Moves the x servo left by 40 degrees until the limit of 180 degrees is reached
  else if (data == "Left 40")
  {
    if ((xPos + 40) <= 180)
    {
      xPos = xPos + 40;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(180);
      delay(15);
    }
  }
  // Moves the x servo left by 50 degrees until the limit of 180 degrees is reached
  else if (data == "Left 50")
  {
    if ((xPos + 50) <= 180)
    {
      xPos = xPos + 50;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(180);
      delay(15);
    }
  }
  // Right
  // Moves the x servo right by 10 degrees until the limit of 180 degrees is reached
  else if (data == "Right 10")
  {
    if ((xPos - 10) >= 0)
    {
      xPos = xPos - 10;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(0);
      delay(15);
    }
  }
  // Moves the x servo right by 20 degrees until the limit of 180 degrees is reached
  else if (data == "Right 20")
  {
    if ((xPos - 20) >= 0)
    {
      xPos = xPos - 20;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(0);
      delay(15);
    }
  }
  // Moves the x servo right by 30 degrees until the limit of 180 degrees is reached
  else if (data == "Right 30")
  {
    if ((xPos - 30) >= 0)
    {
      xPos = xPos - 30;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(0);
      delay(15);
    }
  }
  // Moves the x servo right by 40 degrees until the limit of 180 degrees is reached
  else if (data == "Right 40")
  {
    if ((xPos - 40) >= 0)
    {
      xPos = xPos - 40;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(0);
      delay(15);
    }
  }
  // Moves the x servo right by 50 degrees until the limit of 180 degrees is reached
  else if (data == "Right 50")
  {
    if ((xPos - 50) >= 0)
    {
      xPos = xPos - 50;
      xServo.write(xPos);
      delay(15);
    }
    else
    {
      xServo.write(0);
      delay(15);
    }
  }
  // Vertical
  // Up
  // Moves the y servo up by 5 degrees until the limit of 180 degrees is reached
  else if (data == "Right Left 5" || data == "Left Right 5")
  {
    if ((yPos + 5) <= 180)
    {
      yPos = yPos + 5;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(180);
      delay(15);
    }
  }
  // Moves the y servo up by 10 degrees until the limit of 180 degrees is reached
  else if (data == "Right Left 10" || data == "Left Right 10")
  {
    if ((yPos + 10) <= 180)
    {
      yPos = yPos + 10;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(180);
      delay(15);
    }
  }
  // Moves the y servo up by 15 degrees until the limit of 180 degrees is reached
  else if (data == "Right Left 15" || data == "Left Right 15")
  {
    if ((yPos + 15) <= 180)
    {
      yPos = yPos + 15;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(180);
      delay(15);
    }
  }
  // Moves the y servo up by 20 degrees until the limit of 180 degrees is reached
  else if (data == "Right Left 20" || data == "Left Right 20")
  {
    if ((yPos + 20) <= 180)
    {
      yPos = yPos + 20;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(180);
      delay(15);
    }
  }
  // Moves the y servo up by 25 degrees until the limit of 180 degrees is reached
  else if (data == "Right Left 25" || data == "Left Right 25")
  {
    if ((yPos + 25) <= 180)
    {
      yPos = yPos + 25;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(180);
      delay(15);
    }
  }
  // Down
  // Moves the y servo down by 5 degrees until the limit of 90 degrees is reached
  else if (data == "Right Left -5" || data == "Left Right -5")
  {
    if ((yPos - 5) >= 90)
    {
      yPos = yPos - 5;
      yServo.write(yPos);
      delay(15);
    }
    else
    {
      yServo.write(90);
      delay(15);
    }
  }
  else
  {
    xServo.write(xPos);
    yServo.write(yPos);
    delay(15);
  }
}

void setup()
{
  xServo.attach(9); // Attaches the X servo on pin 9
  yServo.attach(10);  // Sttaches the y servo on pin 10
  Serial.begin(9600); // Initializes serial communication with baud rate of 9600
  xServo.write(0);  // Initializes X Direction Servo at 0 degrees
  yServo.write(90); // Initializes Y DIrection Servo at 90 degrees
}

void loop()
{
  while (Serial.available())    //whatever the data that is coming in serially and assigning the value to the variable data
  {
    dataFromPython = Serial.readStringUntil('\n');  // Reads incoming data from Python until a new line character is reached
    // Serial.write("Successful data transmission \n"); // Testing
    servoMove(dataFromPython);  // Calls servoMove funciton
  }
}

Python Code

Python
This is the code that was ran from the host computer that was connected to the Arduino via USB. This code utilizes OpenCV and CVZone libraries to perform the finger tracking. Once the number of fingers is determined, the data is passed to the Arduino to move the servo.
# Servo Motion Based on Finger Tracking

# Imports for OpenCV, time, and serial communication to the Arduino
import cv2, time
from serial import Serial

# Import for hand detection
from cvzone.HandTrackingModule import HandDetector

# Initialize Arduino Communication
ArduinoSerial = Serial('com4', 9600, timeout=0.1)
time.sleep(1)

# Determines movement angle based on the number of fingers held up on one hands
def oneHandIdentification(oneHandType, oneHandFingers):
    handIdentifier = oneHandType
    fingerIdentifier = oneHandFingers
    movementAngle = 0
    # One finger up on one hand
    if fingerIdentifier == [1, 0, 0, 0, 0]:
        movementAngle = 10
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 1, 0, 0, 0]:
        movementAngle = 10
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 1, 0, 0]:
        movementAngle = 10
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 0, 1, 0]:
        movementAngle = 10
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 0, 0, 1]:
        movementAngle = 10
        moveXArduino(handIdentifier, movementAngle)
    # Two fingers up on one hand
    elif fingerIdentifier == [1, 1, 0, 0, 0]:
        movementAngle = 20
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 1, 1, 0, 0]:
        movementAngle = 20
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 1, 1, 0]:
        movementAngle = 20
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 0, 1, 1]:
        movementAngle = 20
        moveXArduino(handIdentifier, movementAngle)
    # Three fingers up on one hand
    elif fingerIdentifier == [1, 1, 1, 0, 0]:
        movementAngle = 30
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 1, 1, 1, 0]:
        movementAngle = 30
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 0, 1, 1, 1]:
        movementAngle = 30
        moveXArduino(handIdentifier, movementAngle)
    # Four fingers up on one hand
    elif fingerIdentifier == [1, 1, 1, 1, 0]:
        movementAngle = 40
        moveXArduino(handIdentifier, movementAngle)
    elif fingerIdentifier == [0, 1, 1, 1, 1]:
        movementAngle = 40
        moveXArduino(handIdentifier, movementAngle)
    # Five fingers up on one hand
    elif fingerIdentifier == [1, 1, 1, 1, 1]:
        movementAngle = 50
        moveXArduino(handIdentifier, movementAngle)

# Determines movement angle based on the number of fingers held up on two hands
def twoHandIdentification(twoHandType1, twoHandFingers1, twoHandType2, twoHandFingers2):
    hand1Identifier = twoHandType1
    finger1Identifier = twoHandFingers1
    hand2Identifier = twoHandType2
    finger2Identifier = twoHandFingers2
    yMovementAngle = 0
    # One finger up on both hands (same finger)
    if finger1Identifier == [1, 0, 0, 0, 0] and finger2Identifier == [1, 0, 0, 0, 0]:
        yMovementAngle = 5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 1, 0, 0, 0] and finger2Identifier == [0, 1, 0, 0, 0]:
        yMovementAngle = 5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 1, 0, 0] and finger2Identifier == [0, 0, 1, 0, 0]:
        yMovementAngle = 5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 0, 1, 0] and finger2Identifier == [0, 0, 0, 1, 0]:
        yMovementAngle = 5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 0, 0, 1] and finger2Identifier == [0, 0, 0, 0, 1]:
        yMovementAngle = 5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    # Two fingers up on both hands (same fingers)
    elif finger1Identifier == [1, 1, 0, 0, 0] and finger2Identifier == [1, 1, 0, 0, 0]:
        yMovementAngle = 10
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 1, 1, 0, 0] and finger2Identifier == [0, 1, 1, 0, 0]:
        yMovementAngle = 10
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 1, 1, 0] and finger2Identifier == [0, 0, 1, 1, 0]:
        yMovementAngle = 10
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 0, 1, 1] and finger2Identifier == [0, 0, 0, 1, 1]:
        yMovementAngle = 10
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    # Three fingers up on both hands (same fingers)
    elif finger1Identifier == [1, 1, 1, 0, 0] and finger2Identifier == [1, 1, 1, 0, 0]:
        yMovementAngle = 15
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 1, 1, 1, 0] and finger2Identifier == [0, 1, 1, 1, 0]:
        yMovementAngle = 15
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 0, 1, 1, 1] and finger2Identifier == [0, 0, 1, 1, 1]:
        yMovementAngle = 15
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    # Four fingers up on both hands (same fingers)
    elif finger1Identifier == [1, 1, 1, 1, 0] and finger2Identifier == [1, 1, 1, 1, 0]:
        yMovementAngle = 20
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    elif finger1Identifier == [0, 1, 1, 1, 1] and finger2Identifier == [0, 1, 1, 1, 1]:
        yMovementAngle = 20
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    # Five fingers up on both hands (same fingers)
    elif finger1Identifier == [1, 1, 1, 1, 1] and finger2Identifier == [1, 1, 1, 1, 1]:
        yMovementAngle = 25
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)
    # No fingers up on both hands
    elif finger1Identifier == [0, 0, 0, 0, 0] and finger2Identifier == [0, 0, 0, 0, 0]:
        yMovementAngle = -5
        moveYArduino(hand1Identifier, hand2Identifier, yMovementAngle)

# Sends data to move the x servo to the Arduino
def moveXArduino(handIdentifier, movementAngle):
    string = handIdentifier + ' ' + str(movementAngle) + '\n'
    #print("Python", string) # Testing
    ArduinoSerial.write(bytes(string, 'utf-8'))
    #print(ArduinoSerial.read_until('\n'))   # Testing
    time.sleep(.015)

# Sends data to move the y servo to the Arduino
def moveYArduino(hand1Identifier, hand2Identifier, movementAngle):
    string = hand1Identifier + ' ' + hand2Identifier + ' ' + str(movementAngle) + '\n'
    #print("Python", string) # Testing
    ArduinoSerial.write(bytes(string, 'utf-8'))
    #print(ArduinoSerial.read_until('\n'))   # Testing 
    time.sleep(.015)

# Start of main function
# Initiate webcam video capture
webcam = cv2.VideoCapture(0)

# Hand Detection
handDetect = HandDetector(detectionCon=0.8, maxHands=2)

while webcam.isOpened():
    ret, frame = webcam.read()
    foundHands, bbox = handDetect.findHands(frame)

    if foundHands:   # If hands are found in the frame
        if len(foundHands) == 1: # If only one hand is found in the frame
            # 1 hand
            oneHand = foundHands[0]
            oneHandType = oneHand["type"]  # Left or Right

            # Determine number of fingers up on hand
            # Outputs array where the thumb is the most left location, and pinky is the furthest right
            # Example: Thumb ring finger up [1, 0, 0, 1, 0]
            oneHandFingers = handDetect.fingersUp(oneHand)
            #print(handType1, fingers1)  # Testing
            oneHandIdentification(oneHandType, oneHandFingers)
        elif len(foundHands) == 2:   # If 2 hands are found in the frame  
            # 2 hands
            hand1 = foundHands[0]
            hand2 = foundHands[1]
            handType1 = hand1["type"]   # Left or Right
            handType2 = hand2["type"]   # Left or Right
            # Determine number of fingers up on each hand
            # Outputs array where the thumb is the most left location, and pinky is the furthest right
            # Example: Thumb ring finger up [1, 0, 0, 1, 0]
            fingers1 = handDetect.fingersUp(hand1)
            fingers2 = handDetect.fingersUp(hand2)
            #print(handType1, fingers1, handType2, fingers2) # Testing
            twoHandIdentification(handType1, fingers1, handType2, fingers2)
    
    cv2.imshow('Press "ESC" to quit', frame)    # Output webcam feed
    # press 'ESC' to Quit
    if cv2.waitKey(1) == 27:
        break

webcam.release()
cv2.destroyAllWindows()

Credits

Brendon Hales
1 project • 0 followers
Contact

Comments

Please log in or sign up to comment.