Hackster is hosting Hackster Holidays, Ep. 5: Livestream & Giveaway Drawing. Watch previous episodes or stream live on Friday!Stream Hackster Holidays, Ep. 5 on Friday!
Marcus
Published © GPL3+

C3PO - Interactive humanoid robot

This robot is my dream :-) Is fully interactive, it have moving parts and it interact with people using AI. I'll show you how I created it.

AdvancedShowcase (no instructions)Over 41 days318

Things used in this project

Hardware components

TB6600
×4
Visual Studio 2019
Microsoft Visual Studio 2019
×1
Camera Logitech C270
×1
Stepper motor
×6
Servo controler - MiniMaestro
×1
3d printer Artillery Sidewindex SW-X2
×1
3d printer Artillery Genius Pro
×1
Laser engraver SculpFun S10
×1
Arduino Mega 2560
×1

Software apps and online services

OpenCV
OpenCV
OpenCvSharp
Models of AI (head, hand, age, gender and emotions detections).

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)
CNC

Story

Read more

Schematics

C3PO components diagram

Code

Robot v4

C#
The most important parts of my C# code:
using OpenCvSharp;
using OpenCvSharp.Dnn;
using OpenCvSharp.Extensions;
...

//Prepare Nets
private OpenCvSharp.Dnn.Net _ageNet;
private OpenCvSharp.Dnn.Net _genderNet;
private OpenCvSharp.Dnn.Net _handNet;
...
_headNet = CvDnn.ReadNetFromCaffe(configFileFace, faceModel);
_ageNet = CvDnn.ReadNetFromCaffe(configFileAge, ageModel);
_genderNet = CvDnn.ReadNetFromCaffe(configFileGender, genderModel);
_handNet = CvDnn.ReadNetFromDarknet(_configHands, _handModel);

_handNet.SetPreferableBackend(OpenCvSharp.Dnn.Backend.OPENCV);
_handNet.SetPreferableTarget(OpenCvSharp.Dnn.Target.OPENCL);            
...

//Serial port to Arduino
 _serialPort = new SerialPort("COM11", 115200, Parity.None, 8, StopBits.One);
_serialPort.Handshake = Handshake.None;
_serialPort.ReadTimeout = 500;
_serialPort.WriteTimeout = 500;
_serialPort.Open();        
...

//Capture frame from camera
_capture = new OpenCvSharp.VideoCapture(0);
frame = new OpenCvSharp.Mat();
_cameraThread = new Thread(new ThreadStart(CaptureCameraThread));
_cameraThread.Start();
...
_capture.Read(frame);
if (frame.Empty()) return;
var imageResized = new OpenCvSharp.Mat();
Cv2.Resize(frame, imageResized, new Size(320, 240));
Mat imageToHead = imageResized.Clone();
Mat imageToHands = imageResized.Clone();
Mat imageToAgeGender = imageResized.Clone();

//Recognize hands
var blobHands = CvDnn.BlobFromImage(imageToHands, 1.0 / 255, new Size(608, 608), new Scalar(), true, false);
_handNet.SetInput(blobHands);

var outNames = _handNet.GetUnconnectedOutLayersNames();
var outs = outNames.Select(_ => new Mat()).ToArray();

_handNet.Forward(outs, outNames);

GetResult(outs, imageToHands, threshold, nmsThreshold);
...

//head tracking
private static int map(int value, int fromLow, int fromHigh, int toLow, int toHigh)
{
   return (value - fromLow) * (toHigh - toLow) / (fromHigh - fromLow) + toLow;
}

...

peopleCounter = 0;
Area = 0;
int indexMaxArea = 0;
//select person near camera
for (int i = 0; i < detectionMat.Rows; i++)
{
    float confidence = detectionMat.At<float>(i, 2);

    if (confidence > 0.9)
    {
        int x1 = (int)(detectionMat.At<float>(i, 3) * frameWidth);
        int y1 = (int)(detectionMat.At<float>(i, 4) * frameHeight);
        int x2 = (int)(detectionMat.At<float>(i, 5) * frameWidth);
        int y2 = (int)(detectionMat.At<float>(i, 6) * frameHeight);
         //select max area of rois
        if ((x2 - x1) * (y2 - y1) > Area)
        {
            Area = (x2 - x1) * (y2 - y1);
            indexMaxArea = i;
        }
        peopleCounter += 1;
      }
  }
  
if (_peopleCounter > 0)
 {
    int centerImageX = 160;
    int centerImageY = 120;
    int diff = 0;

    if (_XHeadDetection < centerImageX)
     {
        diff = centerImageX - _XHeadDetection;
        CenterPointX = map(centerImageX - diff, 0, 320, 1825, 2025);
     }
     else
     {
        diff = _XHeadDetection - centerImageX;
        CenterPointX = map(centerImageX + diff, 0, 320, 1825, 2025);
     }

     ushort stepInvX = (ushort)((tbHeadLP.Maximum - CenterPointX + tbHeadLP.Minimum) * 4);
     ushort stepY = (ushort)(CenterPointY * 4);
     //Function to control servos by MiniMaestro
     SetTargetServo(2, stepInvX);
     SetTargetServo(2, stepY);
     
     ...

//Age and gender recognize
private void fncAgeGender(int x1, int y1, int x2, int y2, OpenCvSharp.Mat imageResized, OpenCvSharp.Mat image)
{
  var x = x1 - Padding;
  if (x < 0)
    x = 0;
  var y = y1 - Padding;
  if (y < 0)
    y = 0;
  var w = (x2 - x1) + Padding * 3;
  var h = (y2 - y1) + Padding * 3;
  Rect roiNew = new Rect(x, y, w, h);
  if ((y + h) < 240 && (x + w) < 320)
  {
    var face = imageResized[roi: roiNew];

    var meanValues = new Scalar(78.4263377603, 87.7689143744, 114.895847746);
    var blobGender = CvDnn.BlobFromImage(face, 1.0, new Size(227, 227), mean: meanValues, swapRB: false);
    var blobAge = CvDnn.BlobFromImage(face, 1.0, new Size(227, 227), mean: meanValues, swapRB: false);

    _genderNet.SetInput(blobGender);
    var genderPreds = _genderNet.Forward();
    GetMaxClass(genderPreds, out int classId, out double classProbGender);
    var gender = _genderList[classId];

    _ageNet.SetInput(blobAge);
    var agePreds = _ageNet.Forward();
    GetMaxClass(agePreds, out int classIdAge, out double classProbAge);
    var age = _ageList[classIdAge];

    var label = $"{gender}-{age}";
    Cv2.PutText(image, label, new Point(10, 200), HersheyFonts.HersheyComplex, 2, Scalar.Red, 1);
  }
}


//Control MiniMaestro servo contoller
private static void SetTargetServo(Byte channel, UInt16 target)
{
  try
  {
    using (Usc device = connectToDevice())  // Find a device and temporarily connect.
    {
      device.setTarget(channel, target);
    }
  }
  catch (Exception exception)  // Handle exceptions by displaying them to the user.
    {
      displayException(exception);
    }
}
private static Usc connectToDevice()
{
  // Get a list of all connected devices of this type.
  List<DeviceListItem> connectedDevices = Usc.getConnectedDevices();

  foreach (DeviceListItem dli in connectedDevices)
  {
    Usc device = new Usc(dli); // Connect to the device.
    return device;             // Return the device.
  }
  throw new Exception("Could not find device.  Make sure it is plugged in to USB " +
                "and check your Device Manager");
}

Arduino full code for my C3PO.

Arduino
#include <AccelStepper.h>

#include <LiquidCrystal_I2C.h>
#include <Adafruit_NeoPixel.h>
#ifdef __AVR__
#include <avr/power.h>  // Required for 16 MHz Adafruit Trinket
#endif

#define LF          0x0A
LiquidCrystal_I2C lcd(0x27,20,4);

// Which pin on the Arduino is connected to the NeoPixels?
#define PIN 7  // On Trinket or Gemma, suggest changing this to 1

// How many NeoPixels are attached to the Arduino?
#define NUMPIXELS 6  // Popular NeoPixel ring size
Adafruit_NeoPixel pixels(NUMPIXELS, PIN, NEO_GRB + NEO_KHZ800);

#define DELAYVAL 111

char val[5];
int index = 0;
int directionMultiplier = 1;
bool newData = false;
bool runallowed = false;
String readString;
String motor_com;
String pos_com;
String speed_com;
String acceleration_com;
String side_comm;
int RightLeft = 1;
#define enArmR 6
#define enArmL 5
#define enHips_comm 4

AccelStepper stepperArmL(1, 38, 40); //PULL 8, DIR 9 46,48 - gacie, 44, 42 ArmL, 
AccelStepper stepperArmR(1, 42, 44); //PULL 8, DIR 9 46,48 - gacie, 44, 42 ArmL, 
AccelStepper stepperhips_comm(1, 46, 48);
//HardwareSerial Serial1 Serial1;

void setup() 
{
  Serial2.begin(115200);
  Serial2.println("C3PO running...");

  Serial.begin(115200);
  Serial.println("C3PO systems running...");

  pinMode(enArmR, OUTPUT); //enable motor pin
  pinMode(enArmL, OUTPUT); //enable motor pin
  pinMode(enHips_comm, OUTPUT); //enable motor pin
  
  stepperArmR.setMaxSpeed(400); //SPEED = Steps / second
  stepperArmR.setAcceleration(100); //ACCELERATION = Steps /(second)^2
  stepperArmL.setMaxSpeed(400); //SPEED = Steps / second
  stepperArmL.setAcceleration(100); //ACCELERATION = Steps /(second)^2
  stepperhips_comm.setMaxSpeed(400); //SPEED = Steps / second
  stepperhips_comm.setAcceleration(100); //ACCELERATION = Steps /(second)^2
    
  lcd.init();                      // initialize the lcd 
  lcd.init();
  lcd.backlight();
  lcd.setCursor(0,0);
  lcd.print("Hello from C3PO");
  runallowed = false;
  RunStopMotors("STOP");

    pixels.begin();  // INITIALIZE NeoPixel
    fBlink(10, 1);
    fClear();
    fColor(2);
    
}

void fClear()
{
    for (int i = 0; i < NUMPIXELS; i++) 
    {  
      pixels.setPixelColor(i, pixels.Color(0, 0, 0));
      pixels.show();  // Send the updated pixel colors to the hardware.
    }
}
void fBlink(int ile, int kolor)
{
  uint32_t colorUI[] = {pixels.Color(0, 0, 0), pixels.Color(255, 0, 0), pixels.Color(255, 255, 0)};
  for (int i = 0; i < ile; i++)
  {
    for (int i = 0; i < NUMPIXELS; i++) 
    {  
      pixels.setPixelColor(i, colorUI[0]);
      pixels.show(); 
    }
    pixels.show();
    delay(DELAYVAL);
    for (int i = 0; i < NUMPIXELS; i++) 
    {  
      pixels.setPixelColor(i, colorUI[kolor]);
    }
    pixels.show();  
    delay(DELAYVAL);
  }
}

void fColor(int kolor)
{
  uint32_t colorUI[] = {pixels.Color(0, 0, 0), pixels.Color(255, 0, 0), pixels.Color(255, 255, 0)};

    for (int i = 0; i < NUMPIXELS; i++) 
    {  
      pixels.setPixelColor(i, colorUI[kolor]);
      pixels.show();
    }
    pixels.show();
}

void loop() 
{
  checkSerial();
  RunTheMotor();
}

String getValue(String data, char separator, int index)
{
    int found = 0;
    int strIndex[] = { 0, -1 };
    int maxIndex = data.length() - 1;

    for (int i = 0; i <= maxIndex && found <= index; i++) {
        if (data.charAt(i) == separator || i == maxIndex) {
            found++;
            strIndex[0] = strIndex[1] + 1;
            strIndex[1] = (i == maxIndex) ? i+1 : i;
        }
    }
    return found > index ? data.substring(strIndex[0], strIndex[1]) : "";
}

void RunStopMotors(String STAN)
{
  if (STAN == "RUN")
  {
    digitalWrite(enArmR, HIGH);
    digitalWrite(enArmL, HIGH);
    digitalWrite(enHips_comm, HIGH);
  }
  else
  {
    runallowed = false;
    digitalWrite(enArmR, LOW);
    digitalWrite(enArmL, LOW);
    digitalWrite(enHips_comm, LOW);
  }
}


void checkSerial()
{
  if (Serial.available())
  {
     readString = Serial.readStringUntil('\n');

      if (readString.length() >0) 
      {
         Serial.println("Command: " + readString);
         motor_com = getValue(readString, ':', 0);
         pos_com = getValue(readString, ':', 1);
         speed_com = getValue(readString, ':', 2);
         acceleration_com = getValue(readString, ':', 3);
         
         lcd.setCursor(0,0);
         lcd.clear();
         lcd.print(motor_com);
         lcd.print(" ");
         lcd.print(pos_com);
         lcd.print(" ");
         lcd.print(speed_com);
         lcd.print(" ");
         lcd.print(acceleration_com);
         readString = "";

        if(motor_com == "ArmR")
        {
          directionMultiplier = 1;
          Serial.println("Right arm move to: " + String(directionMultiplier * pos_com.toInt()) );
          RunStopMotors("RUN");
          runallowed = true; //allow running - this allows entering the RunTheMotor() function.
          stepperArmR.setMaxSpeed(speed_com.toInt()); //set speed
          stepperArmR.setAcceleration(acceleration_com.toInt());
          stepperArmR.moveTo(directionMultiplier * pos_com.toInt()); //set relative distance  
          Serial.println("CURRENT POSR: " + String(stepperArmR.currentPosition()) );
        }
        else if(motor_com == "ArmL")
        {
          fClear();
          fColor(2);
          directionMultiplier = -1;
          Serial.println("Left arm move to: " + String(directionMultiplier * pos_com.toInt()) );
          RunStopMotors("RUN");
          runallowed = true;
          stepperArmL.setMaxSpeed(speed_com.toInt()); //set speed
          stepperArmL.setAcceleration(acceleration_com.toInt());
          stepperArmL.moveTo(directionMultiplier * pos_com.toInt()); //set relative distance  
          Serial.println("CURRENT POSL: " + String(stepperArmL.currentPosition()) );
        }
        else if(motor_com == "hips_comm") //hips_comm:-25000:50000:5000
        {
          directionMultiplier = -1;
          Serial.println("Hips move to: " + String(directionMultiplier * speed_com.toInt()) );
          RunStopMotors("RUN");
          runallowed = true; //allow running - this allows entering the RunTheMotor() function.
          stepperhips_comm.setMaxSpeed(speed_com.toInt()); //set speed
          stepperhips_comm.setAcceleration(acceleration_com.toInt());
          stepperhips_comm.moveTo(directionMultiplier * pos_com.toInt()); //set relative distance  
        }
        else if (motor_com == "A")
        {
          Serial.println("Left arm");
          RotateAbsolute();
        }
        else if (motor_com == "HOME")
        {
          Serial.println("Home");
          GoHome();
        }
        else if (motor_com == "O")
        {
          Serial.println(runallowed);
        }
        else if (motor_com == "r")
        {
          runallowed = true;
          Serial.println(runallowed);
        }
        else if (motor_com == "Z")
        {
          stepperArmR.setMaxSpeed(500);
          stepperArmR.setSpeed(500);
          stepperArmR.move(500);
        }
        else if (motor_com == "STOP")
        {
          Serial.println("DISABLED MOTORS");
          runallowed = false;
          RunStopMotors("STOP");
        }
        else if (motor_com == "LED")
        {
              fBlink(10, 2);
              fClear();
              fBlink(10, 1);
              fClear();
              fColor(2);
        }
        else if (motor_com == "LEDOFF")
        {
              fClear();
        }
      }
}
}

void RunTheMotor() //function for the motor
{
    if (runallowed == true)
    {
        RunStopMotors("RUN");
        stepperArmR.run();
        stepperArmL.run();
        stepperhips_comm.run();
    }
    else 
    {
        runallowed = false;
        RunStopMotors("STOP");
        return;
    }
}


void GoHome()
{  
    if (stepperArmR.currentPosition() == 0)
    {
        RunStopMotors("STOP");
    }
    else
    {
        runallowed = true; 
        RunStopMotors("RUN");
        stepperArmR.setMaxSpeed(200);
        stepperArmR.moveTo(0);
    }

    if (stepperArmR.currentPosition() == 0)
    {
        RunStopMotors("STOP");
    }
    else
    {
        runallowed = true; 
        RunStopMotors("RUN");
        stepperArmL.setMaxSpeed(200);
        stepperArmL.moveTo(0);
    }

} //end loop

 
void RotateRelative()
{
    //We move X steps from the current position of the stepperArmR motor in a given direction.
    //The direction is determined by the multiplier (+1 or -1)
    RunStopMotors("RUN");
    runallowed = true;
    stepperArmR.setMaxSpeed(acceleration_com.toInt()); //set speed
    stepperArmR.move(directionMultiplier * speed_com.toInt()); //set relative distance and direction
    stepperArmL.setMaxSpeed(acceleration_com.toInt()); //set speed
    stepperArmL.move(directionMultiplier * speed_com.toInt()); //set relative distance and direction
}
 
 
 
void RotateAbsolute()
{
    //We move to an absolute position.
    //The AccelstepperArmR library keeps track of the position.
    //The direction is determined by the multiplier (+1 or -1)
    acceleration_com = "400";
    speed_com = "1000";
    
    RunStopMotors("RUN");
    runallowed = true; //allow running - this allows entering the RunTheMotor() function.
    stepperArmR.setMaxSpeed(acceleration_com.toInt()); //set speed
    stepperArmR.moveTo(speed_com.toInt()); //set relative distance  
    stepperArmL.setMaxSpeed(acceleration_com.toInt()); //set speed
    stepperArmL.moveTo(speed_com.toInt()); //set relative distance
    //stepperArmR.disableOutputs(); 
    Serial.println("POSR: " + String(stepperArmR.currentPosition()) );
    Serial.println("CURRENT POSL: " + String(stepperArmL.currentPosition()) );
}

Credits

Marcus

Marcus

1 project • 0 followers

Comments