Shubham Santosh
Published © GPL3+

Hand Gesture detection on ESP-CAM

Control LED brightness using hand/fist detection using ML on an ESP32-CAM

IntermediateFull instructions provided369

Things used in this project

Hardware components

ESP-CAM
×1

Software apps and online services

Edge Impulse Studio
Edge Impulse Studio
Arduino IDE
Arduino IDE

Story

Read more

Code

ESP Camera data Acquisition

Arduino
Arduino code for data acquisition from an ESP32-CAM. Data acquisition from a smartphone can also be used as an alternative.
/*********************************************************************
****** ESP-CAM Image acquisition on SD Card using web interface*******
************* By Shubham Santosh ************************************/

#include<WiFi.h>
#include "esp_camera.h"
#include "esp_timer.h"
#include "img_converters.h"
#include "Arduino.h"
#include "FS.h"  

#include<LittleFS.h>   // LittleFS for temp image storage
#include "SD_MMC.h"    // For permanenet storage
#include "soc/soc.h"           // Disable brownout problems
#include "soc/rtc_cntl_reg.h"  // Disable brownout problems
#include "driver/rtc_io.h"
#include <ESPAsyncWebServer.h> 

#include "esp_task_wdt.h" //disable watchdog timer during photo capture

// Camera pins initialization
#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27

#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22

#define LED_PIN 33
AsyncWebServer server(80);

const char* ssid="ENTER SSID";
const char* password="ENTER PASSWORD";

// Variabales used for flash. Does not work after SD card is initialized
String ledState="";  
bool led_state=0;

uint8_t image_number=0,image_log_no=0;
//char* parent_path="/photos";
char* image_name="/picture";
const char* input1="input1";

#define FILE_PHOTO "/image.jpg"

String path="";     //this variable stores the image path obtained from the webpage

// Get all the inputs from the webpage. For e.g. LED Flash state, file name,etc.
String processor(const String& var){
  Serial.println(var);
  if(var == "STATE"){
    if(led_state){
      ledState = "ON";
    }
    else{
      ledState = "OFF";
    }
    Serial.print(ledState);
    return ledState;
  }
  else if(var=="PHOTO_NUMBER")
  {
    return String(image_number);
  }
  else if(var== "FILE_NAME")
  {
    return String(path);
  }
  return String();
}

// Manipulate image path for unique identification
String image_path(String img_name)
{
path="/img_"+img_name+"_"+String(image_number)+".jpg";
return(path);
}

// Check if photo capture was successful
bool checkPhoto( fs::FS &fs ) {
  File f_pic = fs.open( FILE_PHOTO );
  unsigned int pic_sz = f_pic.size();
  Serial.println("File size checked:"+f_pic.size());
  return ( pic_sz > 100 );
}

//This function captures image from the OV2640 and stores in the Flash storage using LittleFS
void take_photo(void)
{
  esp_task_wdt_init(60, true);
   camera_fb_t * fb = NULL;
  // image_path(image_number);
    fb = esp_camera_fb_get();  
    if(!fb) 
    {
    Serial.println("Camera capture failed");
    return;
    }
    File file=LittleFS.open("/image.jpg",FILE_WRITE);
    if(!file)
    {
      Serial.println("Photo not saved in LITTE_FS");
    }
    else
    {
      file.write(fb->buf,fb->len);
      Serial.println("photo saved in LittleFS: /image.jpg");
      //Serial.print("Image path: ");
      //Serial.println(path);
     // Serial.print("Size: ");
      Serial.println(fb->len);
    }
     file.close();
    esp_camera_fb_return(fb);
   
}

// Main HTML page
char html[] =
  R"rawliteral(
  <!DOCTYPE html><html>
 <head>
        <title>ESP CAM Wi-Fi capture</title>
         </head>
          <body>
          <h1>ESP Camera Capture and Monitor</h1>  
       <p>
       <button onclick="capture_pic();">CAPTURE</button>
       <button onclick="location.reload();">RELOAD</button>
       </p>
       <form action="/get">
    Label Name: <input type="text" name="input1" autocomplete="on">
    <input type="submit" value="Save to SD card">
  </form><br>
       <p><button onclick="toggle_flash();">TOGGLE FLASH</button>Use LOAD to check FLASH state : %STATE%</p>
        <hr>
        <p>Number of photos captured: %PHOTO_NUMBER%</p>
        <p>File name saved as: %FILE_NAME%</p>
        <hr>
        <img src="photo" id="photo" width="70%"></div>
        </body>
        <script>
        function capture_pic()
        {
          var xhr= new XMLHttpRequest();
          xhr.open('GET',"/capture",true);
          xhr.send();
        }
         function toggle_flash()
        {
          var xhr= new XMLHttpRequest();
          xhr.open('GET',"/toggle_flash",true);
          xhr.send();
          
        }
        </script>
        </html> 
)rawliteral";


void setup() {
 //disable brownout detector
  Serial.begin(115200);
  
  pinMode(4,OUTPUT);  //LED Flash pin
  pinMode(33,OUTPUT); //Red LED pin behind the camera. Used as status indicator to check if wifi is connected.    


  //digitalWrite(4,HIGH);
  camera_config_t config;
  config.ledc_channel = LEDC_CHANNEL_0;
  config.ledc_timer = LEDC_TIMER_0;
  config.pin_d0 = Y2_GPIO_NUM;
  config.pin_d1 = Y3_GPIO_NUM;
  config.pin_d2 = Y4_GPIO_NUM;
  config.pin_d3 = Y5_GPIO_NUM;
  config.pin_d4 = Y6_GPIO_NUM;
  config.pin_d5 = Y7_GPIO_NUM;
  config.pin_d6 = Y8_GPIO_NUM;
  config.pin_d7 = Y9_GPIO_NUM;
  config.pin_xclk = XCLK_GPIO_NUM;
  config.pin_pclk = PCLK_GPIO_NUM;
  config.pin_vsync = VSYNC_GPIO_NUM;
  config.pin_href = HREF_GPIO_NUM;
  config.pin_sscb_sda = SIOD_GPIO_NUM;
  config.pin_sscb_scl = SIOC_GPIO_NUM;
  config.pin_pwdn = PWDN_GPIO_NUM;
  config.pin_reset = RESET_GPIO_NUM;
  config.xclk_freq_hz = 20000000;
  config.pixel_format = PIXFORMAT_JPEG; 
  
  WiFi.begin(ssid,password);
  while(WiFi.status()!=WL_CONNECTED)
  {
    digitalWrite(33,LOW);
    Serial.print(".");
    delay(200);
    digitalWrite(33,HIGH);
  }
  Serial.println(WiFi.localIP());  // pritn IP address
  WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0); 
  if(psramFound()){
    config.frame_size = FRAMESIZE_UXGA; // FRAMESIZE_ + QVGA|CIF|VGA|SVGA|XGA|SXGA|UXGA
    config.jpeg_quality = 10;
    config.fb_count = 2;
  } else {
    config.frame_size = FRAMESIZE_SVGA;
    config.jpeg_quality = 12;
    config.fb_count = 1;
  }
  //SD mmc begin and end here

//SD_MMC.end();
if (!LittleFS.begin(true)) {
  Serial.println("An Error has occurred while mounting LittleFS");
  ESP.restart();
}

  LittleFS.format();
  //LittleFS.mkdir(parent_path);

   esp_err_t err = esp_camera_init(&config);
  if (err != ESP_OK) {
    Serial.printf("Camera init failed with error 0x%x", err);
    return;
  }

  // HTTP 
   server.on("/", HTTP_GET, [](AsyncWebServerRequest * request) {
     
    request->send_P(200, "text/html",html,processor);
   });

   server.on("/get", HTTP_GET, [] (AsyncWebServerRequest *request) {
    String inputMessage;
    String inputParam;
    // GET input1 value on <ESP_IP>/get?input1=<inputMessage>
    if (request->hasParam(input1)) {
      inputMessage = request->getParam(input1)->value();
      inputParam = input1;
    }
    //Serial.println(inputMessage);
    image_path(inputMessage);
    if (!SD_MMC.begin()) {
  Serial.println("An Error has occurred while mounting SD Card");
  ESP.restart();
}
    //fs::FS &fs = SD_MMC;
    File saved_file = SD_MMC.open(path.c_str(), FILE_WRITE);
    File temp_file=LittleFS.open("/image.jpg",FILE_READ);
  if(!saved_file){
    Serial.println("Failed to open file in writing mode");
  } 
  else {
    size_t fileSize = temp_file.size();
    uint8_t buffer[512];  // image transferred in buffer size of 512 bytes
    size_t bytesRead;

    while (fileSize > 0) {
        bytesRead = temp_file.read(buffer, sizeof(buffer));
        saved_file.write(buffer, bytesRead);
        fileSize -= bytesRead;
    }
    Serial.printf("Saved file to path: %s\n", path.c_str());
    request->send(200, "text/html", "File saved as:" + path +
                                     "<br><a href=\"/\">Return to Home Page</a>");
  }
  saved_file.close();
  //esp_camera_fb_return(fb);
  temp_file.close();
  SD_MMC.end();
  //digitalWrite(4,LOW);
  //SD_MMC.close();
  });
 
    server.on("/toggle_flash", HTTP_GET, [](AsyncWebServerRequest * request) {
     led_state=1^led_state;
    request->send_P(200, "text/html","FLASH toggled");
   });


    server.on("/capture", HTTP_GET, [](AsyncWebServerRequest * request) {
    Serial.println("Photo Capturing");
    request->send_P(200, "text/plain","capturing pic");
    //image_path(image_number);
    
    //capturePhotoSaveSpiffs();
    
    // For every capture request, the pic is captured and stored in LittleFS 3 times.
    // Single iteration of capture also appears to store previous image.
    for(int i=0;i<3;i++)
    {
      digitalWrite(4,led_state);
      take_photo();
      digitalWrite(4,LOW);
    }

    image_number+=1;
  });


  server.on("/photo", HTTP_GET, [](AsyncWebServerRequest * request) {
    request->send(LittleFS,FILE_PHOTO,"image/jpg",false);

  });
  
 server.begin();
  // put your setup code here, to run once:
}

void loop() {
 delay(1);
}

ESP CAM gesture detection

Arduino
Main code. Changes LED(pin 33) brightness of ESP-CAM based on hand/fist detection.
/* Includes ---------------------------------------------------------------- */
//#include <ESP_CAM_inferencing.h>   // Model Library file
#include "edge-impulse-sdk/dsp/image/image.hpp"

#include "esp_camera.h"


#define PWDN_GPIO_NUM     32
#define RESET_GPIO_NUM    -1
#define XCLK_GPIO_NUM      0
#define SIOD_GPIO_NUM     26
#define SIOC_GPIO_NUM     27

#define Y9_GPIO_NUM       35
#define Y8_GPIO_NUM       34
#define Y7_GPIO_NUM       39
#define Y6_GPIO_NUM       36
#define Y5_GPIO_NUM       21
#define Y4_GPIO_NUM       19
#define Y3_GPIO_NUM       18
#define Y2_GPIO_NUM        5
#define VSYNC_GPIO_NUM    25
#define HREF_GPIO_NUM     23
#define PCLK_GPIO_NUM     22


/* Constant defines -------------------------------------------------------- */
#define EI_CAMERA_RAW_FRAME_BUFFER_COLS           320
#define EI_CAMERA_RAW_FRAME_BUFFER_ROWS           240
#define EI_CAMERA_FRAME_BYTE_SIZE                 3

#define LED_PIN 33

uint16_t brightness=0;  // LED brightness level
/* Private variables ------------------------------------------------------- */
static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal
static bool is_initialised = false;
uint8_t *snapshot_buf; //points to the output of the capture

static camera_config_t camera_config = {
    .pin_pwdn = PWDN_GPIO_NUM,
    .pin_reset = RESET_GPIO_NUM,
    .pin_xclk = XCLK_GPIO_NUM,
    .pin_sscb_sda = SIOD_GPIO_NUM,
    .pin_sscb_scl = SIOC_GPIO_NUM,

    .pin_d7 = Y9_GPIO_NUM,
    .pin_d6 = Y8_GPIO_NUM,
    .pin_d5 = Y7_GPIO_NUM,
    .pin_d4 = Y6_GPIO_NUM,
    .pin_d3 = Y5_GPIO_NUM,
    .pin_d2 = Y4_GPIO_NUM,
    .pin_d1 = Y3_GPIO_NUM,
    .pin_d0 = Y2_GPIO_NUM,
    .pin_vsync = VSYNC_GPIO_NUM,
    .pin_href = HREF_GPIO_NUM,
    .pin_pclk = PCLK_GPIO_NUM,

    //XCLK 20MHz or 10MHz for OV2640 double FPS (Experimental)
    .xclk_freq_hz = 20000000,
    .ledc_timer = LEDC_TIMER_0,
    .ledc_channel = LEDC_CHANNEL_0,

    .pixel_format = PIXFORMAT_JPEG, //YUV422,GRAYSCALE,RGB565,JPEG
    .frame_size = FRAMESIZE_QVGA,    //QQVGA-UXGA Do not use sizes above QVGA when not JPEG

    .jpeg_quality = 12, //0-63 lower number means higher quality
    .fb_count = 1,       //if more than one, i2s runs in continuous mode. Use only with JPEG
    .fb_location = CAMERA_FB_IN_PSRAM,
    .grab_mode = CAMERA_GRAB_WHEN_EMPTY,
};

/* Function definitions ------------------------------------------------------- */
bool ei_camera_init(void);
void ei_camera_deinit(void);
bool ei_camera_capture(uint32_t img_width, uint32_t img_height, uint8_t *out_buf) ;

/**
* @brief      Arduino setup function
*/
void setup()
{
    // put your setup code here, to run once:
    Serial.begin(115200);
    pinMode(LED_PIN,OUTPUT);
    //comment out the below line to start inference immediately after upload
    while (!Serial);
    Serial.println("Edge Impulse Inferencing Demo");
    if (ei_camera_init() == false) {
        ei_printf("Failed to initialize Camera!\r\n");
    }
    else {
        ei_printf("Camera initialized\r\n");
    }

    ei_printf("\nStarting continious inference in 2 seconds...\n");
    ei_sleep(2000);
}

/**
* @brief      Get data and run inferencing
*
* @param[in]  debug  Get debug info if true
*/
void loop()
{

    // instead of wait_ms, we'll wait on the signal, this allows threads to cancel us...
    if (ei_sleep(5) != EI_IMPULSE_OK) {
        return;
    }

    snapshot_buf = (uint8_t*)malloc(EI_CAMERA_RAW_FRAME_BUFFER_COLS * EI_CAMERA_RAW_FRAME_BUFFER_ROWS * EI_CAMERA_FRAME_BYTE_SIZE);

    // check if allocation was successful
    if(snapshot_buf == nullptr) {
        ei_printf("ERR: Failed to allocate snapshot buffer!\n");
        return;
    }

    ei::signal_t signal;
    signal.total_length = EI_CLASSIFIER_INPUT_WIDTH * EI_CLASSIFIER_INPUT_HEIGHT;
    signal.get_data = &ei_camera_get_data;

    if (ei_camera_capture((size_t)EI_CLASSIFIER_INPUT_WIDTH, (size_t)EI_CLASSIFIER_INPUT_HEIGHT, snapshot_buf) == false) {
        ei_printf("Failed to capture image\r\n");
        free(snapshot_buf);
        return;
    }

    // Run the classifier
    ei_impulse_result_t result = { 0 };

    EI_IMPULSE_ERROR err = run_classifier(&signal, &result, debug_nn);
    if (err != EI_IMPULSE_OK) {
        ei_printf("ERR: Failed to run classifier (%d)\n", err);
        return;
    }

    // print the predictions
    ei_printf("Predictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
                result.timing.dsp, result.timing.classification, result.timing.anomaly);

#if EI_CLASSIFIER_OBJECT_DETECTION == 1
    ei_printf("Object detection bounding boxes:\r\n");
    for (uint32_t i = 0; i < result.bounding_boxes_count; i++) {
        ei_impulse_result_bounding_box_t bb = result.bounding_boxes[i];
        if (bb.value == 0) {
            continue;
        }
        if(bb.label=="hand")
        brightness-=50;
        else if(bb.label=="fist")
        brightness+=50;
        if(brightness<=0)
        brightness=0;
        if(brightness>=255)
        brightness=255;
        ei_printf("  %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
                bb.label
                bb.value,
                bb.x,
                bb.y,
                bb.width,
                bb.height);
    }

    // Print the prediction results (classification)
#else
    ei_printf("Predictions:\r\n");
    for (uint16_t i = 0; i < EI_CLASSIFIER_LABEL_COUNT; i++) {
        ei_printf("  %s: ", ei_classifier_inferencing_categories[i]);
        ei_printf("%.5f\r\n", result.classification[i].value);
    }
#endif

    // Print anomaly result (if it exists)
#if EI_CLASSIFIER_HAS_ANOMALY
    ei_printf("Anomaly prediction: %.3f\r\n", result.anomaly);
#endif

#if EI_CLASSIFIER_HAS_VISUAL_ANOMALY
    ei_printf("Visual anomalies:\r\n");
    for (uint32_t i = 0; i < result.visual_ad_count; i++) {
        ei_impulse_result_bounding_box_t bb = result.visual_ad_grid_cells[i];
        if (bb.value == 0) {
            continue;
        }
        ei_printf("  %s (%f) [ x: %u, y: %u, width: %u, height: %u ]\r\n",
                bb.label,
                bb.value,
                bb.x,
                bb.y,
                bb.width,
                bb.height);
    }
#endif

    analogWrite(LED_PIN,brightness);
    free(snapshot_buf);

}

/**
 * @brief   Setup image sensor & start streaming
 *
 * @retval  false if initialisation failed
 */
bool ei_camera_init(void) {

    if (is_initialised) return true;

#if defined(CAMERA_MODEL_ESP_EYE)
  pinMode(13, INPUT_PULLUP);
  pinMode(14, INPUT_PULLUP);
#endif

    //initialize the camera
    esp_err_t err = esp_camera_init(&camera_config);
    if (err != ESP_OK) {
      Serial.printf("Camera init failed with error 0x%x\n", err);
      return false;
    }

    sensor_t * s = esp_camera_sensor_get();
    // initial sensors are flipped vertically and colors are a bit saturated
    if (s->id.PID == OV3660_PID) {
      s->set_vflip(s, 1); // flip it back
      s->set_brightness(s, 1); // up the brightness just a bit
      s->set_saturation(s, 0); // lower the saturation
    }

#if defined(CAMERA_MODEL_M5STACK_WIDE)
    s->set_vflip(s, 1);
    s->set_hmirror(s, 1);
#elif defined(CAMERA_MODEL_ESP_EYE)
    s->set_vflip(s, 1);
    s->set_hmirror(s, 1);
    s->set_awb_gain(s, 1);
#endif

    is_initialised = true;
    return true;
}

/**
 * @brief      Stop streaming of sensor data
 */
void ei_camera_deinit(void) {

    //deinitialize the camera
    esp_err_t err = esp_camera_deinit();

    if (err != ESP_OK)
    {
        ei_printf("Camera deinit failed\n");
        return;
    }

    is_initialised = false;
    return;
}


/**
 * @brief      Capture, rescale and crop image
 *
 * @param[in]  img_width     width of output image
 * @param[in]  img_height    height of output image
 * @param[in]  out_buf       pointer to store output image, NULL may be used
 *                           if ei_camera_frame_buffer is to be used for capture and resize/cropping.
 *
 * @retval     false if not initialised, image captured, rescaled or cropped failed
 *
 */
bool ei_camera_capture(uint32_t img_width, uint32_t img_height, uint8_t *out_buf) {
    bool do_resize = false;

    if (!is_initialised) {
        ei_printf("ERR: Camera is not initialized\r\n");
        return false;
    }

    camera_fb_t *fb = esp_camera_fb_get();

    if (!fb) {
        ei_printf("Camera capture failed\n");
        return false;
    }

   bool converted = fmt2rgb888(fb->buf, fb->len, PIXFORMAT_JPEG, snapshot_buf);

   esp_camera_fb_return(fb);

   if(!converted){
       ei_printf("Conversion failed\n");
       return false;
   }

    if ((img_width != EI_CAMERA_RAW_FRAME_BUFFER_COLS)
        || (img_height != EI_CAMERA_RAW_FRAME_BUFFER_ROWS)) {
        do_resize = true;
    }

    if (do_resize) {
        ei::image::processing::crop_and_interpolate_rgb888(
        out_buf,
        EI_CAMERA_RAW_FRAME_BUFFER_COLS,
        EI_CAMERA_RAW_FRAME_BUFFER_ROWS,
        out_buf,
        img_width,
        img_height);
    }


    return true;
}

static int ei_camera_get_data(size_t offset, size_t length, float *out_ptr)
{
    // we already have a RGB888 buffer, so recalculate offset into pixel index
    size_t pixel_ix = offset * 3;
    size_t pixels_left = length;
    size_t out_ptr_ix = 0;

    while (pixels_left != 0) {
        // Swap BGR to RGB here
        // due to https://github.com/espressif/esp32-camera/issues/379
        out_ptr[out_ptr_ix] = (snapshot_buf[pixel_ix + 2] << 16) + (snapshot_buf[pixel_ix + 1] << 8) + snapshot_buf[pixel_ix];

        // go to the next pixel
        out_ptr_ix++;
        pixel_ix+=3;
        pixels_left--;
    }
    // and done!
    return 0;
}

#if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_CAMERA
#error "Invalid model for current sensor"
#endif

Credits

Shubham Santosh

Shubham Santosh

13 projects • 100 followers
I am a 25 yo electronic enthusiast.

Comments

Related channels and tags
  • Edge Impulse
  • Machine Learning & AI
  • Photos & Video