GOWTHAM SBalamurugan.K
Created August 30, 2024

Smart First Aid Kit For Visual Impaired Indvidual's

Smart First Aid Kit for the Visually Impaired: Voice-activated box opens with precise guidance, ensuring the right medication every time.

122
Runnerup Visual Impairments
Build2gether 2.0 — Inclusive Innovation Challenge
Smart First Aid Kit For Visual Impaired Indvidual's

Things used in this project

Hardware components

nRF52 Development Kit
Nordic Semiconductor nRF52 Development Kit
×1
Adafruit STM32F411CEU6 Black Pill Development Board
×1
Blues Notecarrier A
Blues Notecarrier A
×1
Blues Notecard Wi-Fi v1
×1
SG90 Micro-servo motor
SG90 Micro-servo motor
×3
3.85v Lithium ion Batteries 120mah 2000mah 5000mah 10000mah 2s 3s 4s 3.7v 7.4v 11.1v 14.8v Lithium Li Polymer Lipo Battery
×1
Seeed Studio XIAO ESP32S3 Sense
Seeed Studio XIAO ESP32S3 Sense
×1

Software apps and online services

Vercel
Blues Notehub.io
Blues Notehub.io
Mongo DB Atlas cloud
Visual Studio 2017
Microsoft Visual Studio 2017
nRF Connect SDK
Nordic Semiconductor nRF Connect SDK
STM32CUBEPROG
STMicroelectronics STM32CUBEPROG
STMicroelectronics STMCUBE IDE

Hand tools and fabrication machines

Soldering iron (generic)
Soldering iron (generic)
Solder Wire, Lead Free
Solder Wire, Lead Free
Premium Female/Male Extension Jumper Wires, 40 x 6" (150mm)
Premium Female/Male Extension Jumper Wires, 40 x 6" (150mm)
10 Pc. Jumper Wire Kit, 5 cm Long
10 Pc. Jumper Wire Kit, 5 cm Long
3D Printer (generic)
3D Printer (generic)

Story

Read more

Schematics

System Process outline

Code

Nordic Nrf52840 code

C/C++
#include <zephyr.h>
#include <device.h>
#include <drivers/gpio.h>
#include <drivers/uart.h>
#include <string.h>
#include <net/http_client.h>

#define UART_DEV_LABEL "UART_1"  
#define UART_XIAO_DEV_LABEL "UART_2"  // New UART for XIAO ESP32S3 Sense
#define SERVO_PIN_1 DT_ALIAS_SERVO1_GPIOS_PIN
#define SERVO_PIN_2 DT_ALIAS_SERVO2_GPIOS_PIN
#define SERVO_PIN_3 DT_ALIAS_SERVO3_GPIOS_PIN
#define SPEAKER_PIN DT_ALIAS_SPEAKER_GPIOS_PIN

#define MAX_BOX_DATA_LENGTH 50
#define UART_BUFFER_SIZE 256
#define GPT_API_URL "http://your_gpt_api_url_here"
#define API_KEY "your_api_key_here"

char box1[MAX_BOX_DATA_LENGTH] = {0};
char box2[MAX_BOX_DATA_LENGTH] = {0};
char box3[MAX_BOX_DATA_LENGTH] = {0};

char uart_buffer[UART_BUFFER_SIZE];
char uart_xiao_buffer[UART_BUFFER_SIZE];  // Buffer for XIAO ESP32S3 Sense UART
const struct device *uart_dev;
const struct device *uart_xiao_dev;  // UART device for XIAO ESP32S3 Sense
const struct device *gpio_dev;

void init_peripherals() {
    uart_dev = device_get_binding(UART_DEV_LABEL);
    uart_xiao_dev = device_get_binding(UART_XIAO_DEV_LABEL);  // Initialize second UART
    gpio_dev = device_get_binding(DT_LABEL(DT_NODELABEL(gpio0)));

    if (!device_is_ready(gpio_dev)) {
        printk("GPIO device not found!\n");
        return;
    }

    gpio_pin_configure(gpio_dev, SERVO_PIN_1, GPIO_OUTPUT);
    gpio_pin_configure(gpio_dev, SERVO_PIN_2, GPIO_OUTPUT);
    gpio_pin_configure(gpio_dev, SERVO_PIN_3, GPIO_OUTPUT);
    gpio_pin_configure(gpio_dev, SPEAKER_PIN, GPIO_OUTPUT);
}

void parse_box_data(const char *data) {
    sscanf(data, "box1=%49[^;];box2=%49[^;];box3=%49[^;];", box1, box2, box3);
    printk("Parsed box data: box1=%s, box2=%s, box3=%s\n", box1, box2, box3);
}

void generate_and_play_audio(const char *box_data) {
    char request_buffer[256];
    sprintf(request_buffer, "{\"prompt\": \"Details for %s\", \"max_tokens\": 100}", box_data);

    struct http_request req = {
        .method = HTTP_POST,
        .url = GPT_API_URL,
        .header = "Content-Type: application/json\r\nAuthorization: Bearer " API_KEY "\r\n",
        .body = request_buffer,
    };

    struct http_response res;
    int ret = http_client_post(&req, &res);

    if (ret == 0 && res.status_code == 200) {
        strncpy(uart_buffer, res.body, sizeof(uart_buffer) - 1);
        uart_buffer[sizeof(uart_buffer) - 1] = '\0';
        play_audio(uart_buffer);
    } else {
        printk("Error: %d\n", res.status_code);
    }
}

void play_audio(const char *text) {
    printk("Playing audio: %s\n", text);
    gpio_pin_set(gpio_dev, SPEAKER_PIN, 1);
    k_sleep(K_MSEC(2000));
    gpio_pin_set(gpio_dev, SPEAKER_PIN, 0);
}

void control_servo(const char *text) {
    if (strstr(text, box1) != NULL) {
        gpio_pin_set(gpio_dev, SERVO_PIN_1, 1);
        generate_and_play_audio(box1);
    } else if (strstr(text, box2) != NULL) {
        gpio_pin_set(gpio_dev, SERVO_PIN_2, 1);
        generate_and_play_audio(box2);
    } else if (strstr(text, box3) != NULL) {
        gpio_pin_set(gpio_dev, SERVO_PIN_3, 1);
        generate_and_play_audio(box3);
    } else {
        printk("No matching box data found.\n");
    }
}

void uart_thread_fn(void) {
    while (1) {
        int len = uart_poll_in(uart_dev, uart_buffer, sizeof(uart_buffer) - 1);
        if (len > 0) {
            uart_buffer[len] = '\0';
            if (strstr(uart_buffer, "box1=") != NULL) {
                parse_box_data(uart_buffer);
            } else {
                control_servo(uart_buffer);
            }
        }
        k_sleep(K_MSEC(500));
    }
}

void uart_xiao_thread_fn(void) {
    while (1) {
        int len = uart_poll_in(uart_xiao_dev, uart_xiao_buffer, sizeof(uart_xiao_buffer) - 1);
        if (len > 0) {
            uart_xiao_buffer[len] = '\0';
            printk("Received from XIAO ESP32S3 Sense: %s\n", uart_xiao_buffer);
            control_servo(uart_xiao_buffer);
        }
        k_sleep(K_MSEC(500));
    }
}

K_THREAD_DEFINE(uart_thread, 1024, uart_thread_fn, NULL, NULL, NULL, 7, 0, 0);
K_THREAD_DEFINE(uart_xiao_thread, 1024, uart_xiao_thread_fn, NULL, NULL, NULL, 7, 0, 0);

void main() {
    init_peripherals();
    while (1) {
        k_sleep(K_MSEC(1000));
    }
}

STM32F411 black pill Arduino code

C/C++
#include <Arduino.h>
#include <HardwareSerial.h>
#include <ArduinoJson.h>
#include<Notecard.h>

#define UART1_BAUD 115200  
#define UART2_BAUD 115200   
#define NOTE_CARRIER_ID "YOUR_CARRIER_ID" 
#define NOTE_AUTH_TOKEN "YOUR_AUTH_TOKEN" 


HardwareSerial notecardSerial(1);  
HardwareSerial nordicSerial(2);   

// Buffer sizes
const size_t bufferSize = 256;
char jsonBuffer[bufferSize];

void setup() {
    Serial.begin(115200);
    notecardSerial.begin(UART1_BAUD, SERIAL_8N1, 30, 31); 
    nordicSerial.begin(UART2_BAUD, SERIAL_8N1, 12, 13); 
    Serial.println("ESP32F411 initialized and ready to communicate with Notecard and Nordic.");

    
    initializeNotecard();
}

void loop() {
    
    if (notecardSerial.available()) {
        size_t len = notecardSerial.readBytesUntil('\n', jsonBuffer, sizeof(jsonBuffer) - 1);
        jsonBuffer[len] = '\0';

        Serial.print("Received JSON Data: ");
        Serial.println(jsonBuffer);

        
        processJsonData(jsonBuffer);
    }

    
    requestDataFromNotehub();
    delay(10000); 
}

void initializeNotecard() {
    
    sendCommand("{\"req\": \"hub.set\", \"token\": \"" NOTE_AUTH_TOKEN "\"}");
    sendCommand("{\"req\": \"card.set\", \"mode\": \"cellular\", \"pwr\": 1}");
}

void sendCommand(const char* command) {
    notecardSerial.println(command);
    Serial.print("Sent command: ");
    Serial.println(command);
}

void requestDataFromNotehub() {
    String request = "{\"req\": \"note.get\", \"hub\": \"" NOTE_CARRIER_ID "\"}";
    sendCommand(request.c_str());
}

void processJsonData(const char* jsonData) {
 
    DynamicJsonDocument doc(bufferSize);
    DeserializationError error = deserializeJson(doc, jsonData);

    if (error) {
        Serial.print("Failed to parse JSON: ");
        Serial.println(error.c_str());
        return;
    }

  
    String box1 = doc["box1"].as<String>();
    String box2 = doc["box2"].as<String>();
    String box3 = doc["box3"].as<String>();

    Serial.print("Box1: ");
    Serial.println(box1);
    Serial.print("Box2: ");
    Serial.println(box2);
    Serial.print("Box3: ");
    Serial.println(box3);

   
    sendDataToNordic(box1, box2, box3);
}

void sendDataToNordic(const String& box1, const String& box2, const String& box3) {
   

    nordicSerial.println(box1);
    nordicSerial.println(box2);
    nordicSerial.println(box3);
    Serial.print("Sent data to Nordic: ");
    Serial.println(box1);
    Serial.println(box2);
    Serial.println(box3);
}

XIAOEsp32sense

C/C++
#include <WiFi.h>
#include <HTTPClient.h>
#include <HardwareSerial.h>
#include <esp_camera.h>
#include <EdgeImpulse.h>


const char* ssid = "your_SSID";
const char* password = "your_PASSWORD";
const char* apiKey = "your_API_key";  

const char* speechToTextUrl = "https://speech.googleapis.com/v1p1beta1/speech:recognize?key=" + String(apiKey);
const char* ocrApiUrl = "http://your-ocr-api-endpoint";


camera_config_t camera_config = {
    .pin_pwdn = -1,
    .pin_reset = -1,
    .pin_xclk = 21,
    .pin_sccb_sda = 26,
    .pin_sccb_scl = 27,
    .pin_d7 = 35,
    .pin_d6 = 34,
    .pin_d5 = 33,
    .pin_d4 = 32,
    .pin_d3 = 39,
    .pin_d2 = 36,
    .pin_d1 = 23,
    .pin_d0 = 22,
    .pin_vsync = 25,
    .pin_href = 19,
    .pin_pclk = 18,
    .xclk_freq_hz = 20000000,
    .ledc_channel = LEDC_CHANNEL_0,
    .ledc_timer = LEDC_TIMER_0,
    .frame_size = FRAMESIZE_SVGA,
    .jpeg_quality = 12,
    .fb_count = 1
};


EdgeImpulse ei;
#define SAMPLE_RATE 16000U
#define SAMPLE_BITS 16
bool record_status = true;


#define UART1_BAUD 115200
HardwareSerial mySerial(1);

void connect_wifi() {
    WiFi.begin(ssid, password);
    Serial.print("Connecting to WiFi");
    while (WiFi.status() != WL_CONNECTED) {
        delay(500);
        Serial.print(".");
    }
    Serial.println("Connected");
}

void setup_camera() {
    esp_err_t err = esp_camera_init(&camera_config);
    if (err != ESP_OK) {
        Serial.printf("Camera Init Failed: %s\n", esp_err_to_name(err));
        return;
    }
}

void setup_voice_recognition() {
    if (!ei.begin()) {
        Serial.println("Failed to initialize Edge Impulse!");
        while (1);
    }
}


String audioToBase64(const uint8_t *audio_data, size_t size) {
    
    return "";
}

void send_audio_to_server(const uint8_t *audio_data, size_t size) {
    if (WiFi.status() == WL_CONNECTED) {
        HTTPClient http;
        http.begin(speechToTextUrl);
        http.addHeader("Content-Type", "application/json");

        String audioBase64 = audioToBase64(audio_data, size); 
        String jsonPayload = "{\"config\": {\"encoding\": \"LINEAR16\", \"sampleRateHertz\": 16000, \"languageCode\": \"en-US\"}, \"audio\": {\"content\": \"" + audioBase64 + "\"}}";

        int httpResponseCode = http.POST(jsonPayload);

        if (httpResponseCode > 0) {
            String response = http.getString();
            Serial.println("Speech-to-Text Response:");
            Serial.println(response);

          
            String recognized_text = extract_text_from_response(response);

            
            mySerial.print("Speech: ");
            mySerial.println(recognized_text);
        } else {
            Serial.printf("Error on HTTP request: %d\n", httpResponseCode);
        }
        http.end();
    } else {
        Serial.println("WiFi not connected");
    }
}

String extract_text_from_response(const String& response) {
  
    return "";
}

void send_image_to_server(const uint8_t *image_data, size_t size) {
    if (WiFi.status() == WL_CONNECTED) {
        HTTPClient http;
        http.begin(ocrApiUrl);
        http.addHeader("Content-Type", "image/jpeg");
        int httpResponseCode = http.POST(image_data, size);
        if (httpResponseCode > 0) {
            String response = http.getString();
            Serial.println("OCR Response:");
            Serial.println(response);

            
            mySerial.print("OCR: ");
            mySerial.println(response);
        } else {
            Serial.println("Error on HTTP request");
        }
        http.end();
    } else {
        Serial.println("WiFi not connected");
    }
}

void setup() {
    Serial.begin(115200);
    connect_wifi();
    setup_camera();
    setup_voice_recognition();

    
    mySerial.begin(UART1_BAUD, SERIAL_8N1, 16, 17); 
}

void loop() {
  
    camera_fb_t *fb = esp_camera_fb_get();
    if (!fb) {
        Serial.println("Camera capture failed");
        return;
    }

    Serial.println("Captured image, sending to server...");
    send_image_to_server(fb->buf, fb->len);
    esp_camera_fb_return(fb);

    
    uint8_t audio_data[1024]; 
    size_t audio_size = sizeof(audio_data);

   
    send_audio_to_server(audio_data, audio_size);

    delay(10000); 
}

nrf52840overlay.dts

C/C++
Device tree file for current project
/dts-v1/; 
/ {
    aliases {
        uart1 = &uart1;  // Alias for UART1
        uart2 = &uart2;  // Alias for UART2 (used for XIAO ESP32S3 Sense)
        servo1 = &gpio0;  // Alias for GPIO used for Servo 1
        servo2 = &gpio0;  // Alias for GPIO used for Servo 2
        servo3 = &gpio0;  // Alias for GPIO used for Servo 3
        speaker = &gpio0; // Alias for GPIO used for Speaker
    };

    chosen {
        zephyr,console = &uart0;
    };

    gpio0: gpio@50000000 {
        compatible = "nordic,nrf-gpio";
        status = "okay";
        servo1-gpios = <&gpio0 13 GPIO_ACTIVE_HIGH>; // Pin P0.13 for Servo 1
        servo2-gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>; // Pin P0.14 for Servo 2
        servo3-gpios = <&gpio0 15 GPIO_ACTIVE_HIGH>; // Pin P0.15 for Servo 3
        speaker-gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>; // Pin P0.16 for Speaker
    };

    uart1: uart@40028000 {  // UART1 configuration
        compatible = "nordic,nrf-uart";
        current-speed = <115200>;
        status = "okay";
        tx-pin = <6>;  // TX on pin P0.06
        rx-pin = <8>;  // RX on pin P0.08
    };

    uart2: uart@40029000 {  // UART2 configuration for XIAO ESP32S3 Sense
        compatible = "nordic,nrf-uart";
        current-speed = <115200>;
        status = "okay";
        tx-pin = <20>;  // TX on pin P0.20
        rx-pin = <22>;  // RX on pin P0.22
    };
};

Credits

GOWTHAM S

GOWTHAM S

2 projects • 2 followers
Balamurugan.K

Balamurugan.K

0 projects • 1 follower

Comments

Awards
Runnerup Visual Impairments
Build2gether 2.0 — Inclusive Innovation Challenge