Hackster is hosting Hackster Holidays, Ep. 6: Livestream & Giveaway Drawing. Watch previous episodes or stream live on Monday!Stream Hackster Holidays, Ep. 6 on Monday!
Prasanna Vijay
Published © GPL3+

AURA: Smart Specs Attachment for Blind Students

A smart specs attachment designed to assist blind students in studying mathematics and navigating within teaching infrastructure environment

IntermediateWork in progressOver 4 days133
Honorable Mention Visual Impairments
Build2gether 2.0 — Inclusive Innovation Challenge
AURA: Smart Specs Attachment for Blind Students

Things used in this project

Hardware components

Seeed Studio XIAO ESP32C3
Seeed Studio XIAO ESP32C3
×1
Grove Vision AI Module V2
Seeed Studio Grove Vision AI Module V2
×1
Grove Shield for Seeeduino XIAO - with embedded battery management chip
Seeed Studio Grove Shield for Seeeduino XIAO - with embedded battery management chip
×1
VL53L0X TOF
×1
TDK Corporation Electromagnetic Buzzer - 5V
×1
Lithium Polymer Battery - 3.7V 500mAh
×1

Software apps and online services

Seeed Studio Sensecraft AI
Google Colab
Google Gemini AI
Firebase
Google Firebase
Google Flutter

Hand tools and fabrication machines

3D Printer (generic)
3D Printer (generic)
Jumper Wire, Bundle
Jumper Wire, Bundle
Soldering iron (generic)
Soldering iron (generic)

Story

Read more

Custom parts and enclosures

3D MODEL OF PROJECT AURA

Sketchfab still processing.

Schematics

Schematics of Project AURA

Code

Seeed Studio XIAO C3 CODE

C/C++
Upload the Code to Seeed Studio XIAO C3.
#include <Wire.h>
#include <WiFi.h>
#include <Seeed_Arduino_SSCMA.h>
#include <VL53L0X.h>             // ToF sensor library
#include <FirebaseESP32.h>       // Firebase library

// Pin configuration
#define SPEAKER_PIN 8            // GPIO D8 pin connected to the speaker via groove expansion board

// I2C addresses (default addresses, adjust if needed)
#define TOF_I2C_ADDR 0x29        // I2C address for ToF sensor
#define AI_I2C_ADDR 0x62         // I2C address for Grove Vision AI

// Firebase configuration
#define FIREBASE_HOST "your-firebase-host"
#define FIREBASE_AUTH "your-firebase-auth"
FirebaseData firebaseData;
FirebaseJson json;

// WiFi credentials
const char* ssid = "your-SSID";
const char* password = "your-PASSWORD";

// Initialize components
SSCMA AI;                        // Grove Vision AI module
VL53L0X tofSensor;               // ToF sensor

// Function to play a simple tone
void playTone() {
    tone(SPEAKER_PIN, 1000, 200);  // Play a 1kHz tone for 200ms
}

// Function to provide audio feedback
void playAudio(String message) {
    Serial.println("Playing audio: " + message);
    playTone();  // Play a simple tone as audio feedback
}

void setup() {
    Serial.begin(115200);

    // Initialize the I2C bus for the Grove Vision AI module
    Wire.begin();  // Default I2C pins (SDA, SCL)

    // Initialize the I2C bus for the ToF sensor
    Wire1.begin(SDA, SCL);  // Use appropriate I2C pins for the XIAO expansion board

    // Initialize the ToF sensor
    if (!tofSensor.init(true, &Wire1)) {  // Initialize ToF with secondary I2C bus
        Serial.println("Failed to detect and initialize ToF sensor!");
        while (1);
    }
    tofSensor.setTimeout(500);
    tofSensor.startContinuous();

    // Initialize the AI module
    AI.begin();  

    // Initialize WiFi
    WiFi.begin(ssid, password);
    while (WiFi.status() != WL_CONNECTED) {
        delay(1000);
        Serial.println("Connecting to WiFi...");
    }
    Serial.println("Connected to WiFi");

    // Initialize Firebase
    Firebase.begin(FIREBASE_HOST, FIREBASE_AUTH);
    Firebase.reconnectWiFi(true);
}

void loop() {
    // Check for obstacles using the ToF sensor
    uint16_t distance = tofSensor.readRangeContinuousMillimeters();
    if (tofSensor.timeoutOccurred()) {
        Serial.println("ToF sensor timeout");
    }

    if (distance < 1000) {  // Trigger distance in millimeters (adjust as needed)
        Serial.println("Obstacle detected at distance: " + String(distance) + " mm");

        // Play audio feedback when an obstacle is detected
        playAudio("Obstacle detected");

        // Trigger AI processing and save the image if a math equation is detected
        if (!AI.invoke()) { 
            for (int i = 0; i < AI.boxes().size(); i++) {
                if (AI.boxes()[i].score > 70) {
                    Serial.println("Math equation detected");

                    // Capture and save image to Firebase
                    captureAndSaveImageToFirebase();
                }
            }
        }
    }

    delay(1000);  // Adjust the delay as needed
}

void captureAndSaveImageToFirebase() {
    String imagePath = "/math_equation.jpg";
    if (saveImageToFirebase(imagePath)) {
        Serial.println("Image successfully uploaded to Firebase");
    } else {
        Serial.println("Failed to upload image to Firebase");
    }
}

bool saveImageToFirebase(const String& path) {
    uint8_t* imgData;
    size_t imgSize;

    if (!captureImageFromAI(imgData, imgSize)) {
        Serial.println("Failed to capture image from AI module");
        return false;
    }

    // Convert image data to a base64 string for Firebase upload
    String base64Image = base64::encode(imgData, imgSize);

    // Upload image to Firebase Storage
    String storagePath = "/images" + path;  // Adjust the path as needed
    if (Firebase.Storage.upload(&firebaseData, storagePath, memStorageType_base64, base64Image)) {
        Serial.println("Image successfully uploaded to Firebase Storage");
        return true;
    } else {
        Serial.println("Failed to upload image to Firebase Storage: " + firebaseData.errorReason());
        return false;
    }

}

Credits

Prasanna Vijay

Prasanna Vijay

2 projects • 1 follower

Comments

Awards
Honorable Mention Visual Impairments
Build2gether 2.0 — Inclusive Innovation Challenge
Related channels and tags
  • Google
  • Machine Learning & AI
  • Machine Learning & AI
  • Seeed Studio
  • TDK Corporation
  • Wearables
  • Wearables