AI for the Eye: Vision to Voice in Real-Time

import gc
import esp
import os
import ubinascii  # MicroPython's built-in binary/hex conversion library
from Wifi import Sta
from umqtt.simple import MQTTClient
from time import sleep

esp.osdebug(None)

# Constants for Wi-Fi and MQTT setup
UID = const('xiao')
PWD = const('mick')

# Wi-Fi connection setup
sta = Sta()
sta.wlan.disconnect()
AP = const('Blur')
PW = const('Blur1234')
sta.connect(AP, PW)
sta.wait()

if not sta.wlan.isconnected():
    print("Wi-Fi not connected.")
    print("System aborted.")
else:
    print("Wi-Fi connected.")

    # Initialize MQTT client
    broker_address = "broker.hivemq.com"
    mqtt_client_id = "ESP32_Client"
    topic_img = "image/test_img"

    def connect_to_mqtt():
        client = MQTTClient(mqtt_client_id, broker_address, keepalive=60)  # Set a longer keep-alive
        client.connect()
        print(f"Connected to {broker_address}")
        return client

    # Function to reconnect the MQTT client
    def reconnect_mqtt(client):
        try:
            client.connect()
            print(f"Reconnected to {broker_address}")
        except Exception as e:
            print(f"Reconnection failed: {e}")

    # Function to read and Base64-encode the entire image file
    def read_and_encode_image(file_path):
        if file_path in os.listdir():
            with open(file_path, "rb") as img_file:
                img_data = img_file.read()  # Read the entire image as binary
                encoded_data = ubinascii.b2a_base64(img_data)  # Encode the entire image as Base64
                return encoded_data
        else:
            print(f"File {file_path} does not exist.")
            return None

    # Function to publish the encoded image data in larger chunks with retry and reconnection handling
    def publish_encoded_image_in_chunks(client, encoded_data, chunk_size=4096, max_retries=10):
        if encoded_data:
            total_length = len(encoded_data)
            total_chunks = (total_length + chunk_size - 1) // chunk_size  # Calculate total number of chunks
            print(f"Total length of encoded image: {total_length} bytes")
            print(f"Total number of chunks: {total_chunks}")

            for i in range(0, total_length, chunk_size):
                chunk = encoded_data[i:i+chunk_size]  # Break the data into larger chunks
                chunk_number = i // chunk_size + 1  # Current chunk number
                attempt = 0
                success = False
                while attempt < max_retries and not success:
                    try:
                        client.publish(topic_img, chunk)
                        print(f"Published chunk {chunk_number}/{total_chunks} of size {len(chunk)}")
                        success = True
                    except Exception as e:
                        attempt += 1
                        print(f"Error sending chunk {chunk_number}/{total_chunks}, attempt {attempt}/{max_retries}: {e}")
                        if attempt < max_retries:
                            reconnect_mqtt(client)  # Try to reconnect to MQTT if not all retries have been exhausted
                        sleep(1)  # Wait before retrying
                if not success:
                    print(f"Failed to send chunk {chunk_number}/{total_chunks} after {max_retries} retries. Aborting.")
                    client.disconnect()  # Disconnect in case of persistent failure
                    return
                sleep(0.1)  # Small delay between sending chunks
            print(f"Entire image sent in chunks to {topic_img}.")
            client.publish(topic_img, "END_OF_IMAGE")
            print(f"Published END_OF_IMAGE message.")
        else:
            print("No image data to send.")

    # Connect to MQTT broker
    mqtt_client = connect_to_mqtt()

    # Read and encode the entire image file
    image_file = "test_img.jpg"
    encoded_image_data = read_and_encode_image(image_file)

    # Publish the encoded image data in larger chunks with retries and reconnections
    publish_encoded_image_in_chunks(mqtt_client, encoded_image_data, chunk_size=4096, max_retries=10)  # Larger chunk size of 1024 bytes

    
    # Disconnect from the MQTT broker after publishing
    mqtt_client.disconnect()
    print("Disconnected from MQTT broker.")

    # Collect garbage to free up memory
    gc.collect()

print("Process completed.")

import tkinter as tk
from gtts import gTTS
from playsound import playsound
import os
import paho.mqtt.client as mqtt
import threading

# Global variables to store the latest data from MQTT topics
scene_description = ""
frequent_objects = []

# Function to display the list of frequent objects
def display_list():
    global frequent_objects
    text_area.delete(1.0, tk.END)
    for item in frequent_objects:
        text_area.insert(tk.END, item + '\n')

# Function to perform TTS for the scene description
def perform_tts():
    global scene_description
    if scene_description:
        tts = gTTS(text=scene_description, lang='en')
        tts.save("output.mp3")
        playsound("output.mp3")
        os.remove("output.mp3")

# MQTT client callbacks
def on_connect(client, userdata, flags, rc):
    if rc == 0:
        print("Connected to MQTT Broker!")
        # Subscribe to both topics after connecting
        client.subscribe("image/scene_description")
        client.subscribe("image/frequent_objects")
    else:
        print("Failed to connect, return code %d\n", rc)

def on_message(client, userdata, message):
    global scene_description, frequent_objects
    
    # Decode message based on topic
    topic = message.topic
    mqtt_message = str(message.payload.decode("utf-8"))
    
    if topic == "image/scene_description":
        scene_description = mqtt_message
        print(f"Scene Description: {scene_description}")
        # Automatically perform TTS when the scene description is updated
        perform_tts()
        
    elif topic == "image/frequent_objects":
        frequent_objects = mqtt_message.split(",")  # Assuming the objects are sent as a comma-separated string
        print(f"Frequent Objects: {frequent_objects}")
        display_list()

# MQTT listener function
def mqtt_listener():
    client = mqtt.Client()
    client.on_connect = on_connect
    client.on_message = on_message

    # Connect to HiveMQ broker
    client.connect("broker.hivemq.com", 1883, 60)

    # Keep the client connected and listening to topics
    client.loop_forever()

# Create the main window
root = tk.Tk()
root.title("UniHiker GUI")

# Create a text area to display the list of frequent objects
text_area = tk.Text(root, height=10, width=40)
text_area.pack()

# Create a button to manually trigger TTS for scene description (optional)
tts_button = tk.Button(root, text="Speak Scene Description", command=perform_tts)
tts_button.pack()

# Start the MQTT listener in a separate thread
mqtt_thread = threading.Thread(target=mqtt_listener)
mqtt_thread.daemon = True
mqtt_thread.start()

# Run the GUI
root.mainloop()