Sarah Han
Created December 25, 2019 © GPL3+

OAT M3AL Cooking Bot

Using MindStorms EV3 and Alexa to cook oat meal

AdvancedProtipOver 1 day143
First 100 Submissions
LEGO MINDSTORMS Voice Challenge

Things used in this project

Hardware components

Mindstorms EV3 Programming Brick / Kit
LEGO Mindstorms EV3 Programming Brick / Kit
×1
Amazon Echo
Amazon Alexa Amazon Echo
×1

Software apps and online services

Microsoft Visual Studio Code
Alexa Gadgets Toolkit
Amazon Alexa Alexa Gadgets Toolkit
Alexa Skills Kit
Amazon Alexa Alexa Skills Kit
Alexa Voice Service
Amazon Alexa Alexa Voice Service

Story

Read more

Schematics

Pen Arm Build Instructions

Pen Arm Build Instructions

Placer LEGO Build Instructions

Placer LEGO Build Instructions

Single Motor Cart Build Instructions

Single Motor Cart Build Instructions

schema

ev3 schema

Code

index.js

JavaScript
index.js file for lambda
/*
 * Copyright 2019 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
 *
 * You may not use this file except in compliance with the terms and conditions 
 * set forth in the accompanying LICENSE.TXT file.
 *
 * THESE MATERIALS ARE PROVIDED ON AN "AS IS" BASIS. AMAZON SPECIFICALLY DISCLAIMS, WITH 
 * RESPECT TO THESE MATERIALS, ALL WARRANTIES, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING 
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.
*/

// This skill sample demonstrates how to send directives and receive events from an Echo connected gadget.
// This skill uses the Alexa Skills Kit SDK (v2). Please visit https://alexa.design/cookbook for additional
// examples on implementing slots, dialog management, session persistence, api calls, and more.

const Alexa = require('ask-sdk-core');
const Util = require('./util');
const Common = require('./common');

// The audio tag to include background music
const BG_MUSIC = '<audio src="soundbank://soundlibrary/ui/gameshow/amzn_ui_sfx_gameshow_waiting_loop_30s_01"></audio>';

// The namespace of the custom directive to be sent by this skill
const NAMESPACE = 'Custom.Mindstorms.Gadget';

// The name of the custom directive to be sent this skill
const NAME_CONTROL = 'control';

const LaunchRequestHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'LaunchRequest';
    },
    handle: async function(handlerInput) {

        const request = handlerInput.requestEnvelope;
        const { apiEndpoint, apiAccessToken } = request.context.System;
        const apiResponse = await Util.getConnectedEndpoints(apiEndpoint, apiAccessToken);
        if ((apiResponse.endpoints || []).length === 0) {
            return handlerInput.responseBuilder
            .speak(`I couldnt find an EV3 Brick connected to this Echo device. Please check to make sure your EV3 Brick is connected, and try again.`)
            .getResponse();
        }

        // Store the gadget endpointId to be used in this skill session
        const endpointId = apiResponse.endpoints[0].endpointId || [];
        Util.putSessionAttribute(handlerInput, 'endpointId', endpointId);

        // Set skill duration to 5 minutes (ten 30-seconds interval)
        Util.putSessionAttribute(handlerInput, 'duration', 10);

        // Set the token to track the event handler
        const token = handlerInput.requestEnvelope.request.requestId;
        Util.putSessionAttribute(handlerInput, 'token', token);

        let speechOutput = "Oat Meal Bot is at your service.";
        return handlerInput.responseBuilder
            .speak(speechOutput)
            .addDirective(Util.buildStartEventHandler(token,60000, {}))
            .getResponse();
    }
};

// Construct and send a custom directive to the connected gadget with data from
// the CookIntentHandler.
const CookIntentHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
            && Alexa.getIntentName(handlerInput.requestEnvelope) === 'CookIntent';
    },
    handle: function (handlerInput) {
        const attributesManager = handlerInput.attributesManager;
        let endpointId = attributesManager.getSessionAttributes().endpointId || [];

        // Construct the directive with the payload containing the move parameters
        let directive = Util.build(endpointId, NAMESPACE, NAME_CONTROL,
            {
                type: "cook"
            });

        let speechOutput = "Starting to cook oat meal, waiting for water to boil.";
        
        return handlerInput.responseBuilder
            .speak(speechOutput)
            .addDirective(directive)
            .getResponse();
    }
};

// Construct and send a custom directive to the connected gadget with data from
// the StopCookIntentHandler.
const StopCookIntentHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
            && Alexa.getIntentName(handlerInput.requestEnvelope) === 'StopCookIntent';
    },
    handle: function (handlerInput) {
        const attributesManager = handlerInput.attributesManager;
        let endpointId = attributesManager.getSessionAttributes().endpointId || [];

        // Construct the directive with the payload containing the move parameters
        let directive = Util.build(endpointId, NAMESPACE, NAME_CONTROL,
            {
                type: "stop"
            });

        let speechOutput = "Stopping the cooking, please turn off the stove";
        
        return handlerInput.responseBuilder
            .speak(speechOutput)
            .addDirective(directive)
            .getResponse();
    }
};

// Construct and send a custom directive to the connected gadget with data from
// the ReadyIntentHandler.
const ReadyIntentHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
            && Alexa.getIntentName(handlerInput.requestEnvelope) === 'ReadyIntent';
    },
    handle: function (handlerInput) {
        const attributesManager = handlerInput.attributesManager;
        let endpointId = attributesManager.getSessionAttributes().endpointId || [];

        // Construct the directive with the payload containing the move parameters
        let directive = Util.build(endpointId, NAMESPACE, NAME_CONTROL,
            {
                type: "ready"
            });

        let speechOutput = "Checking";
        
        return handlerInput.responseBuilder
            .speak(speechOutput)
            .addDirective(directive)
            .getResponse();
    }
};

// Construct and send a custom directive to the connected gadget with data from
// the AutohIntentHandler.
const YesNoIntentHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'IntentRequest'
            && Alexa.getIntentName(handlerInput.requestEnvelope) === 'YesNoIntent';
    },
    handle: function (handlerInput) {

        let yesno = Alexa.getSlotValue(handlerInput.requestEnvelope, 'YesNo');
        if (!yesno) {
            return handlerInput.responseBuilder
                .speak("Can you repeat that?")
                .withShouldEndSession(false)
                .getResponse();
        }
        
        const attributesManager = handlerInput.attributesManager;
        let endpointId = attributesManager.getSessionAttributes().endpointId || [];

        if(yesno.includes("yes"))
        {
            // Construct the directive with the payload containing the move parameters
            let directive = Util.build(endpointId, NAMESPACE, NAME_CONTROL,
                {
                    type: "cook"
                });
    
            let speechOutput = "Starting to cook oat meal, waiting for water to boil.";
            
            return handlerInput.responseBuilder
                .speak(speechOutput)
                .addDirective(directive)
                .getResponse();
        }
        else
        {
            let speechOutput = 'Sure, if you want to cook oat meal you can simply say Alexa, start cooking';
            
            return handlerInput.responseBuilder
                .speak(speechOutput)
                .getResponse();   
        }
    }
};



const EventsReceivedRequestHandler = {
    // Checks for a valid token and endpoint.
    canHandle(handlerInput) {
        let { request } = handlerInput.requestEnvelope;
        console.log('Request type: ' + Alexa.getRequestType(handlerInput.requestEnvelope));
        if (request.type !== 'CustomInterfaceController.EventsReceived') return false;

        const attributesManager = handlerInput.attributesManager;
        let sessionAttributes = attributesManager.getSessionAttributes();
        let customEvent = request.events[0];

        // Validate event token
        if (sessionAttributes.token !== request.token) {
            console.log("Event token doesn't match. Ignoring this event");
            return false;
        }

        // Validate endpoint
        let requestEndpoint = customEvent.endpoint.endpointId;
        if (requestEndpoint !== sessionAttributes.endpointId) {
            console.log("Event endpoint id doesn't match. Ignoring this event");
            return false;
        }
        return true;
    },
    handle(handlerInput) {

        console.log("== Received Custom Event ==");
        let customEvent = handlerInput.requestEnvelope.request.events[0];
        let payload = customEvent.payload;
        let name = customEvent.header.name;

        let speechOutput;
        
        if (payload.speech) {
            speechOutput = payload.speech;
        }
        
        if (name === 'Start') {
            return handlerInput.responseBuilder
                .speak(speechOutput, "REPLACE_ALL")
                .withShouldEndSession(false)
                .getResponse();                   
        }
        else{
            return handlerInput.responseBuilder
            .speak(speechOutput, "REPLACE_ALL")
            .getResponse();
        }
    }
};

const ExpiredRequestHandler = {
    canHandle(handlerInput) {
        return Alexa.getRequestType(handlerInput.requestEnvelope) === 'CustomInterfaceController.Expired'
    },
    handle(handlerInput) {
        console.log("== Custom Event Expiration Input ==");

        // Set the token to track the event handler
        const token = handlerInput.requestEnvelope.request.requestId;
        Util.putSessionAttribute(handlerInput, 'token', token);

        const attributesManager = handlerInput.attributesManager;
        let duration = attributesManager.getSessionAttributes().duration || 0;
        if (duration > 0) {
            Util.putSessionAttribute(handlerInput, 'duration', --duration);

            // Extends skill session
            const speechOutput = `${duration} minutes remaining.`;
            return handlerInput.responseBuilder
                .addDirective(Util.buildStartEventHandler(token, 60000, {}))
                .speak(speechOutput)
                .getResponse();
        }
        else {
            // End skill session
            return handlerInput.responseBuilder
                .speak("Skill duration expired. Goodbye.")
                .withShouldEndSession(true)
                .getResponse();
        }
    }
};

// The SkillBuilder acts as the entry point for your skill, routing all request and response
// payloads to the handlers above. Make sure any new handlers or interceptors you've
// defined are included below. The order matters - they're processed top to bottom.
exports.handler = Alexa.SkillBuilders.custom()
    .addRequestHandlers(
        LaunchRequestHandler,
        CookIntentHandler,
        StopCookIntentHandler,
        ReadyIntentHandler,
        YesNoIntentHandler,
        EventsReceivedRequestHandler,
        ExpiredRequestHandler,
        Common.HelpIntentHandler,
        Common.CancelAndStopIntentHandler,
        Common.SessionEndedRequestHandler,
        Common.IntentReflectorHandler, // make sure IntentReflectorHandler is last so it doesn't override your custom intent handlers
    )
    .addRequestInterceptors(Common.RequestInterceptor)
    .addErrorHandlers(
        Common.ErrorHandler,
    )
    .lambda();

oatmeal.py

Python
ev3dev python file
import os
import sys
import time
import logging
import json
import random
import threading
import math
from sys import stderr
from os import system
from enum import Enum
from smbus2 import SMBus, i2c_msg
from agt import AlexaGadget
from ev3dev2.led import Leds
from ev3dev2.sound import Sound
from ev3dev2.sensor.lego import TouchSensor
from ev3dev2.motor import OUTPUT_A, OUTPUT_B, OUTPUT_C, OUTPUT_D, MediumMotor, LargeMotor, SpeedPercent
from ev3dev2.sensor.lego import Sensor
from ev3dev2.port import LegoPort
from ev3dev2.sensor import INPUT_1


# Set the logging level to INFO to see messages from AlexaGadget
logging.basicConfig(level=logging.INFO, stream=sys.stdout, format='%(message)s')
logging.getLogger().addHandler(logging.StreamHandler(sys.stderr))
logger = logging.getLogger(__name__)

B = 42500 # B value of the thermistor
R0 = 100000 # R0 = 100000

class EventName(Enum):
    """
    The list of custom event name sent from this gadget
    """
    READY = "Ready"
    NOTREADY = "NotReady"
    START = "Start"

class MindstormsGadget(AlexaGadget):
    """
    A Mindstorms gadget that can perform bi-directional interaction with an Alexa skill.
    """

    def __init__(self):
        """
        Performs Alexa Gadget initialization routines and ev3dev resource allocation.
        """
        super().__init__()
        temp = Sensor(INPUT_1)
        temp.mode = 'TEMP'
        
        # Robot state
        self.isCooking = False
        self.isActivated = False
        self.sound = Sound()
        self.leds = Leds()
        self.dropper_load = TouchSensor()
        self.current_time = datetime.datetime.now()
        self.stop_time = datetime.datetime.now()

        #Motor
        self.x_axis = LargeMotor(OUTPUT_C)
        self.y_axis = LargeMotor(OUTPUT_A)
        self.z_axis = LargeMotor(OUTPUT_B)
        self.dropper = MediumMotor(OUTPUT_D)

        #set the z axis up by default
        ##self.z_axis.on_for_degrees(SpeedPercent(10), 80)

        # Start threads
        threading.Thread(target=self._cooking_thread, daemon=True).start()
        threading.Thread(target=self._stiring_thread, daemon=True).start()

    def on_connected(self, device_addr):
        """
        Gadget connected to the paired Echo device.
        :param device_addr: the address of the device we connected to
        """
        self.leds.set_color("LEFT", "GREEN")
        self.leds.set_color("RIGHT", "GREEN")
        logger.info("{} connected to Echo device".format(self.friendly_name))

    def on_disconnected(self, device_addr):
        """
        Gadget disconnected from the paired Echo device.
        :param device_addr: the address of the device we disconnected from
        """
        self.leds.set_color("LEFT", "BLACK")
        self.leds.set_color("RIGHT", "BLACK")
        logger.info("{} disconnected from Echo device".format(self.friendly_name))

    def on_custom_mindstorms_gadget_control(self, directive):
        """
        Handles the Custom.Mindstorms.Gadget control directive.
        :param directive: the custom directive with the matching namespace and name
        """
        try:
            payload = json.loads(directive.payload.decode("utf-8"))
            print("Control payload: {}".format(payload), file=sys.stderr)
            control_type = payload["type"]
            if control_type == "cook":
                self._cook_handler()
            elif control_type == "stop":
                self._stop_handler()
            elif control_type == "ready":
                self._ready_handler()
        except KeyError:
            print("Missing expected parameters: {}".format(directive), file=sys.stderr)

    def _cook_handler(self):
        self.isActivated = True

    def _stop_handler(self):
        if self.isCooking == True:
            self.z_axis.on_for_degrees(SpeedPercent(10), 80)

        self.isActivated = False
        self.isCooking = False

    def _ready_handler(self):
        if self.isActivated == False and self.isCooking == False:
            self._send_event(EventName.START,  {'speech':'cooking have not started yet, would you like to start cooking?'})
        elif self.isActivated == True and self.isCooking == False:
            self._send_event(EventName.READY,  {'speech':'I am waiting for water to boil, please wait a little.'}) 
        elif self.isActivated == False and self.isCooking == True:
            self._send_event(EventName.READY,  {'speech':'The oat meal is almost ready, sit tight for less than 5 minutes'}) 

    def _send_event(self, name: EventName, payload):
        """
        Sends a custom event to trigger a sentry action.
        :param name: the name of the custom event
        :param payload: the sentry JSON payload
        """
        print(name.value)
        print(payload)
        self.send_custom_event('Custom.Mindstorms.Gadget', name.value, payload)


    def _cooking_thread(self):
        while True:

            temperature = temp.value
            print("temperature:" + str(temperature))

            #first check the timer
            if datetime.datetime.now() > self.stop_time and self.isActivated == False and self.isCooking == True:
                self.isActivated = False
                self.isCooking = False
                self.z_axis.on_for_degrees(SpeedPercent(10), 80)
                self._send_event(EventName.READY,  {'speech':'Oat meal is ready, please turn off the stove'})

            if self.isActivated == True and temperature > 75 and self.isCooking == False:

                #if water is 100, the temperature around should be 75+
                self.dropper.on_for_seconds(SpeedPercent(-20), 3)
                self.z_axis.on_for_degrees(SpeedPercent(10), -80)
                time.sleep(1)
                while self.dropper_load.is_released:
                    self.dropper.on_for_seconds(SpeedPercent(20), 1)
                #now we are starting to cook
                self.stop_time = datetime.datetime.now() + datetime.timedelta(minutes=5)
                self.isCooking = True
                self.isActivated = False
                self._send_event(EventName.READY,  {'speech':'The water is almost boiling, we will start cooking oatmeal now'})
            time.sleep(1)
    def _stiring_thread(self):
        while True:
            if self.isCooking == True:
                self.y_axis.on_for_seconds(SpeedPercent(-5), 1.2)
                self.x_axis.on_for_seconds(SpeedPercent(-10), 1.5)
                self.y_axis.on_for_seconds(SpeedPercent(5), 1.5)
                self.x_axis.on_for_seconds(SpeedPercent(10), 1.5)
                time.sleep(5)

if __name__ == '__main__':

    gadget = MindstormsGadget()

    # Set LCD font and turn off blinking LEDs
    os.system('setfont Lat7-Terminus12x6')
    gadget.leds.set_color("LEFT", "BLACK")
    gadget.leds.set_color("RIGHT", "BLACK")

    # Startup sequence
    gadget.sound.play_song((('C4', 'e'), ('D4', 'e'), ('E5', 'q')))
    gadget.leds.set_color("LEFT", "GREEN")
    gadget.leds.set_color("RIGHT", "GREEN")

    # Gadget main entry point
    gadget.main()

    # Shutdown sequence
    gadget.sound.play_song((('E5', 'e'), ('C4', 'e')))
    gadget.leds.set_color("LEFT", "BLACK")
    gadget.leds.set_color("RIGHT", "BLACK")

model.js

JSON
alexa json model
{
    "interactionModel": {
        "languageModel": {
            "invocationName": "oat meal bot",
            "intents": [
                {
                    "name": "AMAZON.CancelIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.HelpIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.StopIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.NavigateHomeIntent",
                    "samples": []
                },
                {
                    "name": "CookIntent",
                    "slots": [],
                    "samples": [
                        "Star cooking",
                        "Cook me oat meal",
                        "Cook me food"
                    ]
                },
                {
                    "name": "StopCookIntent",
                    "slots": [],
                    "samples": [
                        "Stop cooking",
                        "Stop cook"
                    ]
                },
                {
                    "name": "ReadyIntent",
                    "slots": [],
                    "samples": [
                        "Is it ready yet",
                        "Is the food ready",
                        "I am hungry"
                    ]
                },
                {
                    "name": "YesNoIntent",
                    "slots": [
                        {
                            "name": "YesNo",
                            "type": "yesNoType"
                        }
                    ],
                    "samples": [
                        "{YesNo}"
                    ]
                },
                {
                    "name": "AMAZON.MoreIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.NavigateSettingsIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.NextIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.PageUpIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.PageDownIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.PreviousIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.ScrollRightIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.ScrollDownIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.ScrollLeftIntent",
                    "samples": []
                },
                {
                    "name": "AMAZON.ScrollUpIntent",
                    "samples": []
                }
            ],
            "types": [
                {
                    "name": "yesNoType",
                    "values": [
                        {
                            "name": {
                                "value": "yes",
                                "synonyms": [
                                    "yep",
                                    "yeah",
                                    "I do",
                                    "yes please",
                                    "you know it",
                                    "I would"
                                ]
                            }
                        },
                        {
                            "name": {
                                "value": "no",
                                "synonyms": [
                                    "no",
                                    "no thanks",
                                    "nope",
                                    "I do not",
                                    "no thank you",
                                    "don't"
                                ]
                            }
                        }
                    ]
                }
            ]
        }
    }
}

oatmeal.ini

INI
ini file for ev3dev
# Copyright 2019 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
# 
# You may not use this file except in compliance with the terms and conditions 
# set forth in the accompanying LICENSE.TXT file.
#
# THESE MATERIALS ARE PROVIDED ON AN "AS IS" BASIS. AMAZON SPECIFICALLY DISCLAIMS, WITH 
# RESPECT TO THESE MATERIALS, ALL WARRANTIES, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING 
# THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.

[GadgetSettings]
amazonId = [amazonid]
alexaGadgetSecret = [alexaGadgetSecret]

[GadgetCapabilities]
Custom.Mindstorms.Gadget = 1.0

Credits

Sarah Han

Sarah Han

13 projects • 79 followers
Software Engineer, Design, 3D
Thanks to Peter Ma.

Comments

Awards
First 100 Submissions
LEGO MINDSTORMS Voice Challenge
Related channels and tags