Kutluhan Aktar
Published © CC BY

Vegetables and Fruits Ripeness Detection by Color w/ TF

Collate spectral color data of varying fruits and vegetables and interpret this data set with a neural network to predict ripening stages.

ExpertFull instructions provided5 hours19,091

Things used in this project

Hardware components

Arduino Nano 33 IoT
Arduino Nano 33 IoT
×1
DFRobot AS7341 11-Channel Visible Light Sensor
×1
Raspberry Pi 4 Model B
Raspberry Pi 4 Model B
Raspberry Pi 3B+ or 4
×1
Raspberry Pi 3 Model B+
Raspberry Pi 3 Model B+
Raspberry Pi 3B+ or 4
×1
10K Potentiometer (Long-Shaft)
×1
Potentiometer Knob
×1
SparkFun Button (6x6)
×4
5 mm LED: Green
5 mm LED: Green
×1
5 mm LED: Red
5 mm LED: Red
×1
Solderless Breadboard Full Size
Solderless Breadboard Full Size
×2

Software apps and online services

Arduino IDE
Arduino IDE
TensorFlow
TensorFlow
Thonny
Visual Studio 2017
Microsoft Visual Studio 2017

Hand tools and fabrication machines

Hot glue gun (generic)
Hot glue gun (generic)

Story

Read more

Custom parts and enclosures

Data Set

Vegetables_and_Fruits_Data_Logger.zip

ANN_Ripeness_Detection.h5

Schematics

Connections

Code

Vegetables_and_Fruits_Ripeness_Detection.ino

Arduino
         /////////////////////////////////////////////  
        //    Vegetables and Fruits Ripeness       // 
       //    Detection by Color w/ TensorFlow     //
      //           -----------------             //
     //          (Arduino Nano 33 IoT)          //           
    //             by Kutluhan Aktar           // 
   //                                         //
  /////////////////////////////////////////////

// 
// Collate spectral color data of varying fruits and vegetables and interpret this data set w/ a neural network model to predict ripening stages. 
//
// For more information:
// https://www.theamplituhedron.com/projects/Vegetables_and_Fruits_Ripeness_Detection_by_Color_w_TensorFlow/
//
//
// Connections
// Arduino Nano 33 IoT :           
//                                AS7341 11-Channel Spectral Color Sensor
// 3.3V --------------------------- +
// GND  --------------------------- -
// A5   --------------------------- C
// A4   --------------------------- D
//                                10K Potentiometer
// A0   --------------------------- S
//                                Class Button_1 (6x6)
// D2   ---------------------------
//                                Class Button_2 (6x6)
// D3   ---------------------------
//                                Class Button_3 (6x6)
// D4   ---------------------------
//                                Class Button_4 (6x6)
// D5   ---------------------------
//                                5mm Green LED
// D6   ---------------------------
//                                5mm Red LED
// D7   ---------------------------


// Include required libraries:
#include <WiFiNINA.h>
#include "DFRobot_AS7341.h"

char ssid[] = "SSID";        // your network SSID (name)
char pass[] = "PASSWORD";    // your network password (use for WPA, or use as key for WEP)
int keyIndex = 0;              // your network key Index number (needed only for WEP)
int status = WL_IDLE_STATUS;

// Enter the IPAddress of your Raspberry Pi.
IPAddress server(192, 168, 1, 20);

// Initialize the Ethernet client library
WiFiClient client; /* WiFiSSLClient client; */

// Define the AS7341 object.
DFRobot_AS7341 as7341;
// Define AS7341 data objects:
DFRobot_AS7341::sModeOneData_t data1;
DFRobot_AS7341::sModeTwoData_t data2;

// Define the potentiometer pin:
#define pot A0

// Define the class button pins:
#define class_1 2
#define class_2 3
#define class_3 4
#define class_4 5

// Define the control LED pins:
#define green 6
#define red 7

// Define the data holders:
int pot_val, class_1_val, class_2_val, class_3_val, class_4_val;

void setup(){
  
  Serial.begin(9600);

  // Class button settings:
  pinMode(class_1, INPUT_PULLUP);
  pinMode(class_2, INPUT_PULLUP);
  pinMode(class_3, INPUT_PULLUP);
  pinMode(class_4, INPUT_PULLUP);
  // Control LED settings:
  pinMode(green, OUTPUT);
  pinMode(red, OUTPUT);
  
  // Detect if I2C can communicate properly 
  while (as7341.begin() != 0) {
    Serial.println("I2C init failed, please check if the wire connection is correct");
    delay(1000);
  }

  // Enable the built-in LED on the AS7341 sensor.
  as7341.enableLed(true);

  // Check for the WiFi module:
  if (WiFi.status() == WL_NO_MODULE) { Serial.println("Connection Failed!"); while (true); }
  // Attempt to connect to the WiFi network:
  while (status != WL_CONNECTED) {
    Serial.println("Attempting to connect to WiFi !!!");
    // Connect to WPA/WPA2 network. Change this line if using open or WEP network:
    status = WiFi.begin(ssid, pass);
    // Wait 10 seconds for connection:
    delay(10000);
  }

}
void loop(){

  read_controls();
  adjust_brightness();

  // Start spectrum measurement:
  // Channel mapping mode: 1.eF1F4ClearNIR
  as7341.startMeasure(as7341.eF1F4ClearNIR);
  // Read the value of sensor data channel 0~5, under eF1F4ClearNIR
  data1 = as7341.readSpectralDataOne();
  // Channel mapping mode: 2.eF5F8ClearNIR
  as7341.startMeasure(as7341.eF5F8ClearNIR);
  // Read the value of sensor data channel 0~5, under eF5F8ClearNIR
  data2 = as7341.readSpectralDataTwo();
  // Print data:
  Serial.print("F1(405-425nm): "); Serial.println(data1.ADF1);
  Serial.print("F2(435-455nm): "); Serial.println(data1.ADF2);
  Serial.print("F3(470-490nm): "); Serial.println(data1.ADF3);
  Serial.print("F4(505-525nm): "); Serial.println(data1.ADF4);
  Serial.print("F5(545-565nm): "); Serial.println(data2.ADF5);
  Serial.print("F6(580-600nm): "); Serial.println(data2.ADF6);
  Serial.print("F7(620-640nm): "); Serial.println(data2.ADF7);
  Serial.print("F8(670-690nm): "); Serial.println(data2.ADF8);
  // CLEAR and NIR:
  Serial.print("Clear_1: "); Serial.println(data1.ADCLEAR);
  Serial.print("NIR_1: "); Serial.println(data1.ADNIR);
  Serial.print("Clear_2: "); Serial.println(data2.ADCLEAR);
  Serial.print("NIR_2: "); Serial.println(data2.ADNIR);
  Serial.print("\n------------------------------\n");
  delay(1000);

  // Send data to the PHP web application (Vegetables_and_Fruits_Data_Logger) depending on the selected ripeness class [0 - 3]:
  if(!class_1_val) make_a_get_request("/Vegetables_and_Fruits_Data_Logger/", "0");
  if(!class_2_val) make_a_get_request("/Vegetables_and_Fruits_Data_Logger/", "1");
  if(!class_3_val) make_a_get_request("/Vegetables_and_Fruits_Data_Logger/", "2");
  if(!class_4_val) make_a_get_request("/Vegetables_and_Fruits_Data_Logger/", "3");

}

void read_controls(){
  // Potentiometer:
  pot_val = analogRead(pot);
  // Class Buttons:
  class_1_val = digitalRead(class_1);
  class_2_val = digitalRead(class_2);
  class_3_val = digitalRead(class_3);
  class_4_val = digitalRead(class_4);
}

void adjust_brightness(){
  // Set pin current to control brightness (1~20 corresponds to current 4mA,6mA,8mA,10mA,12mA,......,42mA)
  int brightness = map(pot_val, 0, 1023, 1, 21);
  Serial.print("\nBrightness: "); Serial.println(brightness);
  as7341.controlLed(brightness);
}

void make_a_get_request(String application, String _class){
  // Connect to the web application named Vegetables_and_Fruits_Data_Logger. Change '80' with '443' if you are using SSL connection.
  if(client.connect(server, 80)){
    // If successful:
    Serial.println("\n\nConnected to the server!");
    // Create the query string:
    String query = application + "?F1="+data1.ADF1+"&F2="+data1.ADF2+"&F3="+data1.ADF3+"&F4="+data1.ADF4+"&F5="+data2.ADF5+"&F6="+data2.ADF6+"&F7="+data2.ADF7+"&F8="+data2.ADF8+"&nir_1="+data1.ADNIR+"&nir_2="+data2.ADNIR+"&class="+_class;
    // Make an HTTP Get request:
    client.println("GET " + query + " HTTP/1.1");
    client.println("Host: 192.168.1.20");
    client.println("Connection: close");
    client.println();
  }else{
    Serial.println("Server Error!");
    digitalWrite(red, HIGH);
  }
  delay(2000); // Wait 2 seconds after connection...
  // If there are incoming bytes available, get the response from the web application.
  String response = "";
  while (client.available()) { char c = client.read(); response += c; }
  if(response != ""){
    Serial.println(response);
    Serial.println("\n");
    // Check whether the transferred data is inserted successfully or not:
    if(response.indexOf("Data Inserted Successfully!") > 0){
      digitalWrite(green, HIGH);
    }else{
      digitalWrite(red, HIGH);
    }
  }
  // Turn off LEDs:
  delay(3000);
  digitalWrite(green, LOW);
  digitalWrite(red, LOW);
}

build_neural_network_model.py

Python
# Vegetables and Fruits Ripeness Detection by Color w/ TensorFlow
#
# Windows, Linux, or Ubuntu
#
# By Kutluhan Aktar
#
# Collate spectral color data of varying fruits and vegetables and interpret this data set w/ a neural network model to predict ripening stages. 
#
#
# For more information:
# https://www.theamplituhedron.com/projects/Vegetables_and_Fruits_Ripeness_Detection_by_Color_w_TensorFlow/

import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# Create a class to build a neural network model after getting, visualizing, and scaling (normalizing) the data set of fruits and vegetables based on spectral color.
class Ripeness_Detection:
    def __init__(self, data):
        self.df = data
        self.inputs = []
        self.labels = []
        # Define class names for each ripening stage based on color.
        self.ripeness_class_names = ["Early Ripe", "Partially Ripe", "Ripe", "Decay"]
    # Create graphics for requested columns.
    def graphics(self, column_1, column_2, x_label, y_label):
        # Show requested columns from the data set:
        plt.style.use("dark_background")
        plt.gcf().canvas.set_window_title('Vegetables and Fruits Ripeness Detection by Color')
        plt.hist2d(self.df[column_1], self.df[column_2], cmap='RdBu')
        plt.colorbar()
        plt.xlabel(x_label)
        plt.ylabel(y_label)
        plt.title(x_label)
        plt.show()
    # Visualize data before creating and feeding the neural network model.
    def data_visualization(self):
        # Scrutinize requested columns to build a model with appropriately formatted data:
        self.graphics('F1', 'NIR_1', 'F1(405-425nm)', 'Near-Infrared')
        self.graphics('F2', 'NIR_1', 'F2(435-455nm)', 'Near-Infrared')
        self.graphics('F3', 'NIR_1', 'F3(470-490nm)', 'Near-Infrared')
        self.graphics('F4', 'NIR_1', 'F4(505-525nm)', 'Near-Infrared')
        self.graphics('F5', 'NIR_2', 'F5(545-565nm)', 'Near-Infrared')
        self.graphics('F6', 'NIR_2', 'F6(580-600nm)', 'Near-Infrared')
        self.graphics('F7', 'NIR_2', 'F7(620-640nm)', 'Near-Infrared')
        self.graphics('F8', 'NIR_2', 'F8(670-690nm)', 'Near-Infrared')
    # Assign labels for each fruit or vegetable reading according to the pre-defined classes.
    def define_and_assign_labels(self):
        self.labels = self.df.pop("Ripeness")
    # Scale (normalize) data depending on the neural network model and define inputs.
    def scale_data_and_define_inputs(self):
        self.df["scaled_F1"] = self.df.pop("F1") / 1000
        self.df["scaled_F2"] = self.df.pop("F2") / 1000
        self.df["scaled_F3"] = self.df.pop("F3") / 1000
        self.df["scaled_F4"] = self.df.pop("F4") / 1000
        self.df["scaled_F5"] = self.df.pop("F5") / 1000
        self.df["scaled_F6"] = self.df.pop("F6") / 1000
        self.df["scaled_F7"] = self.df.pop("F7") / 1000
        self.df["scaled_F8"] = self.df.pop("F8") / 1000
        self.df["scaled_NIR_1"] = self.df.pop("NIR_1") / 1000
        self.df["scaled_NIR_2"] = self.df.pop("NIR_2") / 1000
        # Create the inputs array using the scaled variables:
        for i in range(len(self.df)):
            self.inputs.append(np.array([self.df["scaled_F1"][i], self.df["scaled_F2"][i], self.df["scaled_F3"][i], self.df["scaled_F4"][i], self.df["scaled_F5"][i], self.df["scaled_F6"][i], self.df["scaled_F7"][i], self.df["scaled_F8"][i], self.df["scaled_NIR_1"][i], self.df["scaled_NIR_2"][i]]))
        self.inputs = np.asarray(self.inputs)
    # Split inputs and labels into training and test sets.
    def split_data(self):
        l = len(self.df)
        # (95%, 5%) - (training, test)
        self.train_inputs = self.inputs[0:int(l*0.95)]
        self.test_inputs = self.inputs[int(l*0.95):]
        self.train_labels = self.labels[0:int(l*0.95)]
        self.test_labels = self.labels[int(l*0.95):]
    # Build and train an artificial neural network (ANN) to make predictions on the ripening stages of fruits and vegetables based on spectral color.
    def build_and_train_model(self):
        # Build the neural network:
        self.model = keras.Sequential([
            keras.Input(shape=(10,)),
            keras.layers.Dense(16),
            keras.layers.Dense(32),
            keras.layers.Dense(64),
            keras.layers.Dense(128),
            keras.layers.Dense(256),
            keras.layers.Dense(4, activation='softmax')
        ])
        # Compile:
        self.model.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics=['accuracy'])
        # Train:
        self.model.fit(self.train_inputs, self.train_labels, epochs=19)
        # Test the accuracy:
        print("\n\nModel Evaluation:")
        test_loss, test_acc = self.model.evaluate(self.test_inputs, self.test_labels) 
        print("Evaluated Accuracy: ", test_acc)
    # Save the model for further usage:
    def save_model(self):
        self.model.save("E:\PYTHON\Vegetables_and_Fruits_Ripeness_Detection\model\ANN_Ripeness_Detection.h5")
    # Run Artificial Neural Network (ANN):
    def Neural_Network(self, save):
        self.define_and_assign_labels()
        self.scale_data_and_define_inputs()
        self.split_data()
        self.build_and_train_model()
        if save:
            self.save_model()
    # Make ripening stage [0 - 3] predictions with the saved model:
    def make_predictions(self):
        saved_model = keras.models.load_model("E:\PYTHON\Vegetables_and_Fruits_Ripeness_Detection\model\ANN_Ripeness_Detection.h5")
        prediction_array = np.array([
            [0.131,0.148,0.184,0.765,1,1,1,0.809,0.276,0.273],
            [0.143,0.122,0.157,0.228,1,1,1,1,0.335,0.334],
            [0.038,0.032,0.032,0.030,0.042,0.055,0.467,0.880,0.100,0.102],
            [0.002,0.001,0.001,0.002,0.003,0.003,0.002,0.007,0.010,0.012]
        ])
        predictions = saved_model.predict(prediction_array)
        print("\n\nModel Predictions:\n")
        for i in range(len(prediction_array)):
            print("INPUT[" + str(i+1) + "] => ", self.ripeness_class_names[np.argmax(predictions[i])])
            
# Read the collated data set of fruits and vegetables:
csv_path = "E:\PYTHON\Vegetables_and_Fruits_Ripeness_Detection\spectral_color_database.csv"
df = pd.read_csv(csv_path)

# Define a new class object named 'fruits_and_vegetables':
fruits_and_vegetables = Ripeness_Detection(df)

# Visualize data:
#fruits_and_vegetables.data_visualization()

# Artificial Neural Network (ANN):
fruits_and_vegetables.Neural_Network(False)

# Make predictions:
#fruits_and_vegetables.make_predictions()

index.php (web app)

PHP
<?php

# Check the incoming data packet:
if(isset($_GET["F1"]) && isset($_GET["F2"]) && isset($_GET["F3"]) && isset($_GET["F4"]) && isset($_GET["F5"]) && isset($_GET["F6"]) && isset($_GET["F7"]) && isset($_GET["F8"]) && isset($_GET["nir_1"]) && isset($_GET["nir_2"]) && isset($_GET["class"])){
	# Create the data array.
	$data = array($_GET["F1"], $_GET["F2"], $_GET["F3"], $_GET["F4"], $_GET["F5"], $_GET["F6"], $_GET["F7"], $_GET["F8"], $_GET["nir_1"], $_GET["nir_2"], $_GET["class"], date("m/d"));
    # Insert the recently generated data array into the CSV file as a new row.	
	$file = fopen("spectral_color_database.csv", "a");
	fputcsv($file, $data);
	fclose($file);
	// Print result:
	echo "Data Inserted Successfully!";
}else{
	echo "Waiting for data from the AS7341 sensor to insert...";
}

?>

Credits

Kutluhan Aktar

Kutluhan Aktar

82 projects • 307 followers
AI & Full-Stack Developer | @EdgeImpulse | @Particle | Maker | Independent Researcher

Comments

Related channels and tags
  • Arduino
  • DFRobot
  • Food & Drinks
  • Home Automation
  • Internet of Things
  • Machine Learning & AI
  • Microsoft
  • Raspberry Pi
  • SparkFun
  • TensorFlow