Hackster is hosting Hackster Holidays, Ep. 6: Livestream & Giveaway Drawing. Watch previous episodes or stream live on Monday!Stream Hackster Holidays, Ep. 6 on Monday!
Kutluhan Aktar
Published © CC BY

Barcode Based Nutrient Profiling and Food Labelling w/ TF

Collect nutrition facts by barcodes to distinguish healthy and unhealthy foods w/ a neural network model predicting Nutri-Score classes.

ExpertFull instructions provided5 hours8,992

Things used in this project

Hardware components

Raspberry Pi 4 Model B
Raspberry Pi 4 Model B
Raspberry Pi 4 or 3B+
×1
Raspberry Pi 3 Model B+
Raspberry Pi 3 Model B+
Raspberry Pi 4 or 3B+
×1
DFRobot GM65 QR & Barcode Scanner Module
×1
7'' HDMI Display with Capacitive Touchscreen
DFRobot 7'' HDMI Display with Capacitive Touchscreen
×1
SparkFun HDMI to Micro HDMI Adapter
For Raspberry Pi 4
×1

Software apps and online services

Raspbian
Raspberry Pi Raspbian
TensorFlow
TensorFlow
Thonny
Visual Studio 2017
Microsoft Visual Studio 2017

Hand tools and fabrication machines

Hot glue gun (generic)
Hot glue gun (generic)
TensorFlow Lite
TensorFlow Lite

Story

Read more

Custom parts and enclosures

product_barcode_list.py

product_database.csv

Schematics

Schematic

Code

index.php (web app)

PHP
<?php

# Define the barcode_scanner class and its functions.
class barcode_scanner{
	# Elicit and print information as to the given barcode - nutrient levels, nutrition facts, etc.
	public function print_product_info($barcode){
		# Make an HTTP Get request to the Open Food Facts JSON API to collate data on the product characteristics with the given barcode.
		$data = json_decode(file_get_contents("https://world.openfoodfacts.org/api/v0/product/".$barcode.".json", TRUE));
		# If the barcode found:
		if($data->status_verbose != "product not found"){
			# Create the query array with the incoming data:
			$query = array(
				"name" => is_null($data->product->brands) ? " " : $data->product->brands,
				"type" => is_null($data->product->product_name) ? " " : $data->product->product_name,
				"manufacturing_places" => (is_null($data->product->manufacturing_places) || $data->product->manufacturing_places == "") ? "Worldwide" : $data->product->manufacturing_places,
				"quantity" => is_null($data->product->product_quantity) ? 100 : (int)$data->product->product_quantity,
				"serving_quantity" => is_null($data->product->serving_quantity) ? 100 : (int)$data->product->serving_quantity,
				"energy-kcal_100g" => is_null($data->product->nutriments->{'energy-kcal_100g'}) ? 0 : $data->product->nutriments->{'energy-kcal_100g'},
				"energy-kcal_serving" => is_null($data->product->nutriments->{'energy-kcal_serving'}) ? $data->product->nutriments->{'energy-kcal_100g'} : $data->product->nutriments->{'energy-kcal_serving'},
				"carbohydrates_100g" => is_null($data->product->nutriments->carbohydrates_100g) ? 0 : $data->product->nutriments->carbohydrates_100g,
				"carbohydrates_serving" => is_null($data->product->nutriments->carbohydrates_serving) ? $data->product->nutriments->carbohydrates_100g : $data->product->nutriments->carbohydrates_serving,
				"sugars_100g" => is_null($data->product->nutriments->sugars_100g) ? 0 : $data->product->nutriments->sugars_100g,
				"sugars_serving" => is_null($data->product->nutriments->sugars_serving) ? $data->product->nutriments->sugars_100g : $data->product->nutriments->sugars_serving,
				"fat_100g" => is_null($data->product->nutriments->fat_100g) ? 0 : $data->product->nutriments->fat_100g,
				"fat_serving" => is_null($data->product->nutriments->fat_serving) ? $data->product->nutriments->fat_100g : $data->product->nutriments->fat_serving,
				"saturated-fat_100g" => is_null($data->product->nutriments->{'saturated-fat_100g'}) ? 0 : $data->product->nutriments->{'saturated-fat_100g'},
				"saturated-fat_serving" => is_null($data->product->nutriments->{'saturated-fat_serving'}) ? $data->product->nutriments->{'saturated-fat_100g'} : $data->product->nutriments->{'saturated-fat_serving'},		
				"fruits_vegetables_100g" => is_null($data->product->nutriments->{'fruits-vegetables-nuts-estimate-from-ingredients_100g'}) ? 0 : $data->product->nutriments->{'fruits-vegetables-nuts-estimate-from-ingredients_100g'},
				"fiber_100g" => (is_null($data->product->nutriments->fiber_100g) || $data->product->nutriments->fiber_100g == "") ? 0 : $data->product->nutriments->fiber_100g,
				"fiber_serving" => (is_null($data->product->nutriments->fiber_serving) || $data->product->nutriments->fiber_serving == "") ? 0 : $data->product->nutriments->fiber_serving,
				"proteins_100g" => is_null($data->product->nutriments->proteins_100g) ? 0 : $data->product->nutriments->proteins_100g,
				"proteins_serving" => is_null($data->product->nutriments->proteins_serving) ? $data->product->nutriments->proteins_100g : $data->product->nutriments->proteins_serving,		
				"salt_100g" => is_null($data->product->nutriments->salt_100g) ? 0 : $data->product->nutriments->salt_100g,
				"salt_serving" => is_null($data->product->nutriments->salt_serving) ? $data->product->nutriments->salt_100g : $data->product->nutriments->salt_serving,
				"sodium_100g" => is_null($data->product->nutriments->sodium_100g) ? 0 : $data->product->nutriments->sodium_100g,
				"sodium_serving" => is_null($data->product->nutriments->sodium_serving) ? $data->product->nutriments->sodium_100g : $data->product->nutriments->sodium_serving,
				"calcium_100g" => (is_null($data->product->nutriments->calcium_100g) || $data->product->nutriments->calcium_100g == "") ? 0 : $data->product->nutriments->calcium_100g,
				"calcium_serving" => is_null($data->product->nutriments->calcium_serving) ? 0 : $data->product->nutriments->calcium_serving,			
				"nutri_score" => is_null($data->product->nutriscore_data->score) ? 0 : $data->product->nutriscore_data->score,
				"nutri_grade" => is_null($data->product->nutriscore_data->grade) ? "Undefined" : $data->product->nutriscore_data->grade,
				"co2_total" => is_null($data->product->ecoscore_data->agribalyse->co2_total) ? 1 : $data->product->ecoscore_data->agribalyse->co2_total,
				"ef_total" => is_null($data->product->ecoscore_data->agribalyse->ef_total) ? 0.25 : $data->product->ecoscore_data->agribalyse->ef_total
			);
			// Print the recent query in JSON with the requested values:
			echo(json_encode($query));
		}	
	}
}

# Define the new 'product' class object.
error_reporting(0);
$product = new barcode_scanner();
# Get the product information of the given barcode:
if(isset($_GET["barcode"]) && $_GET["barcode"] != ""){
	$product->print_product_info($_GET["barcode"]);
}
?>

create_product_database.py

Python
# Barcode Based Nutrient Profiling and Food Labelling w/ TensorFlow
#
# Windows, Linux, or Ubuntu
#
# By Kutluhan Aktar
#
# Collect nutrition facts by barcodes to distinguish healthy and unhealthy foods w/ a neural network model predicting Nutri-Score classes. 
#
#
# For more information:
# https://www.theamplituhedron.com/projects/Barcode-Based-Nutrient-Profiling-and-Food-Labelling-w-TensorFlow

import json
from time import sleep
import requests
from csv import writer
from product_barcode_list import product_barcode_list

class create_database:
    def __init__(self, server):
        self.server = server
        self.product_info = 0
    # Get information from the PHP web application with the given product barcode.
    def get_information_by_barcode(self, barcode):
        data = requests.get(self.server + "?barcode=" + barcode)
        # If incoming data:
        if not (data.text == ""):
            d = json.loads(data.text)
            self.product_data = [barcode, d["quantity"], d["serving_quantity"], d["energy-kcal_100g"], d["carbohydrates_100g"], d["sugars_100g"], d["fat_100g"], d["saturated-fat_100g"], d["fruits_vegetables_100g"], d["fiber_100g"], d["proteins_100g"], d["salt_100g"], d["sodium_100g"], d["calcium_100g"], d["nutri_score"], d["nutri_grade"], d["co2_total"], d["ef_total"]]
    # List the product information depending on the given barcode list to create the database (CSV).
    def list_product_information_as_CSV(self, barcode_list):
        i = 0;
        l = len(barcode_list)
        for product_barcode in barcode_list:
            # Fetch data for each product:
            self.get_information_by_barcode(product_barcode)
            # Insert data to the CSV file for each product:
            with open("product_database.csv", "a", newline="") as f:
                # Add a new row with the product information to the file:
                writer(f).writerow(self.product_data)
                f.close()
            # Print the remaining rows:
            i+=1
            print("Fetching data... (" + str(i) + " / " + str(l) + ")")
            sleep(0.2)
        print("!!! Database created successfully !!!")

# Define a new class object named 'database':
database = create_database("http://192.168.1.20/Barcode_Product_Scanner/")

# Create the database with the product barcode list:
database.list_product_information_as_CSV(product_barcode_list)

build_neural_network_model.py

Python
# Barcode Based Nutrient Profiling and Food Labelling w/ TensorFlow
#
# Windows, Linux, or Ubuntu
#
# By Kutluhan Aktar
#
# Collect nutrition facts by barcodes to distinguish healthy and unhealthy foods w/ a neural network model predicting Nutri-Score classes. 
#
#
# For more information:
# https://www.theamplituhedron.com/projects/Barcode-Based-Nutrient-Profiling-and-Food-Labelling-w-TensorFlow

import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# Create a class to build a neural network after getting, visualizing, and scaling (normalizing) the product information data set (nutrient levels, nutrition facts, etc.).
class Nutrient_Profiling:
    def __init__(self, data):
        self.df = data
        self.inputs = []
        self.labels = []
    # Create graphics for requested columns.
    def graphics(self, column_1, column_2, x_label, y_label):
        # Show requested columns from the data set:
        plt.style.use("dark_background")
        plt.gcf().canvas.set_window_title('Barcode Based Nutrient Profiling')
        plt.hist2d(self.df[column_1], self.df[column_2], cmap='RdBu')
        plt.colorbar()
        plt.xlabel(x_label)
        plt.ylabel(y_label)
        plt.title(x_label)
        plt.show()
    # Visualize data before creating and feeding the neural network model.
    def data_visualization(self):
        # Scrutinize requested columns to build a model with appropriately formatted data:
        self.graphics('carbohydrates_100g', 'energy-kcal_100g', 'Carbohydrates (g)', 'Energy (kcal)')
        self.graphics('sugars_100g', 'energy-kcal_100g', 'Sugars (g)', 'Energy (kcal)')
        self.graphics('fat_100g', 'energy-kcal_100g', 'Fat (g)', 'Energy (kcal)')
        self.graphics('saturated-fat_100g', 'energy-kcal_100g', 'Saturated fat (g)', 'Energy (kcal)')
        self.graphics('fruits_vegetables_100g', 'energy-kcal_100g', 'Fruits vegetables nuts estimate (%)', 'Energy (kcal)')
        self.graphics('fiber_100g', 'energy-kcal_100g', 'Fiber (g)', 'Energy (kcal)')
        self.graphics('proteins_100g', 'energy-kcal_100g', 'Proteins (g)', 'Energy (kcal)')
        self.graphics('salt_100g', 'energy-kcal_100g', 'Salt (g)', 'Energy (kcal)')
        self.graphics('sodium_100g', 'energy-kcal_100g', 'Sodium (g)', 'Energy (kcal)')
    # Assign labels for each product according to the nutrient-profiling model (altered).
    def define_and_assign_labels(self):
        l = len(self.df)
        for i in range(l):
            # Calculate the Nutri-Score value (altered).
            nutri_score = df["nutri_score"][i]
            calcium = df["calcium_100g"][i] * 1000
            co2 = df["co2_total"][i]
            ef = df["ef_total"][i]
            # Calcium:
            if(calcium > 110):
                nutri_score-=1
            elif(calcium > 300):
                nutri_score-=2
            # CO2 (Environmental impact):
            if(co2 > 1 and co2 <= 3):
                nutri_score+=1
            elif(co2 > 3 and co2 <= 5):
                nutri_score+=2
            elif(co2 > 5):
                nutri_score+=3
            # EF (Ecological footprint):
            if(ef > 0.2 and ef <= 0.4):
                nutri_score+=1
            elif(ef > 0.4 and ef <= 0.9):
                nutri_score+=2
            elif(ef > 0.9):
                nutri_score+=3
            # Assign classes (labels) depending on the Nutri-Score value:
            _class = 0
            if(nutri_score <= 3):
                _class = 0
            elif(nutri_score > 3 and nutri_score <= 11):
                _class = 1
            elif(nutri_score > 11 and nutri_score <= 21):
                _class = 2
            elif(nutri_score > 21):
                _class = 3
            self.labels.append(_class)
        self.labels = np.asarray(self.labels)
    # Scale (normalize) data depending on the neural network model and define inputs.
    def scale_data_and_define_inputs(self):
        self.df["scaled_energy"] = self.df.pop("energy-kcal_100g") / 1000
        self.df["scaled_carbohydrates"] = self.df.pop("carbohydrates_100g") / 100
        self.df["scaled_sugars"] = self.df.pop("sugars_100g") / 100
        self.df["scaled_fat"] = self.df.pop("fat_100g") / 100
        self.df["scaled_saturated"] = self.df.pop("saturated-fat_100g") / 100
        self.df["scaled_fiber"] = self.df.pop("fiber_100g") / 10
        self.df["scaled_proteins"] = self.df.pop("proteins_100g") / 100
        self.df["scaled_salt"] = self.df.pop("salt_100g") / 10
        self.df["scaled_sodium"] = self.df.pop("sodium_100g") / 10
        # Create the inputs array using the scaled variables:
        for i in range(len(self.df)):
            self.inputs.append(np.array([self.df["scaled_energy"][i], self.df["scaled_carbohydrates"][i], self.df["scaled_sugars"][i], self.df["scaled_fat"][i], self.df["scaled_saturated"][i], self.df["scaled_fiber"][i], self.df["scaled_proteins"][i], self.df["scaled_salt"][i], self.df["scaled_sodium"][i]]))
        self.inputs = np.asarray(self.inputs)
    # Split inputs and labels into training and test sets.
    def split_data(self):
        l = len(self.df)
        # (90%, 10%) - (training, test)
        self.train_inputs = self.inputs[0:int(l*0.9)]
        self.test_inputs = self.inputs[int(l*0.9):]
        self.train_labels = self.labels[0:int(l*0.9)]
        self.test_labels = self.labels[int(l*0.9):]
    # Build and train an artificial neural network (ANN) to make predictions on the Nutri-Score values (food health category) based on nutrient profiling.
    def build_and_train_model(self):
        # Build the neural network:
        self.model = keras.Sequential([
            keras.Input(shape=(9,)),
            keras.layers.Dense(256, activation='relu'),
            keras.layers.Dense(512, activation='relu'),
            keras.layers.Dense(1024, activation='relu'),
            keras.layers.Dense(2048, activation='relu'),
            keras.layers.Dense(4, activation='softmax')
        ])
        # Compile:
        self.model.compile(optimizer='adam', loss="sparse_categorical_crossentropy", metrics=['accuracy'])
        # Train:
        self.model.fit(self.train_inputs, self.train_labels, epochs=15)
        # Test the accuracy:
        print("\n\nModel Evaluation:")
        test_loss, test_acc = self.model.evaluate(self.test_inputs, self.test_labels) 
        print("Evaluated Accuracy: ", test_acc)
    # Save the model for further usage on Raspberry Pi:
    def save_model(self):
        self.model.save("E:\PYTHON\Barcode_Based_Nutrient_Profiling\ANN_Nutrient_Profiling.h5")
    # Convert the TensorFlow Keras H5 model (.h5) to a TensorFlow Lite model (.tflite) to run it on Raspberry Pi.
    def convert_TF_model(self, path):
        model = tf.keras.models.load_model(path + ".h5")
        converter = tf.lite.TFLiteConverter.from_keras_model(model)
        tflite_model = converter.convert()
        # Save the TensorFlow Lite model.
        with open(path + '.tflite', 'wb') as f:
            f.write(tflite_model)
        print("\r\nTensorFlow Keras H5 model converted to a TensorFlow Lite model...\r\n")
    # Run Artificial Neural Network (ANN):
    def Neural_Network(self, save):
        self.define_and_assign_labels()
        self.scale_data_and_define_inputs()
        self.split_data()
        self.build_and_train_model()
        if save:
            self.save_model()
            
# Read the generated data set of food products:
csv_path = "E:\PYTHON\Barcode_Based_Nutrient_Profiling\product_database.csv"
df = pd.read_csv(csv_path)

# Define a new class object named 'food_products':
food_products = Nutrient_Profiling(df)

# Visualize data:
#food_products.data_visualization()

# Artificial Neural Network (ANN):
food_products.Neural_Network(True)

# Convert the TensorFlow Keras H5 model to a TensorFlow Lite model:
#food_products.convert_TF_model("E:\PYTHON\Barcode_Based_Nutrient_Profiling\ANN_Nutrient_Profiling")

barcode_food_labelling_tf_lite.py

Python
# Barcode Based Nutrient Profiling and Food Labelling w/ TensorFlow
#
# Raspberry Pi 4 Model B
#
# By Kutluhan Aktar
#
# Collect nutrition facts by barcodes to distinguish healthy and unhealthy foods w/ a neural network model predicting Nutri-Score classes. 
#
#
# For more information:
# https://www.theamplituhedron.com/projects/Barcode-Based-Nutrient-Profiling-and-Food-Labelling-w-TensorFlow

import json
from time import sleep
import requests
import numpy as np
import tensorflow as tf

class barcode_food_labelling:
    def __init__(self, server):
        self.server = server
        self.product_data = 0
        # Define class names for each Nutri-Score (food health category) classes based on nutrient profiling.
        self.nutri_score_class_names = ["Nutritious", "Healthy", "Less Healthy", "Unhealthy"]
    # Get information from the PHP web application with the given product barcode.
    def get_information_by_barcode(self, barcode):
        data = requests.get(self.server + "?barcode=" + barcode)
        # If incoming data:
        if not (data.text == ""):
            self.product_data = json.loads(data.text)
            return True
        else:
            print("\r\nBarcode Not Found!!!")
            return False
    # Format the incoming product information to create the input data for the model.
    def format_incoming_data(self):
        if not(self.product_data == 0):
            # Information:
            self.name = self.product_data["name"]
            self.type = self.product_data["type"]
            self.quantity = str(self.product_data["quantity"]) + " / " + str(self.product_data["serving_quantity"])
            # Data:
            self.energy = self.product_data["energy-kcal_100g"] / 1000
            self.carbohydrates = self.product_data["carbohydrates_100g"] / 100
            self.sugars = self.product_data["sugars_100g"] / 100
            self.fat = self.product_data["fat_100g"] / 100
            self.saturated = self.product_data["saturated-fat_100g"] / 100
            self.fiber = self.product_data["fiber_100g"] / 10
            self.proteins = self.product_data["proteins_100g"] / 100
            self.salt = self.product_data["salt_100g"] / 10
            self.sodium = self.product_data["sodium_100g"] / 10
    # Load the TensorFlow Lite model to predict the Nutri-Score class of the given food product by barcode.
    def run_TensorFlow_Lite_model(self, path):
        # Load the TFLite model and allocate tensors.
        interpreter = tf.lite.Interpreter(model_path=path)
        interpreter.allocate_tensors()
        # Get input and output tensors.
        input_details = interpreter.get_input_details()
        output_details = interpreter.get_output_details()
        # Run the model with the formatted input data:
        input_data = np.array([[self.energy, self.carbohydrates, self.sugars, self.fat, self.saturated, self.fiber, self.proteins, self.salt, self.sodium]], dtype=np.float32)
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()
        # Get output data (label): 
        output_data = interpreter.get_tensor(output_details[0]['index'])
        prediction = self.nutri_score_class_names[np.argmax(output_data)]
        print("\r\n--------------\r\n")
        print("Product => " + self.name + " " + self.type)
        print("Quantity => " + self.quantity)
        print()
        print("Prediction => " + prediction)
        print("\r\n--------------\r\n")

# Define a new class object named 'new_product':
new_product = barcode_food_labelling("http://192.168.1.20/Barcode_Product_Scanner/")

while True:
    # Get barcodes from the barcode scanner:
    barcode = input("\r\nWaiting for reading new barcodes...\r\n")
    # If successful:
    if(barcode.find("K19") < 0):
        if (new_product.get_information_by_barcode(barcode)):
            new_product.format_incoming_data()
            new_product.run_TensorFlow_Lite_model("ANN_Nutrient_Profiling.tflite")

Credits

Kutluhan Aktar

Kutluhan Aktar

82 projects • 310 followers
AI & Full-Stack Developer | @EdgeImpulse | @Particle | Maker | Independent Researcher

Comments

Related channels and tags
  • DFRobot
  • Food & Drinks
  • Food & Drinks
  • Health & Medical Devices
  • Health & Medical Devices
  • Home Automation
  • Home Automation
  • Machine Learning & AI
  • Machine Learning & AI
  • Microsoft
  • Raspberry Pi
  • SparkFun
  • TensorFlow