Team Meta Express Bus: minji kim, Andreas Kraft, SeungMyeong Jeong, Bob Flynn, Miguel Angel Reina Ortega, Laurent Velez, Samir Medjiah, Xavier Piednoir, Wonbae Son, 안일엽

Published November 14, 2021 © Apache-2.0

Virtual Store System using digital twins and oneM2M

We started with the idea of connecting real stores and virtual stores in the real world through the Mobius platform.

AdvancedWork in progressOver 3 days585

Virtual Store System using digital twins and oneM2M

Things used in this project

Hardware components

NVIDIA Jetson Nano Developer Kit

Raspberry Pi 4 Model B

DFRobot I2C 16x2 Arduino LCD Display Module

load cell 10Kg

SparkFun Load Cell Amplifier - HX711

Webcam, Logitech® HD Pro

Software apps and online services

oneM2M

Unity

Android Studio

Story

1. Project Outline

2. Things used in this project

2.1 Hardware components

Jetson Nano Developer Kit
Raspberry Pi 4
I2C lcd
Load Cell 10kg
HX711 Load Cell Amplifer
Webcam

2.2 Software apps and online services

Mobius
Unity
WebGL
Android Studio
Postman
Firebase
Visual Studio Code
nCube
3DS MAX

3. Story

3.1 architecture

architecture

Call Flow

3.2 Prerequistites

*(Android Studio) permission

<uses-permission android:name="android.permission.INTERNET" />

<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />

*(Jetson Nano Developer Kit) Initial installation of gray board

$ sudo pip3 install opencv_contrib_python

$ git clone https://github.com/PINTO0309/mediapipe-bin && cd mediapipe-bin

$.v0.8.5/numpy119x/mediapipe-0.8.5_cuda102-cp36-cp36m-linux_aarch64_numpy119x_jetsonnano_L4T32.5.1_download.sh

$ sudo pip3 install \

numpy-1.19.4-cp36-none-manylinux2014_aarch64.whl \

mediapipe-0.8.5_cuda102-cp36-none-linux_aarch64.whl

$ sudo pip3 install opencv-python dataclasses

*(Raspberry Pi 4) nCube installation

$ git clone https://github.com/IoTKETI/nCube-Thyme-Nodejs.git

4. Web page

A web page linked to login was implemented using a fire base. If you succeed in logging in, you can access the virtual store.

Firebase - Authentication

5. manage app

Express each ordered quantity in Listview format. When refreshing the page, the app connects to the Mobius server and sends data from sever to the app. Also, the manager can see the UID that ordered the items when he clicks each list.

Shopping Basket

6. mobius resource browser

Mobius resource browser

7. unity

Unity engine optimized for 3D modeling for metaverse implementation.

Virtual Store

8. real store

real mart

raspberry Pi 4 - Load cell, HX711, LCD

Determine the number of quantities in the real world by changing the value of the weight sensor.

The manager sees the contents of the items ordered from the virtual store through the app and adds them to the real shopping basket.

9. expectation effectiveness

The expected effect of the application service we implemented has advantages for many in that builds a platform where both real and virtual stores can coexist.

In implementing this, We think it is a system that can keep pace with the direction of the future era in that it uses the Mobius platform to efficiently exchange data and easily connect the real world and the virtual world.

10. Demo

10.1 Virtual StoretoReal Store

10.2 Real Store to Virtual Store

Members : Kim Min Ji, Kim Kyoung Hwan, Weon Ju Yeon (Sejong Univ, Korea)

Code

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class ButtonEvent : MonoBehaviour
{
    public class JsonData
    {
        public string ctname;
        public string con;


        public JsonData() { }

        public JsonData(string ctname, string con)
        {
            this.ctname = ctname;
            this.con = con;
        }

        public void Print()
        {
            Debug.Log("ctname = " + ctname);
            Debug.Log("con = " + con);
        }
    }
    static public string ObjectToJson(object obj)
    {
        return JsonUtility.ToJson(obj);
    }
    public void OnClickSend_water()
    {

        TCP_connection tcp = GameObject.Find("hand_line1").GetComponent<TCP_connection>();
        counting cnt = GameObject.Find("Count_txt_water").GetComponent<counting>();


        string cnt_string_water = cnt.count.ToString();
        JsonData jd = new JsonData("Virtual_water", cnt_string_water);
        string jsondata = ObjectToJson(jd);
        string conv_json = jsondata + "<EOF>";

        tcp.SendMessage_(conv_json);


    }
    public void OnClickSend_beer()
    {

        TCP_connection tcp = GameObject.Find("hand_line1").GetComponent<TCP_connection>();
        counting cnt = GameObject.Find("Count_txt_beer").GetComponent<counting>();


        string cnt_string_beer = cnt.count.ToString();
        JsonData jd = new JsonData("Virtual_beer", cnt_string_beer);
        string jsondata = ObjectToJson(jd);
        string conv_json = jsondata + "<EOF>";

        tcp.SendMessage_(conv_json);

    }
    public void OnClickSend_paper()
    {

        TCP_connection tcp = GameObject.Find("hand_line1").GetComponent<TCP_connection>();
        counting cnt = GameObject.Find("Count_txt_A4").GetComponent<counting>();


        string cnt_string_a4 = cnt.count.ToString();
        JsonData jd = new JsonData("Virtual_paper", cnt_string_a4);
        string jsondata = ObjectToJson(jd);
        string conv_json = jsondata + "<EOF>";

        tcp.SendMessage_(conv_json);
 
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;

public class num_script_a4 : MonoBehaviour
{
    [SerializeField] Text num_txt;

    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        counting cnt = GameObject.Find("Count_txt_A4").GetComponent<counting>();
        string count_t = cnt.count.ToString();
        Debug.Log("a4 count : " + count_t);
        num_txt.text = count_t;
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
public class num_script_beer : MonoBehaviour
{
    [SerializeField] Text num_txt;
    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        counting cnt = GameObject.Find("Count_txt_beer").GetComponent<counting>();
        string count_t = cnt.count.ToString();
        num_txt.text = count_t;
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
public class num_script_water : MonoBehaviour
{
    [SerializeField] Text num_txt;
    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        counting cnt = GameObject.Find("Count_txt_water").GetComponent<counting>();
        string count_t = cnt.count.ToString();
        num_txt.text = count_t;
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;

public class counting : MonoBehaviour
{
    [SerializeField] Text count_txt;
    public int count = 0;
    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        count_txt.text = count.ToString();
    }
    public void Click_plus()
    {
        count = count + 1;
    }
    public void Click_minus()
    {
        if (count > 0)
        {
            count = count - 1;
        }
        
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Networking;
using UnityEngine.UI;
using System.IO;
using System.Net;
using System;

public class get_uid_http : MonoBehaviour
{
    [SerializeField] Text UID_TEXT;
    public class m2m
    {
        public m2m_2 m2m_cin;
    }

    [System.Serializable]

    public class m2m_2
    {
        public string pi;
        public string ri;
        public string ty;
        public string ct;
        public string st;
        public string rn;
        public string lt;
        public string et;
        public string cs;
        public string cr;
        public string con;
    }
    // Start is called before the first frame update
    void Start()
    {
        Download_uid("http://203.253.128.161:7579/Mobius/AISL_VirtualStore/Uid/la");
    }
    public void Download_uid(string url)
    {
        StartCoroutine(GetText_uid(url));
    }
    IEnumerator GetText_uid(string url)
    {
        UnityWebRequest www = new UnityWebRequest(url);
        DownloadHandlerBuffer text = new DownloadHandlerBuffer();
        www.SetRequestHeader("Accept", "application/json");
        www.SetRequestHeader("X-M2M-RI", "12345");
        www.SetRequestHeader("X-M2M-Origin", "SOrigin");

        www.downloadHandler = text;
        yield return www.SendWebRequest();

        if (!(www.isNetworkError || www.isHttpError))
        {
            Debug.Log("UID");
            string sensor_text = text.text;
            string convert_cin = sensor_text.Replace("m2m:cin", "m2m_cin");
            var con_data = JsonUtility.FromJson<m2m>(convert_cin);
            Debug.Log(con_data.m2m_cin.con);
            UID_TEXT.text = con_data.m2m_cin.con;
        }

    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class hand_move : MonoBehaviour
{
    public GameObject parents;

    public GameObject WRIST;
    public GameObject THUMB_CMC;
    public GameObject THUMB_MCP;
    public GameObject THUMB_IP;
    public GameObject THUMB_TIP;
    public GameObject INDEX_FINGER_MCP;
    public GameObject INDEX_FINGER_PIP;
    public GameObject INDEX_FINGER_DIP;
    public GameObject INDEX_FINGER_TIP;
    public GameObject MIDDLE_FINGER_MCP;
    public GameObject MIDDLE_FINGER_PIP;
    public GameObject MIDDLE_FINGER_DIP;
    public GameObject MIDDLE_FINGER_TIP;
    public GameObject RING_FINGER_MCP;
    public GameObject RING_FINGER_PIP;
    public GameObject RING_FINGER_DIP;
    public GameObject RING_FINGER_TIP;
    public GameObject PINKY_MCP;
    public GameObject PINKY_PIP;
    public GameObject PINKY_DIP;
    public GameObject PINKY_TIP;

    // Start is called before the first frame update
    void Start()
    {
        
    }

    // Update is called once per frame
    void Update()
    {
        float x_multiplex = -2.5f;
        float y_multiplex = -1.0f;
        float z_multiplex = 4.0f;
        Vector3 parents_vec = parents.transform.position;
        TCP_connection hand_vec = GameObject.Find("hand_line1").GetComponent<TCP_connection>();
        WRIST.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.WRIST.y, y_multiplex * hand_vec.hand_.WRIST.z, z_multiplex * hand_vec.hand_.WRIST.x);

        THUMB_CMC.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.THUMB_CMC.y, y_multiplex * hand_vec.hand_.THUMB_CMC.z, z_multiplex * hand_vec.hand_.THUMB_CMC.x);
        THUMB_MCP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.THUMB_MCP.y, y_multiplex * hand_vec.hand_.THUMB_MCP.z, z_multiplex * hand_vec.hand_.THUMB_MCP.x);
        THUMB_IP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.THUMB_IP.y, y_multiplex * hand_vec.hand_.THUMB_IP.z, z_multiplex * hand_vec.hand_.THUMB_IP.x);
        THUMB_TIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.THUMB_TIP.y, y_multiplex * hand_vec.hand_.THUMB_TIP.z, z_multiplex * hand_vec.hand_.THUMB_TIP.x);

        INDEX_FINGER_MCP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.INDEX_FINGER_MCP.y, y_multiplex * hand_vec.hand_.INDEX_FINGER_MCP.z, z_multiplex * hand_vec.hand_.INDEX_FINGER_MCP.x);
        INDEX_FINGER_PIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.INDEX_FINGER_PIP.y, y_multiplex * hand_vec.hand_.INDEX_FINGER_PIP.z, z_multiplex * hand_vec.hand_.INDEX_FINGER_PIP.x);
        INDEX_FINGER_DIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.INDEX_FINGER_DIP.y, y_multiplex * hand_vec.hand_.INDEX_FINGER_DIP.z, z_multiplex * hand_vec.hand_.INDEX_FINGER_DIP.x);
        INDEX_FINGER_TIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.INDEX_FINGER_TIP.y, y_multiplex * hand_vec.hand_.INDEX_FINGER_TIP.z, z_multiplex * hand_vec.hand_.INDEX_FINGER_TIP.x);

        MIDDLE_FINGER_MCP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.MIDDLE_FINGER_MCP.y, y_multiplex * hand_vec.hand_.MIDDLE_FINGER_MCP.z, z_multiplex * hand_vec.hand_.MIDDLE_FINGER_MCP.x);
        MIDDLE_FINGER_PIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.MIDDLE_FINGER_PIP.y, y_multiplex * hand_vec.hand_.MIDDLE_FINGER_PIP.z, z_multiplex * hand_vec.hand_.MIDDLE_FINGER_PIP.x);
        MIDDLE_FINGER_DIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.MIDDLE_FINGER_DIP.y, y_multiplex * hand_vec.hand_.MIDDLE_FINGER_DIP.z, z_multiplex * hand_vec.hand_.MIDDLE_FINGER_DIP.x);
        MIDDLE_FINGER_TIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.MIDDLE_FINGER_TIP.y, y_multiplex * hand_vec.hand_.MIDDLE_FINGER_TIP.z, z_multiplex * hand_vec.hand_.MIDDLE_FINGER_TIP.x);

        RING_FINGER_MCP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.RING_FINGER_MCP.y, y_multiplex * hand_vec.hand_.RING_FINGER_MCP.z, z_multiplex * hand_vec.hand_.RING_FINGER_MCP.x);
        RING_FINGER_PIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.RING_FINGER_PIP.y, y_multiplex * hand_vec.hand_.RING_FINGER_PIP.z, z_multiplex * hand_vec.hand_.RING_FINGER_PIP.x);
        RING_FINGER_DIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.RING_FINGER_DIP.y, y_multiplex * hand_vec.hand_.RING_FINGER_DIP.z, z_multiplex * hand_vec.hand_.RING_FINGER_DIP.x);
        RING_FINGER_TIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.RING_FINGER_TIP.y, y_multiplex * hand_vec.hand_.RING_FINGER_TIP.z, z_multiplex * hand_vec.hand_.RING_FINGER_TIP.x);

        PINKY_MCP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.PINKY_MCP.y, y_multiplex * hand_vec.hand_.PINKY_MCP.z, z_multiplex * hand_vec.hand_.PINKY_MCP.x);
        PINKY_PIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.PINKY_PIP.y, y_multiplex * hand_vec.hand_.PINKY_PIP.z, z_multiplex * hand_vec.hand_.PINKY_PIP.x);
        PINKY_DIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.PINKY_DIP.y, y_multiplex * hand_vec.hand_.PINKY_DIP.z, z_multiplex * hand_vec.hand_.PINKY_DIP.x);
        PINKY_TIP.transform.position = parents_vec + new Vector3(x_multiplex * hand_vec.hand_.PINKY_TIP.y, y_multiplex * hand_vec.hand_.PINKY_TIP.z, z_multiplex * hand_vec.hand_.PINKY_TIP.x);
    }
}