Understanding Gesture Knowledge Virtual Character Assistant

using UnityEngine;
using LLMUnity;
using UnityEngine.UI;
using UnityEngine.EventSystems;
using UnityEngine.InputSystem;
//using static System.Net.Mime.MediaTypeNames;

namespace LLMUnitySamples
{
    public class SimpleInteraction : MonoBehaviour
    {
        [SerializeField] private GameObject Maincamera;
        [SerializeField] private GameObject NPC_camera;
        public Transform avatar;
        public LLMCharacter llmCharacter;
        public InputField playerText;
        public Text AIText;
        private TexttoSpeech texttospeech;
        private string aitext;

        void Start()
        {
            playerText.onSubmit.AddListener(onInputFieldSubmit);
            playerText.Select();
            texttospeech = FindObjectOfType<TexttoSpeech>();
        }

        void onInputFieldSubmit(string message)
        {
            playerText.interactable = false;
            AIText.text = "...";
            _ = llmCharacter.Chat(message, SetAIText, AIReplyComplete);
        }

        public void SetAIText(string text)
        {
            //AIText.text = text;
            aitext = text;
        }
        public void OnTextToSpeechPlaybackStarted()
        {
            AIText.text = aitext;
        }

        public void AIReplyComplete()
        {
            playerText.interactable = true;
            playerText.Select();
            playerText.text = "";
            if (texttospeech != null)
            {
                texttospeech.SynthesizeSpeech(aitext);
            }
        }

        public void CancelRequests()
        {
            llmCharacter.CancelRequests();
            AIReplyComplete();
        }

        public void ExitGame()
        {
            Debug.Log("Exit button clicked");
            Application.Quit();
            avatar.GetComponent<PlayerInput>().enabled = true;
            Maincamera.SetActive(true);
            NPC_camera.SetActive(false);
            Cursor.visible = false;
            Cursor.lockState = CursorLockMode.Locked;
        }
        bool onValidateWarning = true;
        void OnValidate()
        {
            if (onValidateWarning && !llmCharacter.remote && llmCharacter.llm != null && llmCharacter.llm.model == "")
            {
                Debug.LogWarning($"Please select a model in the {llmCharacter.llm.gameObject.name} GameObject!");
                onValidateWarning = false;
            }
        }
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;
using System.Text;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using TMPro;

public class UDP_hand : MonoBehaviour
{
    private Thread receiveThread;
    private UdpClient client;
    public int port = 5015;
    public bool startReceiving = true;
    public bool printToConsole = false;
    public TextMeshProUGUI textMeshPro;

    private Queue<string> dataQueue = new Queue<string>();
    private readonly int maxDataCount = 5;
    private bool newDataAvailable = false;

    private void Start()
    {
        receiveThread = new Thread(new ThreadStart(ReceiveData));
        receiveThread.IsBackground = true;
        receiveThread.Start();
    }

    private void OnDestroy()
    {
        startReceiving = false;
        if (receiveThread != null && receiveThread.IsAlive)
        {
            receiveThread.Abort();
        }
        if (client != null)
        {
            client.Close();
        }
    }

    private void ReceiveData()
    {
        client = new UdpClient(port);
        while (startReceiving)
        {
            try
            {
                IPEndPoint anyIP = new IPEndPoint(IPAddress.Any, 0);
                byte[] dataByte = client.Receive(ref anyIP);
                string data = Encoding.UTF8.GetString(dataByte);

                lock (dataQueue)
                {
                    if (dataQueue.Count >= maxDataCount)
                    {
                        dataQueue.Dequeue(); 
                    }
                    dataQueue.Enqueue(data); 
                }

                newDataAvailable = true;

                if (printToConsole)
                {
                    print(data);
                }
            }
            catch (Exception err)
            {
                print(err.ToString());
            }
        }
    }

    private void Update()
    {
        if (newDataAvailable)
        {
            int okCount = 0;
            int hiCount = 0;
            lock (dataQueue)
            {
                foreach (string data in dataQueue)
                {
                    string[] words = data.Split(new char[] { ' ', '\t', '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries);
                    okCount += Array.FindAll(words, word => word.Trim().ToLower() == "ok").Length;
                    hiCount += Array.FindAll(words, word => word.Trim().ToLower() == "hi").Length;
                }
            }

            if (okCount >= 3)
            {
                textMeshPro.text = "How to get rare Gwent cards?";
            }
            else if (hiCount >= 3)
            {
                textMeshPro.text = "How to get Netral Card:Xili?";
            }
            else
            {
                textMeshPro.text = "You got another one?";
            }

            newDataAvailable = false;
        }
        //else
        //{
            //textMeshPro.text = "No gesture recognized!";
        //}
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Amazon.Polly;
using Amazon.Runtime;
using Amazon;
using Amazon.Polly.Model;
using System.IO;
using UnityEngine.Networking;
using System.Threading.Tasks;
using UnityEngine.Events;
public class TexttoSpeech : MonoBehaviour
{
    [SerializeField] private AudioSource audiosource;
    private string textTosynthhesize;
    public string accessKey;
    public string secretKey;
    public UnityEvent OnreplyReceived;
    public UnityEvent OnPlaybackStarted;
    private async void Start()
    {

    }
    public async void SynthesizeSpeech(string text)
    {
        var credentials = new BasicAWSCredentials(accessKey, secretKey);
        var client = new AmazonPollyClient(credentials, RegionEndpoint.EUCentral1);
        textTosynthhesize = text;

        var request = new SynthesizeSpeechRequest()
        {
            Text = textTosynthhesize,
            Engine = Engine.Neural,
            VoiceId = VoiceId.Aria,
            //VoiceId.Zhiyu,
            OutputFormat = OutputFormat.Mp3
        };

        var response = await client.SynthesizeSpeechAsync(request);

        WriteIntoFile(response.AudioStream);

        using (var www = UnityWebRequestMultimedia.GetAudioClip($"{Application.persistentDataPath}/audio.mp3", AudioType.MPEG))
        {
            var op = www.SendWebRequest();
            while (!op.isDone) await Task.Yield();

            var clip = DownloadHandlerAudioClip.GetContent(www);

            audiosource.clip = clip;
            audiosource.Play();
            OnreplyReceived.Invoke();
            OnPlaybackStarted.Invoke();
        }
    }




    private void WriteIntoFile(Stream stream)
    {
        using (var filestream = new FileStream($"{Application.persistentDataPath}/audio.mp3", FileMode.Create))
        {
            byte[] buffer = new byte[8 * 1024];
            int bytesRead;

            while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
            {
                filestream.Write(buffer, 0, bytesRead);

            }
        }
    }

}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Windows;
using UnityEngine.InputSystem;
using System.Threading.Tasks;

public class Dialog_with_NPC : MonoBehaviour
{
    [SerializeField] private GameObject Maincamera;
    [SerializeField] private GameObject NPC_camera;

    [SerializeField] private GameObject Standing_Point;

    private Transform avatar;
 
    private async void OnTriggerEnter(Collider other)
    {
        if (other.CompareTag("Player"))
        {
            avatar = other.transform;
            avatar.GetComponent<PlayerInput>().enabled = false;
            await Task.Delay(20);
            avatar.position = Standing_Point.transform.position;
            avatar.rotation = Standing_Point.transform.rotation;

            Maincamera.SetActive(false);
            NPC_camera.SetActive(true);
            Cursor.visible = true;
            Cursor.lockState = CursorLockMode.None;
        }
    }
}