MetaSoul

Loading...
Searching...
No Matches
ESSubSystem.h
Go to the documentation of this file.
1//Copyright (c) 2022 Emoshape Inc.
2//All Rights Reserved
3
4//This product is protected by copyright and distributed under
5//licenses restricting copying, distribution, and decompilation.
6
7#pragma once
8
9#include "CoreMinimal.h"
10#include "SocketSubsystem.h"
11#include "Common/UdpSocketReceiver.h"
12#include "Interfaces/IHttpRequest.h"
13#include "HttpModule.h"
14#include "Subsystems/GameInstanceSubsystem.h"
15#include "ESSubSystem.generated.h"
16
17USTRUCT(BlueprintType)
18struct FViseme
19{
20 GENERATED_BODY()
21 public:
22 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
23 int32 audio_offset;
24
25 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
26 int32 viseme_id;
27
28 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
29 FString animation;
30};
31
32USTRUCT(BlueprintType)
33struct FAPIViseme {
34
35 GENERATED_BODY()
36 public:
37 int32 privAudioOffset;
38 int32 privVisemeId;
39 FString privAnimation;
40};
41
42 DECLARE_LOG_CATEGORY_EXTERN(EmoShapeCategory, Display, All);
43 DECLARE_DYNAMIC_MULTICAST_DELEGATE(FMetaSoulEventSignature);
44 DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FMetaSoulEventSignatureAudio, const TArray<uint8>&, audio_buffer, int64, audio_length, const TArray<FViseme>&, visems, FString, reply);
45
46 class ESIOClient;
47UENUM(BlueprintType)
48enum PossibleResponses
49{
50 ResponseNone UMETA(DisplayName = "ResponseNone"),
51 ResponseText UMETA(DisplayName = "ResponseText"),
52 ResponseBuffer UMETA(DisplayName = "ResponseBuffer"),
53 ResponseLogin UMETA(DisplayName = "ResponseLogin"),
54 ResponseSER UMETA(DisplayName = "ResponseSER"),
55 ResponseHeartBeat UMETA(DisplayName = "ResponseHeartBeat"),
56};
57
58UCLASS()
59class EMOSHAPESUBSYSTEM_API UESSubSystem : public UGameInstanceSubsystem
60{
61 GENERATED_BODY()
62
63 public:
64
65 virtual void Initialize(FSubsystemCollectionBase& Collection) override;
66
67 ESIOClient* IOClient;
68
71 int32 Port = 2424;
72 int32 BufferSize = 256;
73 FSocket* Socket = ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateSocket(NAME_Stream, TEXT("default"), false);
74 FSocket* ListenSocket;
75 FString CustomLineBreaker = " 5c72";
76 FUdpSocketReceiver* UDPReceiver = nullptr;
77
78 UFUNCTION(BlueprintCallable, Category = "EmoShapeSubSystem")
79 bool StartUDPReceiver(
80 const FString& YourChosenSocketName ,
81 const FString& TheIP ,
82 const int32 ThePort
83 );
84
85 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapeSubSystem)
86 FString UdpText = "";
87
88 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapeSubSystem)
89 FString RemoveIP = "";
90
91 UFUNCTION(BlueprintCallable, Category = "EmoShapeSubSystem")
92 void CloseReceiver();
93
94 void Recv(const FArrayReaderPtr& ArrayReaderPtr, const FIPv4Endpoint& EndPt);
95
98
99
100 FString Message;
101
103
104 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
105 FString MetaSoulSecret; // fnXHexW9pCf5RacfllIgigKGyr8mYsh8
106 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
107 FString Epu_ID = "";
108 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
109 FString AzureKey = "";
110 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
111 FString AzureKeyRegion = "westeurope";
112 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
113 FString OpenAIKey = "";
114 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
115 FString Token;
116
117 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
118 FString ChatHistory = "";
119
121
122
123 UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = EmoShapePlugin)
124 bool bTalkmode = false;
125 UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = EmoShapePlugin)
126 bool bConnected = false;
127 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
128 int32 BytesSent = 0;
129 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
130 TArray<uint8> ReceivedData;
131
132 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
133 TMap<FString, FString> MEmotions;
134 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
135 FString GptData;
136 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
137 TArray<int> VisemeOffset;
138 UPROPERTY(EditDefaultsOnly, BlueprintReadOnly, Category = EmoShapePlugin)
139 TArray<int> VisemeID;
140
141 bool SharedMem = false;
142 void* SharedMemoryHandle;
143 unsigned char* SharedMemoryData;
144 size_t SharedMemorySize;
145 void* SharedMemoryMutex;
146
148
149
150 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
151 void Login();
152
153 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
154 void Logout();
155
156 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
157 void CustomWave(FString type, int32 apex, int32 curve, int32 level, int32 duration, int32 origin);
158
159 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
160 void AudioData(FString text, FString voice_name = "en-US-JennyNeural", bool generateEmo = false, FString chat_history = "", int32 coef = 30);
161
162 UFUNCTION(BlueprintCallable)
163 static TArray<FViseme> parseVisems(const TArray<FViseme>& visems);
164
165 UPROPERTY(BlueprintAssignable, meta=(DisplayName = "on login"))
166 FMetaSoulEventSignature onLogin;
167
168 UPROPERTY(BlueprintAssignable, meta = (DisplayName = "on logout"))
169 FMetaSoulEventSignature onLogout;
170
171 UPROPERTY(BlueprintAssignable)
172 FMetaSoulEventSignature onConnectionFail;
173
174 UPROPERTY(BlueprintAssignable, meta = (DisplayName = "on audio data"))
175 FMetaSoulEventSignatureAudio onAudioData;
176
178
179
180 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
181 void StartUDP();
182
183 //Connects to the EPU SDK
184 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
185 void Connect(const FString & ipAddress);
186
187 //Alternative Method to Connect to the EPU SDK
188 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
189 void ConnectAlternative(const FString & ipAddress);
190
191 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
192 void CheckConnection();
193
194 //Using this while active cause a overhead at the processor, try to call something else first to get a EPU_OFF message
195 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
196 void StartEPU(FString method, FString key);
197 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
198 void StartEPUProcess(FString path, bool silent);
199 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
200 void StopEPUProcess();
201
202 //Beware using this make sure its running first
203 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
204 void PauseEPU();
205
206 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
207 void CloseEPU();
208
209 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
210 void SendCustomMessage(FString Msg, bool end);
211
212 void LongLineSend(FString &Msg);
213 //Same goes here make sure its paused first
214 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
215 void ResumeEPU();
216
217 //Reset EPU
218 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
219 void ResetEPU();
220
221 //Send a message for the EPU to process and generate a emotional response
222 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
223 void SendMessage(FString message);
224
225 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
226 void SendBroadcast(FString message);
227
228 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
229 void SendTextToSpeech(FString message);
230
231 //Set the Sensibility (user and robot messages) of the EPU
232 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
233 void SetSenses(bool robot, FString value);
234
235 //Set the Persistence (user and robot messages) of the EPU
236 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
237 void SetPersistence(bool robot, FString value);
238
239 //Set the Objective Appraisal ON (user and robot messages) of the EPU
240 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
241 void SetObjectiveAppraisal(bool robot);
242
243 //This sets him to read a text and generate a emotion for further use, Example: Copy a text from Barrack Obama in the Wikipedia and send it,
244 //he will generate a emotional response and you can save it. (Max of 100 words)
245 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
246 void SetSimbolicLearning(FString message);
247
248 //Save the SimbolicLearning to a word
249 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
250 void SaveSimbolicLearning(FString word);
251
252 //Erase all learned words and personality(not sure about this one)
253 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
254 void EraseLexic();
255
256 //Inject manually a emotion this can be usefull to be sure something will trigger within the EPU
257 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
258 void InjectEmotion(FString emotion, FString level, FString duration, FString origin_id, FString apex, FString curve);
259
260 //This one is used to ask for a emotional buffer for later retrieval with RecieveMessage.
261 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
262 void AskForEmotionalBuffer();
263
264 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
265 void BroadcastMessageToAllClients(FString message);
266
267 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
268 void EnableDisableToneOfVoiceDetection(bool on);
269
270 //This just close the socket connection, not the EPU
271 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
272 void EnableDisableFaceTracking(bool on);
273
274 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
275 void AddRemoveUserForFaceTracking(FString username, bool add);
276
277 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
278 void SetSpeechRecognition(bool on);
279
280 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
281 void GetCurrentFaceTrackingUser();
282 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
283 void StartSharedMem();
284 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
285 void StopSharedMem();
286 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
287 void GetSupportedLanguages();
288 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
289 void SetLanguages(FString index);
290 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
291 void StopConnection();
292
293 //Handy function to convert the binary response to a FString
294 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
295 FString UTF8toString(const TArray<uint8>& BinaryArray);
296
297 //Usually you will always use the ReceivedData array for this but, I let open if case someone wants to make something different
298 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
299 void UpdateEmotionTMap(TArray<FString> mArray);
300 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
301 void ReadMDADSharedMemory();
302 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
303 void ReadGPTSharedMemory();
304 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
305 void ReadVisemeSharedMemory();
306
310
311 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
312 void ReceiveMessage();
313
314 //-------- HTTP SECTION -------//
315
316 FString HeartBeatParam;
317
318 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
319 void SendPut(FString PathSuffix, UESJsonObject* Json, PossibleResponses functionName);
320
321 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
322 void SendPost(FString PathSuffix, UESJsonObject* Json, PossibleResponses functionName);
323
324 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
325 void SendGet(FString PathSuffix, PossibleResponses functionName);
326
327 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
328 void PostSER(FString Pathsuffix, FSecretParam Secret_Param, TArray<uint8> audio);
329
330 auto GetResponse(FString name) -> void(UESSubSystem::*)(TSharedPtr<IHttpRequest, ESPMode::ThreadSafe>, TSharedPtr<IHttpResponse, ESPMode::ThreadSafe>, bool);
331
332 void PostResponse(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
333 void PostResponseAudioData(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
334 void PostResponseBuffer(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
335 void PostResponseText(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
336 void PostResponseLogin(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
337 void PostResponseLoginWeb(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
338 void PostResponseLogout(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
339 void PostResponseSER(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
340 void PostResponseHeartBeat(FHttpRequestPtr Request, FHttpResponsePtr Response, bool bWasSuccessful);
341
342 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
343 void ConvertSERToEmotions(TArray<int> SEREmotionalValues);
344
345 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
346 void HandleSERResponse(FString text);
347
348 //-------- AUDIO SECTION -------//
349
350 UFUNCTION(BlueprintCallable, Category = "EmoShapePlugin")
351 void StereoToMono(TArray<uint8> stereoWavBytes, TArray<uint8>& monoWavBytes);
352
353 //-------- UTILITIES SECTION -------//
354
355 TMap<FString, void(UESSubSystem::*)(TSharedPtr<IHttpRequest, ESPMode::ThreadSafe>, TSharedPtr<IHttpResponse, ESPMode::ThreadSafe>, bool)> PseudoReflectionMap;
356
357protected:
358 /*// Called when the game starts
359 virtual void BeginPlay() override;*/
360
361
362private:
363
364 //-------- HTTP SECTION -------//
365
366 FHttpModule* Http;
367 FString ApiBaseUrl = "https://www.emohuman.com/";
368
369 FString AuthorizationHeader = TEXT("Authorization");
370 void SetAuthorizationHash(FString Hash, TSharedRef<IHttpRequest>& Request);
371
372 TSharedRef<IHttpRequest, ESPMode::ThreadSafe> RequestWithRoute(FString Pathsuffix);
373 static void SetRequestHeaders(TSharedRef<IHttpRequest, ESPMode::ThreadSafe>& Request);
374
375 TSharedRef<IHttpRequest,ESPMode::ThreadSafe> GetRequest(FString Pathsuffix);
376 TSharedRef<IHttpRequest,ESPMode::ThreadSafe> PostRequest(FString Pathsuffix, FString ContentJsonString);
377 TSharedRef<IHttpRequest,ESPMode::ThreadSafe> PutRequest(FString Pathsuffix, FString ContentJsonString);
378 static void Send(TSharedRef<IHttpRequest,ESPMode::ThreadSafe>& Request);
379 FProcHandle ProcessHandle;
380
381 static bool ResponseIsValid(FHttpResponsePtr Response, bool bWasSuccessful);
382
383 //-------- UTILITIES SECTION -------//
384
385 template <typename StructType>
386 void GetJsonStringFromStruct(StructType FilledStruct, FString& StringOutput);
387 template <typename StructType>
388 void GetStructFromJsonString(FHttpResponsePtr Response, StructType& StructOutput);
389
390};
PossibleResponses
PossibleResponses
Definition ESSubSystem.h:49
UMETA
@ UMETA
Definition ESSubSystem.h:50
DECLARE_LOG_CATEGORY_EXTERN
DECLARE_LOG_CATEGORY_EXTERN(EmoShapeCategory, Display, All)
DECLARE_DYNAMIC_MULTICAST_DELEGATE
DECLARE_DYNAMIC_MULTICAST_DELEGATE(FMetaSoulEventSignature)
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams
DECLARE_DYNAMIC_MULTICAST_DELEGATE_FourParams(FMetaSoulEventSignatureAudio, const TArray< uint8 > &, audio_buffer, int64, audio_length, const TArray< FViseme > &, visems, FString, reply)
ESIOClient
Definition ESIOClient.h:8
EmoShapeSubSystem
Definition EmoShapeSubSystem.Build.cs:7
UESJsonObject
Definition ESJsonObject.h:10
UESSubSystem
Definition ESSubSystem.h:60
UESSubSystem::ListenSocket
FSocket * ListenSocket
Definition ESSubSystem.h:74
UESSubSystem::IOClient
ESIOClient * IOClient
Definition ESSubSystem.h:67
FAPIViseme
Definition ESSubSystem.h:33
FAPIViseme::privVisemeId
int32 privVisemeId
Definition ESSubSystem.h:38
FAPIViseme::privAudioOffset
int32 privAudioOffset
Definition ESSubSystem.h:37
FAPIViseme::privAnimation
FString privAnimation
Definition ESSubSystem.h:39
FSecretParam
Definition HttpStruct.h:68
FViseme
Definition ESSubSystem.h:19