Examples

Practical recipes for using ElysGenAI in your game.

Recipe 1: Voice Command System

Build a voice-controlled player controller.

// PlayerController.h
UCLASS()
class AMyPlayerController : public APlayerController
{
    GENERATED_BODY()
    
    UPROPERTY(VisibleAnywhere)
    UERP_STTComponent* STTComponent;
    
    UFUNCTION()
    void OnVoiceCommand(const FERP_STTResult& Result);
};

// PlayerController.cpp
void AMyPlayerController::BeginPlay()
{
    Super::BeginPlay();
    
    if (IsLocalController())
    {
        STTComponent->SetLanguageCode(TEXT("en"));
        STTComponent->OnTranscriptionComplete.AddDynamic(
            this, &AMyPlayerController::OnVoiceCommand);
        STTComponent->StartListening();
    }
}

void AMyPlayerController::OnVoiceCommand(const FERP_STTResult& Result)
{
    if (Result.Confidence < 0.7f) return;
    
    FString Command = Result.TranscribedText.ToLower();
    
    if (Command.Contains(TEXT("attack")))
    {
        Cast<AMyCharacter>(GetPawn())->PerformAttack();
    }
    else if (Command.Contains(TEXT("inventory")))
    {
        OpenInventoryUI();
    }
    else if (Command.Contains(TEXT("map")))
    {
        ToggleMap();
    }
}

---

Recipe 1.5: Voice-to-LLM Pipeline (Blueprint)

Build an AI assistant that listens to your voice and responds using LLM.

Blueprint Event Graph Flow:

[Event BeginPlay]
  ↓
[Add STT Component]
  ↓
[Add LLM Component]
  ↓
[Bind STT OnTranscriptionComplete → OnPlayerSpoke]
  ↓
[Bind LLM OnGenerationComplete → OnLLMResponse]
  ↓
[STT Component → Start Listening]

[Custom Event: OnPlayerSpoke]
  ↓
[STT Result → Get Transcribed Text]
  ↓
[LLM Component → Send Message]

[Custom Event: OnLLMResponse]
  ↓
[LLM Result → Get Generated Text]
  ↓
[Display Text / Play Audio]

C++ Implementation:

// VoiceAssistantComponent.h
UCLASS()
class UVoiceAssistantComponent : public UActorComponent
{
    GENERATED_BODY()

    UPROPERTY(VisibleAnywhere)
    UERP_STTComponent* STTComponent;

    UPROPERTY(VisibleAnywhere)
    UERP_LLMComponent* LLMComponent;

    UFUNCTION()
    void OnPlayerSpoke(const FERP_STTResult& Result);

    UFUNCTION()
    void OnLLMResponse(const FERP_LLMResult& Result);
};

// VoiceAssistantComponent.cpp
void UVoiceAssistantComponent::BeginPlay()
{
    Super::BeginPlay();

    // Create STT component if not assigned
    if (!STTComponent)
    {
        STTComponent = NewObject<UERP_STTComponent>();
        STTComponent->RegisterComponent();
    }

    // Create LLM component if not assigned
    if (!LLMComponent)
    {
        LLMComponent = NewObject<UERP_LLMComponent>();
        LLMComponent->RegisterComponent();
    }

    // Configure LLM with system prompt
    LLMComponent->SetSystemPrompt(TEXT(
        "You are a helpful AI assistant in a video game. "
        "Keep responses concise (1-2 sentences). "
        "Be friendly and natural."));
    LLMComponent->SetTemperature(0.7f);
    LLMComponent->SetMaxTokens(128);

    // Bind events
    STTComponent->OnTranscriptionComplete.AddDynamic(
        this, &UVoiceAssistantComponent::OnPlayerSpoke);
    LLMComponent->OnGenerationComplete.AddDynamic(
        this, &UVoiceAssistantComponent::OnLLMResponse);

    // Start listening
    STTComponent->StartListening();
    UE_LOG(LogTemp, Log, TEXT("Voice Assistant: Listening for speech..."));
}

void UVoiceAssistantComponent::OnPlayerSpoke(const FERP_STTResult& Result)
{
    if (Result.Confidence < 0.6f)
    {
        UE_LOG(LogTemp, Warning, TEXT("Low confidence: %s"), *Result.TranscribedText);
        return;
    }

    UE_LOG(LogTemp, Log, TEXT("You said: %s"), *Result.TranscribedText);

    // Send player's speech to LLM
    LLMComponent->SendMessage(Result.TranscribedText);
}

void UVoiceAssistantComponent::OnLLMResponse(const FERP_LLMResult& Result)
{
    UE_LOG(LogTemp, Log, TEXT("Assistant: %s"), *Result.GeneratedText);

    // Optional: Play text-to-speech audio of response
    // PlayResponseAudio(Result.GeneratedText);

    // Resume listening for next message
    STTComponent->StartListening();
}

Usage:

1. Add UVoiceAssistantComponent to your PlayerController or Character
2. Player speaks into microphone
3. Speech is transcribed to text (STT)
4. Text is sent to LLM for processing
5. LLM generates response
6. Response is logged and can be displayed/spoken

---

Recipe 2: NPC Dialogue

Create an interactive merchant NPC with LLM-powered dialogue.

// MerchantNPC.h
UCLASS()
class AMerchantNPC : public ACharacter
{
    GENERATED_BODY()
    
    UPROPERTY(VisibleAnywhere)
    UERP_LLMComponent* LLMComponent;
    
    UPROPERTY(VisibleAnywhere)
    UWidgetComponent* DialogueWidget;
    
    void InitializePersonality();
    
    UFUNCTION(BlueprintCallable)
    void StartConversation(AActor* Player);
    
    UFUNCTION()
    void OnResponseGenerated(const FERP_LLMResult& Result);
};

// MerchantNPC.cpp
void AMerchantNPC::BeginPlay()
{
    Super::BeginPlay();
    InitializePersonality();
    
    LLMComponent->OnGenerationComplete.AddDynamic(
        this, &AMerchantNPC::OnResponseGenerated);
}

void AMerchantNPC::InitializePersonality()
{
    FString Personality = FString::Printf(
        TEXT("You are %s, a merchant in a medieval fantasy town. "
             "You sell weapons, armor, and supplies. "
             "You're friendly but shrewd about prices. "
             "Keep responses under 50 words."),
        *GetName());
    
    LLMComponent->SetSystemPrompt(Personality);
    LLMComponent->SetTemperature(0.75f);
    LLMComponent->SetMaxTokens(80);
}

void AMerchantNPC::StartConversation(AActor* Player)
{
    // Get player's STT component
    if (UERP_STTComponent* PlayerSTT = 
        Player->FindComponentByClass<UERP_STTComponent>())
    {
        PlayerSTT->OnTranscriptionComplete.AddDynamic(
            this, &AMerchantNPC::OnPlayerSpoke);
    }
    
    // Greeting
    LLMComponent->SendMessage(TEXT("Greet the player warmly."));
}

void AMerchantNPC::OnPlayerSpoke(const FERP_STTResult& Result)
{
    // Player spoke, generate response
    LLMComponent->SendMessage(Result.TranscribedText);
}

void AMerchantNPC::OnResponseGenerated(const FERP_LLMResult& Result)
{
    // Display dialogue bubble
    DialogueWidget->SetText(FText::FromString(Result.GeneratedText));
}

---

Recipe 3: Multiplayer Voice Chat with Subtitles

Voice chat that automatically generates subtitles using STT.

// VoiceChatComponent.h
UCLASS()
class UVoiceChatComponent : public UActorComponent, public IERP_AudioConsumer
{
    GENERATED_BODY()
    
    UPROPERTY(VisibleAnywhere)
    UERP_STTComponent* STTComponent;
    
    // IERP_AudioConsumer
    virtual void OnAudioDataReceived_Implementation(const FERP_AudioBuffer& Buffer) override;
    
    UFUNCTION(Server, Unreliable, WithValidation)
    void ServerStreamAudio(const TArray<uint8>& CompressedAudio);
    
    UFUNCTION(Server, Reliable, WithValidation)
    void ServerSendTranscription(const FString& Text);
    
    UFUNCTION(NetMulticast, Reliable)
    void MulticastDisplaySubtitles(APlayerController* Speaker, const FString& Text);
};

// VoiceChatComponent.cpp
void UVoiceChatComponent::BeginPlay()
{
    Super::BeginPlay();
    
    if (IsLocallyControlled())
    {
        // Register as audio consumer
        UERP_AudioCaptureSubsystem* AudioSubsystem = 
            GetGameInstance()->GetSubsystem<UERP_AudioCaptureSubsystem>();
        AudioSubsystem->RegisterConsumer(this);
        
        // Setup STT for subtitles
        STTComponent->OnTranscriptionComplete.AddDynamic(
            this, &UVoiceChatComponent::OnTranscription);
    }
}

void UVoiceChatComponent::OnAudioDataReceived_Implementation(const FERP_AudioBuffer& Buffer)
{
    // Compress audio (Opus codec)
    TArray<uint8> Compressed = CompressAudioOpus(Buffer.AudioData);
    
    // Send to server (audio only, NOT replicated)
    ServerStreamAudio(Compressed);
}

void UVoiceChatComponent::OnTranscription(const FERP_STTResult& Result)
{
    // Send transcription to server
    ServerSendTranscription(Result.TranscribedText);
}

void UVoiceChatComponent::ServerSendTranscription_Implementation(const FString& Text)
{
    // Broadcast subtitles to all clients
    MulticastDisplaySubtitles(GetOwner<APlayerController>(), Text);
}

void UVoiceChatComponent::MulticastDisplaySubtitles_Implementation(
    APlayerController* Speaker, const FString& Text)
{
    // Display subtitle widget for this player
    ShowSubtitle(Speaker, Text);
}

---

Recipe 4: Push-to-Talk Voice Commands

Voice commands with push-to-talk input binding.

// PlayerController.h
UCLASS()
class AMyPlayerController : public APlayerController
{
    GENERATED_BODY()
    
    UPROPERTY(VisibleAnywhere)
    UERP_STTComponent* STTComponent;
    
    void StartVoiceCapture();
    void StopVoiceCapture();
    
    UFUNCTION()
    void OnCommandReceived(const FERP_STTResult& Result);
};

// PlayerController.cpp
void AMyPlayerController::SetupInputComponent()
{
    Super::SetupInputComponent();
    
    // Bind push-to-talk key
    InputComponent->BindAction("VoiceCommand", IE_Pressed, 
        this, &AMyPlayerController::StartVoiceCapture);
    InputComponent->BindAction("VoiceCommand", IE_Released, 
        this, &AMyPlayerController::StopVoiceCapture);
}

void AMyPlayerController::StartVoiceCapture()
{
    auto* AudioSubsystem = GetGameInstance()->GetSubsystem<UERP_AudioCaptureSubsystem>();
    AudioSubsystem->SetPushToTalkActive(true);
    
    // Visual feedback
    ShowVoiceIndicator(true);
}

void AMyPlayerController::StopVoiceCapture()
{
    auto* AudioSubsystem = GetGameInstance()->GetSubsystem<UERP_AudioCaptureSubsystem>();
    AudioSubsystem->SetPushToTalkActive(false);
    
    ShowVoiceIndicator(false);
}

---

Recipe 5: Custom Audio Consumer

Record audio to file using the consumer pattern.

// AudioRecorderComponent.h
UCLASS()
class UAudioRecorderComponent : public UActorComponent, public IERP_AudioConsumer
{
    GENERATED_BODY()
    
public:
    UFUNCTION(BlueprintCallable)
    void StartRecording(const FString& Filename);
    
    UFUNCTION(BlueprintCallable)
    void StopRecording();
    
    // IERP_AudioConsumer
    virtual void OnAudioDataReceived_Implementation(const FERP_AudioBuffer& Buffer) override;
    virtual FString GetConsumerName_Implementation() const override;
    
private:
    bool bIsRecording = false;
    TArray<float> RecordedAudio;
    FString OutputFilename;
};

// AudioRecorderComponent.cpp
void UAudioRecorderComponent::BeginPlay()
{
    Super::BeginPlay();
    
    // Register as consumer
    auto* AudioSubsystem = GetGameInstance()->GetSubsystem<UERP_AudioCaptureSubsystem>();
    AudioSubsystem->RegisterConsumer(this);
}

void UAudioRecorderComponent::StartRecording(const FString& Filename)
{
    bIsRecording = true;
    OutputFilename = Filename;
    RecordedAudio.Empty();
}

void UAudioRecorderComponent::StopRecording()
{
    bIsRecording = false;
    
    // Save to WAV file
    SaveAudioToWAV(RecordedAudio, OutputFilename);
}

void UAudioRecorderComponent::OnAudioDataReceived_Implementation(const FERP_AudioBuffer& Buffer)
{
    if (bIsRecording)
    {
        RecordedAudio.Append(Buffer.AudioData);
    }
}

FString UAudioRecorderComponent::GetConsumerName_Implementation() const
{
    return TEXT("AudioRecorder");
}

---

Recipe 6: Custom STT Backend

Implement a custom STT backend for alternative models.

// MyCustomSTTBackend.h
UCLASS()
class UMyCustomSTTBackend : public UObject, public IERP_STTBackend
{
    GENERATED_BODY()
    
public:
    // IERP_GenAIService interface
    virtual bool Initialize() override;
    virtual void Shutdown() override;
    virtual bool IsReady() const override;
    
    // IERP_STTBackend interface
    virtual bool LoadModel(const FString& ModelPath) override;
    virtual void UnloadModel() override;
    virtual bool IsModelLoaded() const override;
    virtual void SetLanguage(const FString& LanguageCode) override;
    virtual void SetSampleRate(int32 SampleRate) override;
    virtual FERP_STTResult ProcessAudio(const TArray<float>& AudioData) override;
    virtual void ProcessAudioAsync(const TArray<float>& AudioData, 
                                    const FElysSTTResultDelegate& Callback) override;
    
private:
    bool bIsInitialized = false;
    bool bModelLoaded = false;
    FString CurrentLanguage;
    int32 CurrentSampleRate;
    void* CustomSTTEngine;
};

// MyCustomSTTBackend.cpp
bool UMyCustomSTTBackend::Initialize()
{
    CustomSTTEngine = CreateYourSTTEngine();
    bIsInitialized = (CustomSTTEngine != nullptr);
    return bIsInitialized;
}

bool UMyCustomSTTBackend::LoadModel(const FString& ModelPath)
{
    if (!bIsInitialized) return false;
    
    bModelLoaded = YourSTTEngine_LoadModel(CustomSTTEngine, ModelPath);
    return bModelLoaded;
}

FERP_STTResult UMyCustomSTTBackend::ProcessAudio(const TArray<float>& AudioData)
{
    FERP_STTResult Result;
    
    if (!IsReady())
    {
        Result.TranscribedText = TEXT("");
        Result.Confidence = 0.0f;
        return Result;
    }
    
    FString Transcription = YourSTTEngine_Transcribe(CustomSTTEngine, AudioData);
    float Confidence = YourSTTEngine_GetConfidence(CustomSTTEngine);
    
    Result.TranscribedText = Transcription;
    Result.Confidence = Confidence;
    Result.Language = CurrentLanguage;
    Result.bIsFinal = true;
    
    return Result;
}

Register in Subsystem:

// In UERP_STTSubsystem
TScriptInterface<IERP_STTBackend> UERP_STTSubsystem::CreateBackend()
{
    UMyCustomSTTBackend* CustomBackend = NewObject<UMyCustomSTTBackend>(this);
    
    TScriptInterface<IERP_STTBackend> BackendInterface;
    BackendInterface.SetObject(CustomBackend);
    BackendInterface.SetInterface(Cast<IERP_STTBackend>(CustomBackend));
    
    return BackendInterface;
}

---

Next Steps

• API Reference - Complete class documentation
• Troubleshooting - Common issues and solutions
• Multiplayer Guide - Network architecture best practices