From bd037d6d305b6f5d5f9d5374c3555caa16b8e0d8 Mon Sep 17 00:00:00 2001 From: dev-Develope Date: Fri, 12 Jun 2026 11:31:32 +0530 Subject: [PATCH] added 60db services here --- .../Common/Scripts/ImageEncodingUtil.cs | 58 +++++ .../Common/Scripts/MicrophoneStreamer.cs | 142 ++++++++++++ .../Runtime/Common/Scripts/PcmAudioPlayer.cs | 72 ++++++ .../Runtime/Scripts/SixtyDbChatClient.cs | 109 +++++++++ .../Runtime/Scripts/SixtyDbConfig.cs | 29 +++ .../Runtime/Scripts/SixtyDbConvManager.cs | 218 ++++++++++++++++++ .../Runtime/Scripts/SixtyDbSttSocket.cs | 139 +++++++++++ .../Runtime/Scripts/SixtyDbToolRegistry.cs | 35 +++ .../Runtime/Scripts/SixtyDbTtsSocket.cs | 144 ++++++++++++ .../Runtime/Scripts/SixtyDbWebSocketEvents.cs | 57 +++++ .../Runtime/com.convai.sixtydb.asmdef | 17 ++ com.convai.sixtydb/package.json | 22 ++ 12 files changed, 1042 insertions(+) create mode 100644 com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs create mode 100644 com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs create mode 100644 com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs create mode 100644 com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs create mode 100644 com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef create mode 100644 com.convai.sixtydb/package.json diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs b/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs new file mode 100644 index 0000000..a987ca2 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs @@ -0,0 +1,58 @@ +using System; +using UnityEngine; + +public static class ImageEncodingUtil +{ + /// + /// Captures a frame from a , downsizes it to fit within maxSize while preserving + /// aspect ratio, and returns a data URL (PNG or JPEG) as a string. Must be called on the main thread. + /// + public static string CaptureDataUrlFromWebCam(WebCamTexture cam, int maxSize, bool useJpeg, int jpegQuality) + { + if (cam == null || !cam.isPlaying) return string.Empty; + + var srcW = cam.width; + var srcH = cam.height; + if (srcW <= 0 || srcH <= 0) return string.Empty; + + var scale = 1f; + var maxDim = Mathf.Max(srcW, srcH); + if (maxDim > maxSize) + { + scale = (float)maxSize / maxDim; + } + + var dstW = Mathf.Max(1, Mathf.RoundToInt(srcW * scale)); + var dstH = Mathf.Max(1, Mathf.RoundToInt(srcH * scale)); + + RenderTexture rt = null; + Texture2D tex = null; + try + { + rt = new RenderTexture(dstW, dstH, 0, RenderTextureFormat.ARGB32); + Graphics.Blit(cam, rt); + + var prev = RenderTexture.active; + RenderTexture.active = rt; + tex = new Texture2D(dstW, dstH, TextureFormat.RGBA32, false); + tex.ReadPixels(new Rect(0, 0, dstW, dstH), 0, 0); + tex.Apply(); + RenderTexture.active = prev; + + var bytes = useJpeg ? tex.EncodeToJPG(jpegQuality) : tex.EncodeToPNG(); + var mime = useJpeg ? "image/jpeg" : "image/png"; + return $"data:{mime};base64,{Convert.ToBase64String(bytes)}"; + } + finally + { + if (tex != null) UnityEngine.Object.Destroy(tex); + if (rt != null) + { + rt.Release(); + UnityEngine.Object.Destroy(rt); + } + } + } +} + + diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs new file mode 100644 index 0000000..c880c76 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs @@ -0,0 +1,142 @@ +using System; +using UnityEngine; + +/// +/// Streams microphone audio as Base64-encoded 16-bit mono PCM chunks. +/// Each chunk is 1,024 samples (~64 ms at 16 kHz). +/// +public class MicrophoneStreamer : MonoBehaviour +{ + public Action OnAudioChunk; + + private const int SampleRateOut = 16_000; + private const int ChunkSamplesOut = 1_024; + + private AudioClip _microphoneClip; + private string _micDevice; + private int _micSampleRate; + private int _chunkSamplesIn; + private int _lastSamplePos; + + #region Unity Lifecycle + + private void Start() + { + if (Microphone.devices.Length == 0) + { + Debug.LogError("MicrophoneStreamer: no microphone devices."); + enabled = false; + return; + } + + _micDevice = Microphone.devices[0]; + } + + private void Update() + { + if (!_microphoneClip) return; + + var currentPos = Microphone.GetPosition(_micDevice); + var samplesAvailable = currentPos - _lastSamplePos; + if (samplesAvailable < 0) + samplesAvailable += _microphoneClip.samples; + + if (samplesAvailable < _chunkSamplesIn) return; + + var inBuf = new float[_chunkSamplesIn]; + ReadCircular(_microphoneClip, _lastSamplePos, inBuf); + _lastSamplePos = (_lastSamplePos + _chunkSamplesIn) % _microphoneClip.samples; + var pcm16 = DownsampleAndConvert(inBuf, _micSampleRate, SampleRateOut); + OnAudioChunk?.Invoke(Convert.ToBase64String(pcm16)); + } + + #endregion + + #region Public Methods + + public void StartStreaming() + { + _microphoneClip = Microphone.Start(_micDevice, true, 1, SampleRateOut); + _micSampleRate = _microphoneClip.frequency; + _chunkSamplesIn = Mathf.RoundToInt(ChunkSamplesOut * (float)_micSampleRate / SampleRateOut); + _lastSamplePos = 0; + + Debug.Log($"[MicrophoneStreamer] device={_micDevice}, " + + $"realRate={_micSampleRate} Hz, chunkIn={_chunkSamplesIn} samples"); + } + + public void StopStreaming() + { + if (Microphone.IsRecording(_micDevice)) + Microphone.End(_micDevice); + + _microphoneClip = null; + } + + #endregion + + #region Helpers + + private static void ReadCircular(AudioClip clip, int start, float[] buffer) + { + var len = buffer.Length; + var clipSamples = clip.samples; + var tail = clipSamples - start; + + if (len <= tail) + { + clip.GetData(buffer, start); + } + else + { + var tempTail = new float[tail]; + var tempHead = new float[len - tail]; + + clip.GetData(tempTail, start); + clip.GetData(tempHead, 0); + + Array.Copy(tempTail, 0, buffer, 0, tail); + Array.Copy(tempHead, 0, buffer, tail, tempHead.Length); + } + } + + private static byte[] DownsampleAndConvert(float[] inBuf, int inRate, int outRate) + { + if (inRate == outRate) + return ConvertToPcm16(inBuf); + + var ratio = (float)inRate / outRate; + var outLen = Mathf.RoundToInt(inBuf.Length / ratio); + var pcmOut = new byte[outLen * 2]; + + var pos = 0f; + for (var o = 0; o < outLen; o++, pos += ratio) + { + var i0 = Mathf.Clamp((int)pos, 0, inBuf.Length - 1); + var i1 = Mathf.Min(i0 + 1, inBuf.Length - 1); + var frac = pos - i0; + + var sample = Mathf.Lerp(inBuf[i0], inBuf[i1], frac); + var s16 = (short)Mathf.Clamp(sample * 32767f, short.MinValue, short.MaxValue); + + pcmOut[o * 2] = (byte)(s16 & 0xFF); + pcmOut[o * 2 + 1] = (byte)((s16 >> 8) & 0xFF); + } + + return pcmOut; + } + + private static byte[] ConvertToPcm16(float[] buf) + { + var pcm = new byte[buf.Length * 2]; + for (var i = 0; i < buf.Length; i++) + { + var s = (short)Mathf.Clamp(buf[i] * 32767f, short.MinValue, short.MaxValue); + pcm[i * 2] = (byte)(s & 0xFF); + pcm[i * 2 + 1] = (byte)((s >> 8) & 0xFF); + } + return pcm; + } + + #endregion +} diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs new file mode 100644 index 0000000..fe485b6 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using UnityEngine; +using UnityEngine.Assertions; + +/// +/// Handles playback of PCM audio received as Base64-encoded strings. +/// Uses an internal queue to play audio clips sequentially through an AudioSource. +/// +public class PcmAudioPlayer : MonoBehaviour +{ + [SerializeField] private AudioSource audioSource; + [SerializeField] private int sampleRate = 16000; + + private readonly Queue _clipQueue = new(); + + #region Unity Lifecycle + + private void Start() + { + Assert.IsNotNull(audioSource, "Audio source is required"); + } + + private void Update() + { + if (audioSource.isPlaying || _clipQueue.Count == 0) + return; + + audioSource.clip = _clipQueue.Dequeue(); + audioSource.Play(); + } + + #endregion + + #region Public Methods + + /// + /// Converts Base64-encoded PCM audio into a Unity AudioClip + /// and adds it to the playback queue. + /// + /// Base64-encoded PCM16 audio data. + public void EnqueueBase64Audio(string base64Audio) + { + // Decode Base64 string into raw bytes + var bytes = System.Convert.FromBase64String(base64Audio); + + // Each PCM16 sample uses 2 bytes + var sampleCount = bytes.Length / 2; + var samples = new float[sampleCount]; + + for (var i = 0; i < sampleCount; i++) + { + var sample = (short)((bytes[i * 2 + 1] << 8) | bytes[i * 2]); + samples[i] = sample / 32768f; + } + + var clip = AudioClip.Create("AIConversationalClip", sampleCount, 1, sampleRate, false); + clip.SetData(samples, 0); + + _clipQueue.Enqueue(clip); + } + + /// + /// Stops playback immediately and clears any queued audio. + /// + public void StopImmediately() + { + _clipQueue.Clear(); + audioSource.Stop(); + } + + #endregion +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs new file mode 100644 index 0000000..1ba99c0 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using UnityEngine; +using UnityEngine.Networking; + +namespace SixtyDb +{ + // REST client for POST /v1/chat/completions. OpenAI-compatible payload, + // so the tool array is passed through verbatim. + // + // Returns a small (content, toolCalls) tuple; SixtyDbConvManager owns the + // tool-execution loop (dispatch → tool message → re-call until content). + public sealed class SixtyDbChatClient + { + private readonly SixtyDbConfig _config; + + public SixtyDbChatClient(SixtyDbConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + public struct Result + { + public string Content; + public List ToolCalls; + } + + public struct ToolCall + { + public string Id; + public string Name; + public string ArgumentsJson; + } + + public async Task CompleteAsync( + List messages, + JArray tools, + string model = null, + float temperature = 0.7f, + int topK = 20) + { + if (string.IsNullOrWhiteSpace(_config.apiKey)) + throw new InvalidOperationException("60db apiKey is not set in the config asset."); + + var body = new JObject + { + ["model"] = model ?? _config.llmModel, + ["messages"] = new JArray(messages), + ["stream"] = false, + ["temperature"] = temperature, + ["top_k"] = topK, + ["chat_template_kwargs"] = new JObject { ["enable_thinking"] = false }, + }; + if (tools != null && tools.Count > 0) body["tool"] = tools; + + var url = $"{_config.apiBase.TrimEnd('/')}/v1/chat/completions"; + using var req = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPOST); + var raw = Encoding.UTF8.GetBytes(body.ToString(Formatting.None)); + req.uploadHandler = new UploadHandlerRaw(raw) { contentType = "application/json" }; + req.downloadHandler = new DownloadHandlerBuffer(); + req.SetRequestHeader("Authorization", $"Bearer {_config.apiKey}"); + req.SetRequestHeader("Accept", "application/json"); + + var op = req.SendWebRequest(); + while (!op.isDone) await Task.Yield(); + + if (req.result != UnityWebRequest.Result.Success) + throw new InvalidOperationException( + $"[SixtyDbChatClient] HTTP {(int)req.responseCode}: {req.error} / {req.downloadHandler?.text}"); + + var json = JObject.Parse(req.downloadHandler.text); + var msg = json["choices"]?[0]?["message"] as JObject ?? new JObject(); + + var result = new Result + { + Content = msg["content"]?.ToString() ?? string.Empty, + ToolCalls = new List(), + }; + + if (msg["tool_calls"] is JArray toolCalls) + { + foreach (var tc in toolCalls) + { + if (tc is not JObject obj) continue; + var fn = obj["function"] as JObject; + result.ToolCalls.Add(new ToolCall + { + Id = obj["id"]?.ToString(), + Name = fn?["name"]?.ToString(), + ArgumentsJson = fn?["arguments"]?.ToString() ?? "{}", + }); + } + } + + return result; + } + + // Convenience: build a {role, content} message in the shape /v1/chat/completions expects. + public static JObject UserMessage(string text) => new() { ["role"] = "user", ["content"] = text }; + public static JObject SystemMessage(string text) => new() { ["role"] = "system", ["content"] = text }; + public static JObject AssistantMessage(string text) => new() { ["role"] = "assistant", ["content"] = text }; + public static JObject ToolMessage(string toolCallId, string name, string resultJson) => + new() { ["role"] = "tool", ["tool_call_id"] = toolCallId, ["name"] = name, ["content"] = resultJson }; + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs new file mode 100644 index 0000000..6405c25 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs @@ -0,0 +1,29 @@ +using UnityEngine; + +namespace SixtyDb +{ + // ScriptableObject holding 60db credentials and endpoint overrides. + // Mirrors the role of ElevenLabsConfig / OpenAIConfig in the sibling packages. + [CreateAssetMenu(menuName = "SixtyDb/Config", fileName = "SixtyDbConfig")] + public class SixtyDbConfig : ScriptableObject + { + [Header("Authentication")] + public string apiKey; + + [Header("Endpoints (override only if you're on staging / self-hosted)")] + public string apiBase = "https://api.60db.ai"; + public string sttWebsocketUrl = "wss://api.60db.ai/ws/stt"; + public string ttsWebsocketUrl = "wss://api.60db.ai/ws/tts"; + + [Header("Model defaults")] + public string llmModel = "60db-tiny"; + public string defaultVoiceId = "fbb75ed2-975a-40c7-9e06-38e30524a9a1"; + + [Header("Audio")] + [Tooltip("Sample rate sent to /ws/stt. 60db browser mode expects 16 kHz linear PCM.")] + public int sttSampleRate = 16_000; + + [Tooltip("Sample rate requested from /ws/tts. 60db returns LINEAR16 at this rate.")] + public int ttsSampleRate = 24_000; + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs new file mode 100644 index 0000000..20371d7 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs @@ -0,0 +1,218 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using UnityEngine; +using UnityEngine.Events; + +namespace SixtyDb +{ + // 60db conversation orchestrator. Mirrors the public surface of + // ElevenLabs.AgentConversationManager and OpenAI.RealtimeConversationManager + // so existing scene wiring (UnityEvents, PcmAudioPlayer, MicrophoneStreamer) + // ports straight across. + // + // Per-turn loop: + // user mic chunk -> SttSocket + // final transcript -> SixtyDbChatClient (with tool registry) + // |- tool_calls present -> dispatch via SixtyDbToolRegistry -> re-call + // reply.content -> TtsSocket -> audio_chunk -> PcmAudioPlayer + // + // Barge-in: when STT emits speech_started while a TTS context is active, + // close that context immediately + clear the audio player queue. + public class SixtyDbConvManager : MonoBehaviour + { + [Header("Voice")] + [Tooltip("60db voice id. Leave blank to use SixtyDbConfig.defaultVoiceId.")] + [SerializeField] private string voiceId = ""; + + [Header("System prompt")] + [TextArea(2, 8)] + [SerializeField] private string systemPrompt = + "You are a helpful VR assistant. Keep replies under two short sentences " + + "unless the user explicitly asks for depth. Sound human."; + + [Header("Startup")] + [SerializeField] private bool startOnAwake = true; + + [Header("Behaviour")] + [Tooltip("When true, the user speaking interrupts the agent (closes TTS context, clears playback).")] + [SerializeField] private bool bargeInEnabled = true; + [Tooltip("Max tool-call round-trips per turn before falling back to plain content.")] + [SerializeField] private int maxToolRounds = 2; + + [Header("Dependencies")] + [SerializeField] private SixtyDbConfig config; + [SerializeField] private MicrophoneStreamer micStreamer; + [SerializeField] private PcmAudioPlayer audioPlayer; + + [Header("Events")] + public UnityEvent onAgentTranscript; + public UnityEvent onUserTranscript; + public UnityEvent onUserPartialTranscript; + public UnityEvent onAgentSpeaking; + + private SixtyDbSttSocket _stt; + private SixtyDbTtsSocket _tts; + private SixtyDbChatClient _chat; + private readonly List _history = new(); + private bool _busy; + + private void Start() + { + if (startOnAwake) StartAgent(); + } + + private void Update() + { + _stt?.DispatchMessageQueue(); + _tts?.DispatchMessageQueue(); + } + + private async void OnDisable() => await Shutdown(); + private async void OnApplicationQuit() => await Shutdown(); + + // Same external surface as the OpenAI / ElevenLabs managers. + public async void StartAgent() + { + try + { + if (!config) throw new InvalidOperationException("SixtyDbConfig is not assigned."); + if (audioPlayer == null) throw new InvalidOperationException("PcmAudioPlayer is not assigned."); + if (micStreamer == null) throw new InvalidOperationException("MicrophoneStreamer is not assigned."); + + _history.Clear(); + _history.Add(SixtyDbChatClient.SystemMessage(systemPrompt)); + + _chat = new SixtyDbChatClient(config); + _stt = new SixtyDbSttSocket(config); + _tts = new SixtyDbTtsSocket(config, voiceId); + + WireSttEvents(); + WireTtsEvents(); + micStreamer.OnAudioChunk += OnMicChunk; + + await Task.WhenAll(_stt.Connect(), _tts.Connect()); + micStreamer.StartStreaming(); + } + catch (Exception ex) + { + Debug.LogError($"[SixtyDb] Startup failed: {ex}"); + enabled = false; + } + } + + public async void InterruptNow() + { + _tts?.Interrupt(); + audioPlayer.StopImmediately(); + onAgentSpeaking?.Invoke(false); + await Task.Yield(); + } + + private async Task Shutdown() + { + try + { + micStreamer.OnAudioChunk -= OnMicChunk; + micStreamer?.StopStreaming(); + audioPlayer?.StopImmediately(); + if (_stt != null) await _stt.Close(); + if (_tts != null) await _tts.Close(); + } + catch (Exception ex) + { + Debug.LogError($"[SixtyDb] Shutdown error: {ex}"); + } + } + + private void OnMicChunk(string base64Pcm) => _stt?.SendAudioChunk(base64Pcm); + + private void WireSttEvents() + { + _stt.OnSpeechStarted += () => + { + if (!bargeInEnabled) return; + if (_tts == null || !_tts.ContextActive) return; + _tts.Interrupt(); + audioPlayer.StopImmediately(); + onAgentSpeaking?.Invoke(false); + }; + + _stt.OnPartialTranscript += text => onUserPartialTranscript?.Invoke(text); + + _stt.OnFinalTranscript += async text => + { + if (_busy || string.IsNullOrWhiteSpace(text)) return; + onUserTranscript?.Invoke(text); + _history.Add(SixtyDbChatClient.UserMessage(text)); + + _busy = true; + try { await RunTurn(); } + catch (Exception ex) { Debug.LogError($"[SixtyDb] Turn failed: {ex}"); } + finally { _busy = false; } + }; + + _stt.OnError += msg => Debug.LogError($"[SixtyDb] STT error: {msg}"); + } + + private void WireTtsEvents() + { + _tts.OnAudioChunk += audioPlayer.EnqueueBase64Audio; + _tts.OnSpeakStart += () => onAgentSpeaking?.Invoke(true); + _tts.OnSpeakEnd += () => onAgentSpeaking?.Invoke(false); + _tts.OnError += msg => Debug.LogError($"[SixtyDb] TTS error: {msg}"); + } + + private async Task RunTurn() + { + var tools = SixtyDbToolRegistry.HasTools ? SixtyDbToolRegistry.GetToolsSpec() : null; + + for (var round = 0; round < maxToolRounds; round++) + { + var reply = await _chat.CompleteAsync(_history, tools); + + if (reply.ToolCalls.Count == 0) + { + var content = reply.Content ?? string.Empty; + if (string.IsNullOrWhiteSpace(content)) return; + _history.Add(SixtyDbChatClient.AssistantMessage(content)); + onAgentTranscript?.Invoke(content); + await _tts.Speak(content); + return; + } + + // Append the assistant's tool-call turn, then run each tool locally + // and append the result so the next round can incorporate it. + _history.Add(SixtyDbChatClient.AssistantMessage(reply.Content ?? string.Empty)); + foreach (var call in reply.ToolCalls) + { + var argsJson = call.ArgumentsJson; + JObject args; + try { args = JObject.Parse(string.IsNullOrEmpty(argsJson) ? "{}" : argsJson); } + catch { args = new JObject(); } + + string resultJson; + try + { + if (SixtyDbToolRegistry.TryGetHandler(call.Name, out var handler)) + { + var res = await handler(args); + resultJson = (res ?? new JObject()).ToString(Newtonsoft.Json.Formatting.None); + } + else + { + resultJson = $"{{\"error\":\"unknown tool '{call.Name}'\"}}"; + } + } + catch (Exception ex) + { + resultJson = $"{{\"error\":\"{ex.Message}\"}}"; + } + _history.Add(SixtyDbChatClient.ToolMessage(call.Id, call.Name, resultJson)); + } + } + Debug.LogWarning("[SixtyDb] Tool round budget exhausted; aborting turn."); + } + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs new file mode 100644 index 0000000..e868d92 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using NativeWebSocket; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace SixtyDb +{ + // Wraps wss://api.60db.ai/ws/stt (browser mode, linear PCM 16k). + // Lifecycle: + // connect() + // OnOpen -> server emits connection_established + // -> send 'start' -> server emits 'connected' -> OnReady + // per mic chunk: send { type: 'audio', audio: , encoding: 'linear', sample_rate: 16000 } + // server emits: speech_started, transcription (is_final + speech_final), session_stopped + public sealed class SixtyDbSttSocket + { + public event Action OnReady; + public event Action OnSpeechStarted; + public event Action OnPartialTranscript; + public event Action OnFinalTranscript; + public event Action OnError; + public event Action OnClose; + + private readonly SixtyDbConfig _config; + private WebSocket _ws; + private bool _ready; + + public WebSocket Underlying => _ws; + public bool IsReady => _ready; + + public SixtyDbSttSocket(SixtyDbConfig config) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + } + + public async Task Connect() + { + var url = $"{_config.sttWebsocketUrl.TrimEnd('/')}?apiKey={Uri.EscapeDataString(_config.apiKey)}"; + _ws = new WebSocket(url); + _ws.OnOpen += HandleOpen; + _ws.OnMessage += HandleMessage; + _ws.OnError += msg => OnError?.Invoke(msg); + _ws.OnClose += _ => { _ready = false; OnClose?.Invoke(); }; + await _ws.Connect(); + } + + public void DispatchMessageQueue() => _ws?.DispatchMessageQueue(); + + public void SendAudioChunk(string base64Pcm) + { + if (_ws == null || _ws.State != WebSocketState.Open || !_ready) return; + var payload = new Dictionary + { + { "type", "audio" }, + { "audio", base64Pcm }, + { "encoding", "linear" }, + { "sample_rate", _config.sttSampleRate }, + }; + _ = _ws.SendText(JsonConvert.SerializeObject(payload)); + } + + public async Task Close() + { + try + { + if (_ws != null && _ws.State == WebSocketState.Open) + { + // Best-effort 'stop' for billing summary; ignore failures. + await _ws.SendText("{\"type\":\"stop\"}"); + await _ws.Close(); + } + } + catch { /* socket already gone */ } + } + + private void HandleOpen() + { + // Wait for the server's 'connection_established' before sending 'start'. + // That hand-off is done inside HandleMessage to keep the state machine local. + } + + private void HandleMessage(byte[] bytes) + { + var text = Encoding.UTF8.GetString(bytes); + JObject msg; + try { msg = JObject.Parse(text); } + catch { return; } + + switch (msg["type"]?.ToString()) + { + case "connecting": + return; + case "connection_established": + SendStart(); + return; + case "connected": + _ready = true; + OnReady?.Invoke(); + return; + case "speech_started": + OnSpeechStarted?.Invoke(); + return; + case "transcription": + { + var transcript = msg["transcript"]?.ToString() ?? msg["text"]?.ToString(); + if (string.IsNullOrEmpty(transcript)) return; + var isFinal = (bool?)msg["is_final"] ?? false; + var speechFinal = (bool?)msg["speech_final"] ?? false; + if (isFinal && speechFinal) OnFinalTranscript?.Invoke(transcript); + else OnPartialTranscript?.Invoke(transcript); + return; + } + case "session_stopped": + _ready = false; + return; + case "error": + OnError?.Invoke(msg["message"]?.ToString() ?? "unknown STT error"); + return; + } + } + + private void SendStart() + { + var payload = new Dictionary + { + { "type", "start" }, + { "encoding", "linear" }, + { "sample_rate", _config.sttSampleRate }, + { "utterance_end_ms", 500 }, + { "continuous_mode", true }, + }; + _ = _ws.SendText(JsonConvert.SerializeObject(payload)); + } + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs new file mode 100644 index 0000000..8ed8909 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace SixtyDb +{ + // Generic registry for 60db agent tools. Mirrors com.convai.openai's + // AgentToolRegistry so existing scene wiring patterns port directly. + // + // Tool specs use the OpenAI-compatible JSON shape — 60db's + // /v1/chat/completions accepts the same `tool` array. + public static class SixtyDbToolRegistry + { + public delegate Task ToolHandler(JObject args); + + private static readonly Dictionary NameToHandler = + new(StringComparer.OrdinalIgnoreCase); + private static readonly List ToolSpecs = new(); + + public static void Register(string name, ToolHandler handler, JObject toolSpec = null) + { + if (string.IsNullOrWhiteSpace(name) || handler == null) return; + NameToHandler[name] = handler; + if (toolSpec != null) ToolSpecs.Add(toolSpec); + } + + public static bool TryGetHandler(string name, out ToolHandler handler) => + NameToHandler.TryGetValue(name, out handler); + + public static JArray GetToolsSpec() => new JArray(ToolSpecs); + + public static bool HasTools => ToolSpecs.Count > 0; + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs new file mode 100644 index 0000000..7a20786 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs @@ -0,0 +1,144 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using NativeWebSocket; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using UnityEngine; + +namespace SixtyDb +{ + // Wraps wss://api.60db.ai/ws/tts. Lifecycle: + // Connect() + // OnOpen -> server emits connection_established -> OnReady + // Speak(text): + // CreateContext if needed -> context_created + // send_text + flush_context -> audio_chunk frames -> flush_completed + // Interrupt(): close_context (drops the in-flight reply, used for barge-in). + public sealed class SixtyDbTtsSocket + { + public event Action OnReady; + public event Action OnSpeakStart; + public event Action OnSpeakEnd; + public event Action OnAudioChunk; + public event Action OnError; + public event Action OnClose; + + private readonly SixtyDbConfig _config; + private readonly string _voiceId; + private WebSocket _ws; + private string _contextId; + private TaskCompletionSource _contextReady; + private int _chunksSinceFlush; + + public WebSocket Underlying => _ws; + public bool ContextActive => !string.IsNullOrEmpty(_contextId); + + public SixtyDbTtsSocket(SixtyDbConfig config, string voiceId) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _voiceId = string.IsNullOrWhiteSpace(voiceId) ? config.defaultVoiceId : voiceId; + } + + public async Task Connect() + { + var url = $"{_config.ttsWebsocketUrl.TrimEnd('/')}?apiKey={Uri.EscapeDataString(_config.apiKey)}"; + _ws = new WebSocket(url); + _ws.OnMessage += HandleMessage; + _ws.OnError += msg => OnError?.Invoke(msg); + _ws.OnClose += _ => { _contextId = null; OnClose?.Invoke(); }; + await _ws.Connect(); + } + + public void DispatchMessageQueue() => _ws?.DispatchMessageQueue(); + + public async Task Speak(string text) + { + if (_ws == null || _ws.State != WebSocketState.Open) return; + if (string.IsNullOrEmpty(text)) return; + + if (string.IsNullOrEmpty(_contextId)) + await EnsureContext(); + + _chunksSinceFlush = 0; + await _ws.SendText(JsonConvert.SerializeObject(new Dictionary + { + { "type", "send_text" }, { "text", text }, { "context_id", _contextId }, + })); + await _ws.SendText(JsonConvert.SerializeObject(new Dictionary + { + { "type", "flush_context" }, { "context_id", _contextId }, + })); + } + + // Cancel the current reply mid-speech. Used by SixtyDbConvManager for + // barge-in when the STT WS reports speech_started. + public void Interrupt() + { + if (_ws == null || _ws.State != WebSocketState.Open || string.IsNullOrEmpty(_contextId)) + return; + var json = JsonConvert.SerializeObject(new Dictionary + { + { "type", "close_context" }, { "context_id", _contextId }, + }); + _ = _ws.SendText(json); + _contextId = null; + } + + public async Task Close() + { + Interrupt(); + try { if (_ws != null && _ws.State == WebSocketState.Open) await _ws.Close(); } + catch { /* socket already gone */ } + } + + private Task EnsureContext() + { + _contextReady = new TaskCompletionSource(); + _ = _ws.SendText(JsonConvert.SerializeObject(new Dictionary + { + { "type", "create_context" }, + { "voice_id", _voiceId }, + { "audio_encoding", "LINEAR16" }, + { "sample_rate_hertz", _config.ttsSampleRate }, + })); + return _contextReady.Task; + } + + private void HandleMessage(byte[] bytes) + { + var text = Encoding.UTF8.GetString(bytes); + JObject msg; + try { msg = JObject.Parse(text); } + catch { return; } + + switch (msg["type"]?.ToString()) + { + case "connection_established": + OnReady?.Invoke(); + return; + case "context_created": + _contextId = msg["context_id"]?.ToString(); + _contextReady?.TrySetResult(true); + return; + case "audio_chunk": + { + var b64 = msg["audioContent"]?.ToString() ?? msg["audio"]?.ToString(); + if (string.IsNullOrEmpty(b64)) return; + if (_chunksSinceFlush == 0) OnSpeakStart?.Invoke(); + _chunksSinceFlush++; + OnAudioChunk?.Invoke(b64); + return; + } + case "flush_completed": + OnSpeakEnd?.Invoke(); + _chunksSinceFlush = 0; + return; + case "error": + OnError?.Invoke(msg["message"]?.ToString() ?? "unknown TTS error"); + return; + } + } + } +} diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs new file mode 100644 index 0000000..f8a33c2 --- /dev/null +++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs @@ -0,0 +1,57 @@ +using Newtonsoft.Json; + +namespace SixtyDb +{ + // POCOs mapped to the JSON envelope used by 60db's /ws/stt and /ws/tts + // streams. Mirrors the role of ElevenLabsWebSocketEvents / OpenAIWebSocketEvents. + // Field names lifted verbatim from https://docs.60db.ai/websocket-api/{stt,tts}. + + public class SixtyDbBaseEvent + { + [JsonProperty("type")] public string Type { get; set; } + } + + // ---- STT ---------------------------------------------------------------- + + public class SttConnectedEvent : SixtyDbBaseEvent {} + + public class SttSpeechStartedEvent : SixtyDbBaseEvent {} + + public class SttTranscriptionEvent : SixtyDbBaseEvent + { + [JsonProperty("transcript")] public string Transcript { get; set; } + [JsonProperty("text")] public string Text { get; set; } + [JsonProperty("is_final")] public bool IsFinal { get; set; } + [JsonProperty("speech_final")] public bool SpeechFinal { get; set; } + } + + public class SttSessionStoppedEvent : SixtyDbBaseEvent {} + + public class SttErrorEvent : SixtyDbBaseEvent + { + [JsonProperty("message")] public string Message { get; set; } + } + + // ---- TTS ---------------------------------------------------------------- + + public class TtsContextCreatedEvent : SixtyDbBaseEvent + { + [JsonProperty("context_id")] public string ContextId { get; set; } + } + + public class TtsAudioChunkEvent : SixtyDbBaseEvent + { + [JsonProperty("audioContent")] public string AudioBase64 { get; set; } + [JsonProperty("audio")] public string AudioFallback { get; set; } + + public string ResolveAudio() => + !string.IsNullOrEmpty(AudioBase64) ? AudioBase64 : AudioFallback; + } + + public class TtsFlushCompletedEvent : SixtyDbBaseEvent {} + + public class TtsErrorEvent : SixtyDbBaseEvent + { + [JsonProperty("message")] public string Message { get; set; } + } +} diff --git a/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef b/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef new file mode 100644 index 0000000..d81b493 --- /dev/null +++ b/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef @@ -0,0 +1,17 @@ +{ + "name": "com.convai.sixtydb", + "rootNamespace": "SixtyDb", + "references": [ + "endel.nativewebsocket", + "Unity.Nuget.Newtonsoft-Json" + ], + "includePlatforms": [], + "excludePlatforms": [], + "allowUnsafeCode": false, + "overrideReferences": false, + "precompiledReferences": [], + "autoReferenced": true, + "defineConstraints": [], + "versionDefines": [], + "noEngineReferences": false +} diff --git a/com.convai.sixtydb/package.json b/com.convai.sixtydb/package.json new file mode 100644 index 0000000..dec36b9 --- /dev/null +++ b/com.convai.sixtydb/package.json @@ -0,0 +1,22 @@ +{ + "name": "com.convai.sixtydb", + "displayName": "ConversationalAI [60db]", + "version": "1.0.0", + "unity": "6000.0", + "description": "A Unity package that enables real-time conversational AI interactions by composing 60db's STT WebSocket, LLM chat completions, and TTS WebSocket into a single Quest-friendly conversation manager. Mirrors the surface of com.convai.elevenlabs and com.convai.openai.", + "dependencies": { + "com.unity.nuget.newtonsoft-json": "3.0.0" + }, + "author": { + "name": "Daniel Oquelis", + "email": "daniel.oquelis@gmail.com", + "url": "https://github.com/danieloquelis" + }, + "samples": [ + { + "displayName": "Conversational AI Samples", + "description": "Please clone the main repository to access to samples", + "path": "Samples~/ConversationalSample" + } + ] +}