diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs b/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs
new file mode 100644
index 0000000..a987ca2
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Common/Scripts/ImageEncodingUtil.cs
@@ -0,0 +1,58 @@
+using System;
+using UnityEngine;
+
+public static class ImageEncodingUtil
+{
+ ///
+ /// Captures a frame from a , downsizes it to fit within maxSize while preserving
+ /// aspect ratio, and returns a data URL (PNG or JPEG) as a string. Must be called on the main thread.
+ ///
+ public static string CaptureDataUrlFromWebCam(WebCamTexture cam, int maxSize, bool useJpeg, int jpegQuality)
+ {
+ if (cam == null || !cam.isPlaying) return string.Empty;
+
+ var srcW = cam.width;
+ var srcH = cam.height;
+ if (srcW <= 0 || srcH <= 0) return string.Empty;
+
+ var scale = 1f;
+ var maxDim = Mathf.Max(srcW, srcH);
+ if (maxDim > maxSize)
+ {
+ scale = (float)maxSize / maxDim;
+ }
+
+ var dstW = Mathf.Max(1, Mathf.RoundToInt(srcW * scale));
+ var dstH = Mathf.Max(1, Mathf.RoundToInt(srcH * scale));
+
+ RenderTexture rt = null;
+ Texture2D tex = null;
+ try
+ {
+ rt = new RenderTexture(dstW, dstH, 0, RenderTextureFormat.ARGB32);
+ Graphics.Blit(cam, rt);
+
+ var prev = RenderTexture.active;
+ RenderTexture.active = rt;
+ tex = new Texture2D(dstW, dstH, TextureFormat.RGBA32, false);
+ tex.ReadPixels(new Rect(0, 0, dstW, dstH), 0, 0);
+ tex.Apply();
+ RenderTexture.active = prev;
+
+ var bytes = useJpeg ? tex.EncodeToJPG(jpegQuality) : tex.EncodeToPNG();
+ var mime = useJpeg ? "image/jpeg" : "image/png";
+ return $"data:{mime};base64,{Convert.ToBase64String(bytes)}";
+ }
+ finally
+ {
+ if (tex != null) UnityEngine.Object.Destroy(tex);
+ if (rt != null)
+ {
+ rt.Release();
+ UnityEngine.Object.Destroy(rt);
+ }
+ }
+ }
+}
+
+
diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs
new file mode 100644
index 0000000..c880c76
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Common/Scripts/MicrophoneStreamer.cs
@@ -0,0 +1,142 @@
+using System;
+using UnityEngine;
+
+///
+/// Streams microphone audio as Base64-encoded 16-bit mono PCM chunks.
+/// Each chunk is 1,024 samples (~64 ms at 16 kHz).
+///
+public class MicrophoneStreamer : MonoBehaviour
+{
+ public Action OnAudioChunk;
+
+ private const int SampleRateOut = 16_000;
+ private const int ChunkSamplesOut = 1_024;
+
+ private AudioClip _microphoneClip;
+ private string _micDevice;
+ private int _micSampleRate;
+ private int _chunkSamplesIn;
+ private int _lastSamplePos;
+
+ #region Unity Lifecycle
+
+ private void Start()
+ {
+ if (Microphone.devices.Length == 0)
+ {
+ Debug.LogError("MicrophoneStreamer: no microphone devices.");
+ enabled = false;
+ return;
+ }
+
+ _micDevice = Microphone.devices[0];
+ }
+
+ private void Update()
+ {
+ if (!_microphoneClip) return;
+
+ var currentPos = Microphone.GetPosition(_micDevice);
+ var samplesAvailable = currentPos - _lastSamplePos;
+ if (samplesAvailable < 0)
+ samplesAvailable += _microphoneClip.samples;
+
+ if (samplesAvailable < _chunkSamplesIn) return;
+
+ var inBuf = new float[_chunkSamplesIn];
+ ReadCircular(_microphoneClip, _lastSamplePos, inBuf);
+ _lastSamplePos = (_lastSamplePos + _chunkSamplesIn) % _microphoneClip.samples;
+ var pcm16 = DownsampleAndConvert(inBuf, _micSampleRate, SampleRateOut);
+ OnAudioChunk?.Invoke(Convert.ToBase64String(pcm16));
+ }
+
+ #endregion
+
+ #region Public Methods
+
+ public void StartStreaming()
+ {
+ _microphoneClip = Microphone.Start(_micDevice, true, 1, SampleRateOut);
+ _micSampleRate = _microphoneClip.frequency;
+ _chunkSamplesIn = Mathf.RoundToInt(ChunkSamplesOut * (float)_micSampleRate / SampleRateOut);
+ _lastSamplePos = 0;
+
+ Debug.Log($"[MicrophoneStreamer] device={_micDevice}, " +
+ $"realRate={_micSampleRate} Hz, chunkIn={_chunkSamplesIn} samples");
+ }
+
+ public void StopStreaming()
+ {
+ if (Microphone.IsRecording(_micDevice))
+ Microphone.End(_micDevice);
+
+ _microphoneClip = null;
+ }
+
+ #endregion
+
+ #region Helpers
+
+ private static void ReadCircular(AudioClip clip, int start, float[] buffer)
+ {
+ var len = buffer.Length;
+ var clipSamples = clip.samples;
+ var tail = clipSamples - start;
+
+ if (len <= tail)
+ {
+ clip.GetData(buffer, start);
+ }
+ else
+ {
+ var tempTail = new float[tail];
+ var tempHead = new float[len - tail];
+
+ clip.GetData(tempTail, start);
+ clip.GetData(tempHead, 0);
+
+ Array.Copy(tempTail, 0, buffer, 0, tail);
+ Array.Copy(tempHead, 0, buffer, tail, tempHead.Length);
+ }
+ }
+
+ private static byte[] DownsampleAndConvert(float[] inBuf, int inRate, int outRate)
+ {
+ if (inRate == outRate)
+ return ConvertToPcm16(inBuf);
+
+ var ratio = (float)inRate / outRate;
+ var outLen = Mathf.RoundToInt(inBuf.Length / ratio);
+ var pcmOut = new byte[outLen * 2];
+
+ var pos = 0f;
+ for (var o = 0; o < outLen; o++, pos += ratio)
+ {
+ var i0 = Mathf.Clamp((int)pos, 0, inBuf.Length - 1);
+ var i1 = Mathf.Min(i0 + 1, inBuf.Length - 1);
+ var frac = pos - i0;
+
+ var sample = Mathf.Lerp(inBuf[i0], inBuf[i1], frac);
+ var s16 = (short)Mathf.Clamp(sample * 32767f, short.MinValue, short.MaxValue);
+
+ pcmOut[o * 2] = (byte)(s16 & 0xFF);
+ pcmOut[o * 2 + 1] = (byte)((s16 >> 8) & 0xFF);
+ }
+
+ return pcmOut;
+ }
+
+ private static byte[] ConvertToPcm16(float[] buf)
+ {
+ var pcm = new byte[buf.Length * 2];
+ for (var i = 0; i < buf.Length; i++)
+ {
+ var s = (short)Mathf.Clamp(buf[i] * 32767f, short.MinValue, short.MaxValue);
+ pcm[i * 2] = (byte)(s & 0xFF);
+ pcm[i * 2 + 1] = (byte)((s >> 8) & 0xFF);
+ }
+ return pcm;
+ }
+
+ #endregion
+}
diff --git a/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs b/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs
new file mode 100644
index 0000000..fe485b6
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Common/Scripts/PcmAudioPlayer.cs
@@ -0,0 +1,72 @@
+using System.Collections.Generic;
+using UnityEngine;
+using UnityEngine.Assertions;
+
+///
+/// Handles playback of PCM audio received as Base64-encoded strings.
+/// Uses an internal queue to play audio clips sequentially through an AudioSource.
+///
+public class PcmAudioPlayer : MonoBehaviour
+{
+ [SerializeField] private AudioSource audioSource;
+ [SerializeField] private int sampleRate = 16000;
+
+ private readonly Queue _clipQueue = new();
+
+ #region Unity Lifecycle
+
+ private void Start()
+ {
+ Assert.IsNotNull(audioSource, "Audio source is required");
+ }
+
+ private void Update()
+ {
+ if (audioSource.isPlaying || _clipQueue.Count == 0)
+ return;
+
+ audioSource.clip = _clipQueue.Dequeue();
+ audioSource.Play();
+ }
+
+ #endregion
+
+ #region Public Methods
+
+ ///
+ /// Converts Base64-encoded PCM audio into a Unity AudioClip
+ /// and adds it to the playback queue.
+ ///
+ /// Base64-encoded PCM16 audio data.
+ public void EnqueueBase64Audio(string base64Audio)
+ {
+ // Decode Base64 string into raw bytes
+ var bytes = System.Convert.FromBase64String(base64Audio);
+
+ // Each PCM16 sample uses 2 bytes
+ var sampleCount = bytes.Length / 2;
+ var samples = new float[sampleCount];
+
+ for (var i = 0; i < sampleCount; i++)
+ {
+ var sample = (short)((bytes[i * 2 + 1] << 8) | bytes[i * 2]);
+ samples[i] = sample / 32768f;
+ }
+
+ var clip = AudioClip.Create("AIConversationalClip", sampleCount, 1, sampleRate, false);
+ clip.SetData(samples, 0);
+
+ _clipQueue.Enqueue(clip);
+ }
+
+ ///
+ /// Stops playback immediately and clears any queued audio.
+ ///
+ public void StopImmediately()
+ {
+ _clipQueue.Clear();
+ audioSource.Stop();
+ }
+
+ #endregion
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs
new file mode 100644
index 0000000..1ba99c0
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbChatClient.cs
@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using UnityEngine;
+using UnityEngine.Networking;
+
+namespace SixtyDb
+{
+ // REST client for POST /v1/chat/completions. OpenAI-compatible payload,
+ // so the tool array is passed through verbatim.
+ //
+ // Returns a small (content, toolCalls) tuple; SixtyDbConvManager owns the
+ // tool-execution loop (dispatch → tool message → re-call until content).
+ public sealed class SixtyDbChatClient
+ {
+ private readonly SixtyDbConfig _config;
+
+ public SixtyDbChatClient(SixtyDbConfig config)
+ {
+ _config = config ?? throw new ArgumentNullException(nameof(config));
+ }
+
+ public struct Result
+ {
+ public string Content;
+ public List ToolCalls;
+ }
+
+ public struct ToolCall
+ {
+ public string Id;
+ public string Name;
+ public string ArgumentsJson;
+ }
+
+ public async Task CompleteAsync(
+ List messages,
+ JArray tools,
+ string model = null,
+ float temperature = 0.7f,
+ int topK = 20)
+ {
+ if (string.IsNullOrWhiteSpace(_config.apiKey))
+ throw new InvalidOperationException("60db apiKey is not set in the config asset.");
+
+ var body = new JObject
+ {
+ ["model"] = model ?? _config.llmModel,
+ ["messages"] = new JArray(messages),
+ ["stream"] = false,
+ ["temperature"] = temperature,
+ ["top_k"] = topK,
+ ["chat_template_kwargs"] = new JObject { ["enable_thinking"] = false },
+ };
+ if (tools != null && tools.Count > 0) body["tool"] = tools;
+
+ var url = $"{_config.apiBase.TrimEnd('/')}/v1/chat/completions";
+ using var req = new UnityWebRequest(url, UnityWebRequest.kHttpVerbPOST);
+ var raw = Encoding.UTF8.GetBytes(body.ToString(Formatting.None));
+ req.uploadHandler = new UploadHandlerRaw(raw) { contentType = "application/json" };
+ req.downloadHandler = new DownloadHandlerBuffer();
+ req.SetRequestHeader("Authorization", $"Bearer {_config.apiKey}");
+ req.SetRequestHeader("Accept", "application/json");
+
+ var op = req.SendWebRequest();
+ while (!op.isDone) await Task.Yield();
+
+ if (req.result != UnityWebRequest.Result.Success)
+ throw new InvalidOperationException(
+ $"[SixtyDbChatClient] HTTP {(int)req.responseCode}: {req.error} / {req.downloadHandler?.text}");
+
+ var json = JObject.Parse(req.downloadHandler.text);
+ var msg = json["choices"]?[0]?["message"] as JObject ?? new JObject();
+
+ var result = new Result
+ {
+ Content = msg["content"]?.ToString() ?? string.Empty,
+ ToolCalls = new List(),
+ };
+
+ if (msg["tool_calls"] is JArray toolCalls)
+ {
+ foreach (var tc in toolCalls)
+ {
+ if (tc is not JObject obj) continue;
+ var fn = obj["function"] as JObject;
+ result.ToolCalls.Add(new ToolCall
+ {
+ Id = obj["id"]?.ToString(),
+ Name = fn?["name"]?.ToString(),
+ ArgumentsJson = fn?["arguments"]?.ToString() ?? "{}",
+ });
+ }
+ }
+
+ return result;
+ }
+
+ // Convenience: build a {role, content} message in the shape /v1/chat/completions expects.
+ public static JObject UserMessage(string text) => new() { ["role"] = "user", ["content"] = text };
+ public static JObject SystemMessage(string text) => new() { ["role"] = "system", ["content"] = text };
+ public static JObject AssistantMessage(string text) => new() { ["role"] = "assistant", ["content"] = text };
+ public static JObject ToolMessage(string toolCallId, string name, string resultJson) =>
+ new() { ["role"] = "tool", ["tool_call_id"] = toolCallId, ["name"] = name, ["content"] = resultJson };
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs
new file mode 100644
index 0000000..6405c25
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConfig.cs
@@ -0,0 +1,29 @@
+using UnityEngine;
+
+namespace SixtyDb
+{
+ // ScriptableObject holding 60db credentials and endpoint overrides.
+ // Mirrors the role of ElevenLabsConfig / OpenAIConfig in the sibling packages.
+ [CreateAssetMenu(menuName = "SixtyDb/Config", fileName = "SixtyDbConfig")]
+ public class SixtyDbConfig : ScriptableObject
+ {
+ [Header("Authentication")]
+ public string apiKey;
+
+ [Header("Endpoints (override only if you're on staging / self-hosted)")]
+ public string apiBase = "https://api.60db.ai";
+ public string sttWebsocketUrl = "wss://api.60db.ai/ws/stt";
+ public string ttsWebsocketUrl = "wss://api.60db.ai/ws/tts";
+
+ [Header("Model defaults")]
+ public string llmModel = "60db-tiny";
+ public string defaultVoiceId = "fbb75ed2-975a-40c7-9e06-38e30524a9a1";
+
+ [Header("Audio")]
+ [Tooltip("Sample rate sent to /ws/stt. 60db browser mode expects 16 kHz linear PCM.")]
+ public int sttSampleRate = 16_000;
+
+ [Tooltip("Sample rate requested from /ws/tts. 60db returns LINEAR16 at this rate.")]
+ public int ttsSampleRate = 24_000;
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs
new file mode 100644
index 0000000..20371d7
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbConvManager.cs
@@ -0,0 +1,218 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+using UnityEngine;
+using UnityEngine.Events;
+
+namespace SixtyDb
+{
+ // 60db conversation orchestrator. Mirrors the public surface of
+ // ElevenLabs.AgentConversationManager and OpenAI.RealtimeConversationManager
+ // so existing scene wiring (UnityEvents, PcmAudioPlayer, MicrophoneStreamer)
+ // ports straight across.
+ //
+ // Per-turn loop:
+ // user mic chunk -> SttSocket
+ // final transcript -> SixtyDbChatClient (with tool registry)
+ // |- tool_calls present -> dispatch via SixtyDbToolRegistry -> re-call
+ // reply.content -> TtsSocket -> audio_chunk -> PcmAudioPlayer
+ //
+ // Barge-in: when STT emits speech_started while a TTS context is active,
+ // close that context immediately + clear the audio player queue.
+ public class SixtyDbConvManager : MonoBehaviour
+ {
+ [Header("Voice")]
+ [Tooltip("60db voice id. Leave blank to use SixtyDbConfig.defaultVoiceId.")]
+ [SerializeField] private string voiceId = "";
+
+ [Header("System prompt")]
+ [TextArea(2, 8)]
+ [SerializeField] private string systemPrompt =
+ "You are a helpful VR assistant. Keep replies under two short sentences " +
+ "unless the user explicitly asks for depth. Sound human.";
+
+ [Header("Startup")]
+ [SerializeField] private bool startOnAwake = true;
+
+ [Header("Behaviour")]
+ [Tooltip("When true, the user speaking interrupts the agent (closes TTS context, clears playback).")]
+ [SerializeField] private bool bargeInEnabled = true;
+ [Tooltip("Max tool-call round-trips per turn before falling back to plain content.")]
+ [SerializeField] private int maxToolRounds = 2;
+
+ [Header("Dependencies")]
+ [SerializeField] private SixtyDbConfig config;
+ [SerializeField] private MicrophoneStreamer micStreamer;
+ [SerializeField] private PcmAudioPlayer audioPlayer;
+
+ [Header("Events")]
+ public UnityEvent onAgentTranscript;
+ public UnityEvent onUserTranscript;
+ public UnityEvent onUserPartialTranscript;
+ public UnityEvent onAgentSpeaking;
+
+ private SixtyDbSttSocket _stt;
+ private SixtyDbTtsSocket _tts;
+ private SixtyDbChatClient _chat;
+ private readonly List _history = new();
+ private bool _busy;
+
+ private void Start()
+ {
+ if (startOnAwake) StartAgent();
+ }
+
+ private void Update()
+ {
+ _stt?.DispatchMessageQueue();
+ _tts?.DispatchMessageQueue();
+ }
+
+ private async void OnDisable() => await Shutdown();
+ private async void OnApplicationQuit() => await Shutdown();
+
+ // Same external surface as the OpenAI / ElevenLabs managers.
+ public async void StartAgent()
+ {
+ try
+ {
+ if (!config) throw new InvalidOperationException("SixtyDbConfig is not assigned.");
+ if (audioPlayer == null) throw new InvalidOperationException("PcmAudioPlayer is not assigned.");
+ if (micStreamer == null) throw new InvalidOperationException("MicrophoneStreamer is not assigned.");
+
+ _history.Clear();
+ _history.Add(SixtyDbChatClient.SystemMessage(systemPrompt));
+
+ _chat = new SixtyDbChatClient(config);
+ _stt = new SixtyDbSttSocket(config);
+ _tts = new SixtyDbTtsSocket(config, voiceId);
+
+ WireSttEvents();
+ WireTtsEvents();
+ micStreamer.OnAudioChunk += OnMicChunk;
+
+ await Task.WhenAll(_stt.Connect(), _tts.Connect());
+ micStreamer.StartStreaming();
+ }
+ catch (Exception ex)
+ {
+ Debug.LogError($"[SixtyDb] Startup failed: {ex}");
+ enabled = false;
+ }
+ }
+
+ public async void InterruptNow()
+ {
+ _tts?.Interrupt();
+ audioPlayer.StopImmediately();
+ onAgentSpeaking?.Invoke(false);
+ await Task.Yield();
+ }
+
+ private async Task Shutdown()
+ {
+ try
+ {
+ micStreamer.OnAudioChunk -= OnMicChunk;
+ micStreamer?.StopStreaming();
+ audioPlayer?.StopImmediately();
+ if (_stt != null) await _stt.Close();
+ if (_tts != null) await _tts.Close();
+ }
+ catch (Exception ex)
+ {
+ Debug.LogError($"[SixtyDb] Shutdown error: {ex}");
+ }
+ }
+
+ private void OnMicChunk(string base64Pcm) => _stt?.SendAudioChunk(base64Pcm);
+
+ private void WireSttEvents()
+ {
+ _stt.OnSpeechStarted += () =>
+ {
+ if (!bargeInEnabled) return;
+ if (_tts == null || !_tts.ContextActive) return;
+ _tts.Interrupt();
+ audioPlayer.StopImmediately();
+ onAgentSpeaking?.Invoke(false);
+ };
+
+ _stt.OnPartialTranscript += text => onUserPartialTranscript?.Invoke(text);
+
+ _stt.OnFinalTranscript += async text =>
+ {
+ if (_busy || string.IsNullOrWhiteSpace(text)) return;
+ onUserTranscript?.Invoke(text);
+ _history.Add(SixtyDbChatClient.UserMessage(text));
+
+ _busy = true;
+ try { await RunTurn(); }
+ catch (Exception ex) { Debug.LogError($"[SixtyDb] Turn failed: {ex}"); }
+ finally { _busy = false; }
+ };
+
+ _stt.OnError += msg => Debug.LogError($"[SixtyDb] STT error: {msg}");
+ }
+
+ private void WireTtsEvents()
+ {
+ _tts.OnAudioChunk += audioPlayer.EnqueueBase64Audio;
+ _tts.OnSpeakStart += () => onAgentSpeaking?.Invoke(true);
+ _tts.OnSpeakEnd += () => onAgentSpeaking?.Invoke(false);
+ _tts.OnError += msg => Debug.LogError($"[SixtyDb] TTS error: {msg}");
+ }
+
+ private async Task RunTurn()
+ {
+ var tools = SixtyDbToolRegistry.HasTools ? SixtyDbToolRegistry.GetToolsSpec() : null;
+
+ for (var round = 0; round < maxToolRounds; round++)
+ {
+ var reply = await _chat.CompleteAsync(_history, tools);
+
+ if (reply.ToolCalls.Count == 0)
+ {
+ var content = reply.Content ?? string.Empty;
+ if (string.IsNullOrWhiteSpace(content)) return;
+ _history.Add(SixtyDbChatClient.AssistantMessage(content));
+ onAgentTranscript?.Invoke(content);
+ await _tts.Speak(content);
+ return;
+ }
+
+ // Append the assistant's tool-call turn, then run each tool locally
+ // and append the result so the next round can incorporate it.
+ _history.Add(SixtyDbChatClient.AssistantMessage(reply.Content ?? string.Empty));
+ foreach (var call in reply.ToolCalls)
+ {
+ var argsJson = call.ArgumentsJson;
+ JObject args;
+ try { args = JObject.Parse(string.IsNullOrEmpty(argsJson) ? "{}" : argsJson); }
+ catch { args = new JObject(); }
+
+ string resultJson;
+ try
+ {
+ if (SixtyDbToolRegistry.TryGetHandler(call.Name, out var handler))
+ {
+ var res = await handler(args);
+ resultJson = (res ?? new JObject()).ToString(Newtonsoft.Json.Formatting.None);
+ }
+ else
+ {
+ resultJson = $"{{\"error\":\"unknown tool '{call.Name}'\"}}";
+ }
+ }
+ catch (Exception ex)
+ {
+ resultJson = $"{{\"error\":\"{ex.Message}\"}}";
+ }
+ _history.Add(SixtyDbChatClient.ToolMessage(call.Id, call.Name, resultJson));
+ }
+ }
+ Debug.LogWarning("[SixtyDb] Tool round budget exhausted; aborting turn.");
+ }
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs
new file mode 100644
index 0000000..e868d92
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbSttSocket.cs
@@ -0,0 +1,139 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using NativeWebSocket;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using UnityEngine;
+
+namespace SixtyDb
+{
+ // Wraps wss://api.60db.ai/ws/stt (browser mode, linear PCM 16k).
+ // Lifecycle:
+ // connect()
+ // OnOpen -> server emits connection_established
+ // -> send 'start' -> server emits 'connected' -> OnReady
+ // per mic chunk: send { type: 'audio', audio: , encoding: 'linear', sample_rate: 16000 }
+ // server emits: speech_started, transcription (is_final + speech_final), session_stopped
+ public sealed class SixtyDbSttSocket
+ {
+ public event Action OnReady;
+ public event Action OnSpeechStarted;
+ public event Action OnPartialTranscript;
+ public event Action OnFinalTranscript;
+ public event Action OnError;
+ public event Action OnClose;
+
+ private readonly SixtyDbConfig _config;
+ private WebSocket _ws;
+ private bool _ready;
+
+ public WebSocket Underlying => _ws;
+ public bool IsReady => _ready;
+
+ public SixtyDbSttSocket(SixtyDbConfig config)
+ {
+ _config = config ?? throw new ArgumentNullException(nameof(config));
+ }
+
+ public async Task Connect()
+ {
+ var url = $"{_config.sttWebsocketUrl.TrimEnd('/')}?apiKey={Uri.EscapeDataString(_config.apiKey)}";
+ _ws = new WebSocket(url);
+ _ws.OnOpen += HandleOpen;
+ _ws.OnMessage += HandleMessage;
+ _ws.OnError += msg => OnError?.Invoke(msg);
+ _ws.OnClose += _ => { _ready = false; OnClose?.Invoke(); };
+ await _ws.Connect();
+ }
+
+ public void DispatchMessageQueue() => _ws?.DispatchMessageQueue();
+
+ public void SendAudioChunk(string base64Pcm)
+ {
+ if (_ws == null || _ws.State != WebSocketState.Open || !_ready) return;
+ var payload = new Dictionary
+ {
+ { "type", "audio" },
+ { "audio", base64Pcm },
+ { "encoding", "linear" },
+ { "sample_rate", _config.sttSampleRate },
+ };
+ _ = _ws.SendText(JsonConvert.SerializeObject(payload));
+ }
+
+ public async Task Close()
+ {
+ try
+ {
+ if (_ws != null && _ws.State == WebSocketState.Open)
+ {
+ // Best-effort 'stop' for billing summary; ignore failures.
+ await _ws.SendText("{\"type\":\"stop\"}");
+ await _ws.Close();
+ }
+ }
+ catch { /* socket already gone */ }
+ }
+
+ private void HandleOpen()
+ {
+ // Wait for the server's 'connection_established' before sending 'start'.
+ // That hand-off is done inside HandleMessage to keep the state machine local.
+ }
+
+ private void HandleMessage(byte[] bytes)
+ {
+ var text = Encoding.UTF8.GetString(bytes);
+ JObject msg;
+ try { msg = JObject.Parse(text); }
+ catch { return; }
+
+ switch (msg["type"]?.ToString())
+ {
+ case "connecting":
+ return;
+ case "connection_established":
+ SendStart();
+ return;
+ case "connected":
+ _ready = true;
+ OnReady?.Invoke();
+ return;
+ case "speech_started":
+ OnSpeechStarted?.Invoke();
+ return;
+ case "transcription":
+ {
+ var transcript = msg["transcript"]?.ToString() ?? msg["text"]?.ToString();
+ if (string.IsNullOrEmpty(transcript)) return;
+ var isFinal = (bool?)msg["is_final"] ?? false;
+ var speechFinal = (bool?)msg["speech_final"] ?? false;
+ if (isFinal && speechFinal) OnFinalTranscript?.Invoke(transcript);
+ else OnPartialTranscript?.Invoke(transcript);
+ return;
+ }
+ case "session_stopped":
+ _ready = false;
+ return;
+ case "error":
+ OnError?.Invoke(msg["message"]?.ToString() ?? "unknown STT error");
+ return;
+ }
+ }
+
+ private void SendStart()
+ {
+ var payload = new Dictionary
+ {
+ { "type", "start" },
+ { "encoding", "linear" },
+ { "sample_rate", _config.sttSampleRate },
+ { "utterance_end_ms", 500 },
+ { "continuous_mode", true },
+ };
+ _ = _ws.SendText(JsonConvert.SerializeObject(payload));
+ }
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs
new file mode 100644
index 0000000..8ed8909
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbToolRegistry.cs
@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Newtonsoft.Json.Linq;
+
+namespace SixtyDb
+{
+ // Generic registry for 60db agent tools. Mirrors com.convai.openai's
+ // AgentToolRegistry so existing scene wiring patterns port directly.
+ //
+ // Tool specs use the OpenAI-compatible JSON shape — 60db's
+ // /v1/chat/completions accepts the same `tool` array.
+ public static class SixtyDbToolRegistry
+ {
+ public delegate Task ToolHandler(JObject args);
+
+ private static readonly Dictionary NameToHandler =
+ new(StringComparer.OrdinalIgnoreCase);
+ private static readonly List ToolSpecs = new();
+
+ public static void Register(string name, ToolHandler handler, JObject toolSpec = null)
+ {
+ if (string.IsNullOrWhiteSpace(name) || handler == null) return;
+ NameToHandler[name] = handler;
+ if (toolSpec != null) ToolSpecs.Add(toolSpec);
+ }
+
+ public static bool TryGetHandler(string name, out ToolHandler handler) =>
+ NameToHandler.TryGetValue(name, out handler);
+
+ public static JArray GetToolsSpec() => new JArray(ToolSpecs);
+
+ public static bool HasTools => ToolSpecs.Count > 0;
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs
new file mode 100644
index 0000000..7a20786
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbTtsSocket.cs
@@ -0,0 +1,144 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading.Tasks;
+using NativeWebSocket;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using UnityEngine;
+
+namespace SixtyDb
+{
+ // Wraps wss://api.60db.ai/ws/tts. Lifecycle:
+ // Connect()
+ // OnOpen -> server emits connection_established -> OnReady
+ // Speak(text):
+ // CreateContext if needed -> context_created
+ // send_text + flush_context -> audio_chunk frames -> flush_completed
+ // Interrupt(): close_context (drops the in-flight reply, used for barge-in).
+ public sealed class SixtyDbTtsSocket
+ {
+ public event Action OnReady;
+ public event Action OnSpeakStart;
+ public event Action OnSpeakEnd;
+ public event Action OnAudioChunk;
+ public event Action OnError;
+ public event Action OnClose;
+
+ private readonly SixtyDbConfig _config;
+ private readonly string _voiceId;
+ private WebSocket _ws;
+ private string _contextId;
+ private TaskCompletionSource _contextReady;
+ private int _chunksSinceFlush;
+
+ public WebSocket Underlying => _ws;
+ public bool ContextActive => !string.IsNullOrEmpty(_contextId);
+
+ public SixtyDbTtsSocket(SixtyDbConfig config, string voiceId)
+ {
+ _config = config ?? throw new ArgumentNullException(nameof(config));
+ _voiceId = string.IsNullOrWhiteSpace(voiceId) ? config.defaultVoiceId : voiceId;
+ }
+
+ public async Task Connect()
+ {
+ var url = $"{_config.ttsWebsocketUrl.TrimEnd('/')}?apiKey={Uri.EscapeDataString(_config.apiKey)}";
+ _ws = new WebSocket(url);
+ _ws.OnMessage += HandleMessage;
+ _ws.OnError += msg => OnError?.Invoke(msg);
+ _ws.OnClose += _ => { _contextId = null; OnClose?.Invoke(); };
+ await _ws.Connect();
+ }
+
+ public void DispatchMessageQueue() => _ws?.DispatchMessageQueue();
+
+ public async Task Speak(string text)
+ {
+ if (_ws == null || _ws.State != WebSocketState.Open) return;
+ if (string.IsNullOrEmpty(text)) return;
+
+ if (string.IsNullOrEmpty(_contextId))
+ await EnsureContext();
+
+ _chunksSinceFlush = 0;
+ await _ws.SendText(JsonConvert.SerializeObject(new Dictionary
+ {
+ { "type", "send_text" }, { "text", text }, { "context_id", _contextId },
+ }));
+ await _ws.SendText(JsonConvert.SerializeObject(new Dictionary
+ {
+ { "type", "flush_context" }, { "context_id", _contextId },
+ }));
+ }
+
+ // Cancel the current reply mid-speech. Used by SixtyDbConvManager for
+ // barge-in when the STT WS reports speech_started.
+ public void Interrupt()
+ {
+ if (_ws == null || _ws.State != WebSocketState.Open || string.IsNullOrEmpty(_contextId))
+ return;
+ var json = JsonConvert.SerializeObject(new Dictionary
+ {
+ { "type", "close_context" }, { "context_id", _contextId },
+ });
+ _ = _ws.SendText(json);
+ _contextId = null;
+ }
+
+ public async Task Close()
+ {
+ Interrupt();
+ try { if (_ws != null && _ws.State == WebSocketState.Open) await _ws.Close(); }
+ catch { /* socket already gone */ }
+ }
+
+ private Task EnsureContext()
+ {
+ _contextReady = new TaskCompletionSource();
+ _ = _ws.SendText(JsonConvert.SerializeObject(new Dictionary
+ {
+ { "type", "create_context" },
+ { "voice_id", _voiceId },
+ { "audio_encoding", "LINEAR16" },
+ { "sample_rate_hertz", _config.ttsSampleRate },
+ }));
+ return _contextReady.Task;
+ }
+
+ private void HandleMessage(byte[] bytes)
+ {
+ var text = Encoding.UTF8.GetString(bytes);
+ JObject msg;
+ try { msg = JObject.Parse(text); }
+ catch { return; }
+
+ switch (msg["type"]?.ToString())
+ {
+ case "connection_established":
+ OnReady?.Invoke();
+ return;
+ case "context_created":
+ _contextId = msg["context_id"]?.ToString();
+ _contextReady?.TrySetResult(true);
+ return;
+ case "audio_chunk":
+ {
+ var b64 = msg["audioContent"]?.ToString() ?? msg["audio"]?.ToString();
+ if (string.IsNullOrEmpty(b64)) return;
+ if (_chunksSinceFlush == 0) OnSpeakStart?.Invoke();
+ _chunksSinceFlush++;
+ OnAudioChunk?.Invoke(b64);
+ return;
+ }
+ case "flush_completed":
+ OnSpeakEnd?.Invoke();
+ _chunksSinceFlush = 0;
+ return;
+ case "error":
+ OnError?.Invoke(msg["message"]?.ToString() ?? "unknown TTS error");
+ return;
+ }
+ }
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs b/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs
new file mode 100644
index 0000000..f8a33c2
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/Scripts/SixtyDbWebSocketEvents.cs
@@ -0,0 +1,57 @@
+using Newtonsoft.Json;
+
+namespace SixtyDb
+{
+ // POCOs mapped to the JSON envelope used by 60db's /ws/stt and /ws/tts
+ // streams. Mirrors the role of ElevenLabsWebSocketEvents / OpenAIWebSocketEvents.
+ // Field names lifted verbatim from https://docs.60db.ai/websocket-api/{stt,tts}.
+
+ public class SixtyDbBaseEvent
+ {
+ [JsonProperty("type")] public string Type { get; set; }
+ }
+
+ // ---- STT ----------------------------------------------------------------
+
+ public class SttConnectedEvent : SixtyDbBaseEvent {}
+
+ public class SttSpeechStartedEvent : SixtyDbBaseEvent {}
+
+ public class SttTranscriptionEvent : SixtyDbBaseEvent
+ {
+ [JsonProperty("transcript")] public string Transcript { get; set; }
+ [JsonProperty("text")] public string Text { get; set; }
+ [JsonProperty("is_final")] public bool IsFinal { get; set; }
+ [JsonProperty("speech_final")] public bool SpeechFinal { get; set; }
+ }
+
+ public class SttSessionStoppedEvent : SixtyDbBaseEvent {}
+
+ public class SttErrorEvent : SixtyDbBaseEvent
+ {
+ [JsonProperty("message")] public string Message { get; set; }
+ }
+
+ // ---- TTS ----------------------------------------------------------------
+
+ public class TtsContextCreatedEvent : SixtyDbBaseEvent
+ {
+ [JsonProperty("context_id")] public string ContextId { get; set; }
+ }
+
+ public class TtsAudioChunkEvent : SixtyDbBaseEvent
+ {
+ [JsonProperty("audioContent")] public string AudioBase64 { get; set; }
+ [JsonProperty("audio")] public string AudioFallback { get; set; }
+
+ public string ResolveAudio() =>
+ !string.IsNullOrEmpty(AudioBase64) ? AudioBase64 : AudioFallback;
+ }
+
+ public class TtsFlushCompletedEvent : SixtyDbBaseEvent {}
+
+ public class TtsErrorEvent : SixtyDbBaseEvent
+ {
+ [JsonProperty("message")] public string Message { get; set; }
+ }
+}
diff --git a/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef b/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef
new file mode 100644
index 0000000..d81b493
--- /dev/null
+++ b/com.convai.sixtydb/Runtime/com.convai.sixtydb.asmdef
@@ -0,0 +1,17 @@
+{
+ "name": "com.convai.sixtydb",
+ "rootNamespace": "SixtyDb",
+ "references": [
+ "endel.nativewebsocket",
+ "Unity.Nuget.Newtonsoft-Json"
+ ],
+ "includePlatforms": [],
+ "excludePlatforms": [],
+ "allowUnsafeCode": false,
+ "overrideReferences": false,
+ "precompiledReferences": [],
+ "autoReferenced": true,
+ "defineConstraints": [],
+ "versionDefines": [],
+ "noEngineReferences": false
+}
diff --git a/com.convai.sixtydb/package.json b/com.convai.sixtydb/package.json
new file mode 100644
index 0000000..dec36b9
--- /dev/null
+++ b/com.convai.sixtydb/package.json
@@ -0,0 +1,22 @@
+{
+ "name": "com.convai.sixtydb",
+ "displayName": "ConversationalAI [60db]",
+ "version": "1.0.0",
+ "unity": "6000.0",
+ "description": "A Unity package that enables real-time conversational AI interactions by composing 60db's STT WebSocket, LLM chat completions, and TTS WebSocket into a single Quest-friendly conversation manager. Mirrors the surface of com.convai.elevenlabs and com.convai.openai.",
+ "dependencies": {
+ "com.unity.nuget.newtonsoft-json": "3.0.0"
+ },
+ "author": {
+ "name": "Daniel Oquelis",
+ "email": "daniel.oquelis@gmail.com",
+ "url": "https://github.com/danieloquelis"
+ },
+ "samples": [
+ {
+ "displayName": "Conversational AI Samples",
+ "description": "Please clone the main repository to access to samples",
+ "path": "Samples~/ConversationalSample"
+ }
+ ]
+}