diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 61220a0fae..8db8e8d82d 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -145,6 +145,7 @@ "arm-teleop-module": "dimos.teleop.quest.quest_extensions.ArmTeleopModule", "b-box-navigation-module": "dimos.navigation.bbox_navigation.BBoxNavigationModule", "b1-connection-module": "dimos.robot.unitree.b1.connection.B1ConnectionModule", + "babylon-viewer-module": "dimos.web.viewer.module.BabylonViewerModule", "basic-path-follower": "dimos.navigation.basic_path_follower.module.BasicPathFollower", "camera-module": "dimos.hardware.sensors.camera.module.CameraModule", "cartesian-motion-controller": "dimos.manipulation.control.servo_control.cartesian_motion_controller.CartesianMotionController", @@ -194,6 +195,7 @@ "joystick-module": "dimos.robot.unitree.b1.joystick_module.JoystickModule", "keyboard-teleop": "dimos.robot.unitree.keyboard_teleop.KeyboardTeleop", "keyboard-teleop-module": "dimos.teleop.keyboard.keyboard_teleop_module.KeyboardTeleopModule", + "lcm-web-socket-bridge-module": "dimos.web.lcm_bridge.module.LcmWebSocketBridgeModule", "local-planner": "dimos.navigation.cmu_nav.modules.local_planner.local_planner.LocalPlanner", "manipulation-module": "dimos.manipulation.manipulation_module.ManipulationModule", "map": "dimos.robot.unitree.type.map.Map", diff --git a/dimos/robot/unitree/g1/blueprints/basic/unitree_g1_groot_wbc.py b/dimos/robot/unitree/g1/blueprints/basic/unitree_g1_groot_wbc.py index 7721fba9ff..ae501990e7 100644 --- a/dimos/robot/unitree/g1/blueprints/basic/unitree_g1_groot_wbc.py +++ b/dimos/robot/unitree/g1/blueprints/basic/unitree_g1_groot_wbc.py @@ -46,6 +46,7 @@ from __future__ import annotations +import os from pathlib import Path from typing import Any, cast @@ -293,6 +294,56 @@ def _precomposed_g1_scene(package: ScenePackage) -> Path | None: (VoxelGridMapper, "lidar", "pointcloud"), (ControlCoordinator, "twist_command", "cmd_vel"), ] + + def _babylon_viewer() -> Any | None: + """Browser operator console (dimos/web/viewer) alongside the sim. + + Mirrors the robot from coordinator joint state + odom; pointclouds, + path and camera stream over the LCM<->WS bridge; WASD in the page + publishes cmd_vel — the same source-blind channel the planner uses. + On by default in sim; DIMOS_ENABLE_BABYLON=0 to suppress. + """ + if os.environ.get("DIMOS_ENABLE_BABYLON", "1").lower() in ("0", "false", "no"): + return None + from dimos.web.viewer.module import BabylonViewerModule + + kwargs: dict[str, Any] = dict( + mjcf_path=_ROBOT_ONLY_MJCF_PATH, + mesh_dir=LfsPath("g1_urdf/meshes"), + port=int(os.environ.get("DIMOS_BABYLON_PORT", "8091")), + ) + if global_config.scene_package is not None: + from dimos.simulation.scenes.catalog import resolve_scene_package + + try: + package = resolve_scene_package(global_config.scene_package) + except (FileNotFoundError, ValueError): + package = None + # Prefer a Babylon-cooked GLB; legacy packages only carry the + # generic browser visual. + visual = ( + package.browser_visual_path("babylon") or package.visual_path + if package is not None + else None + ) + if package is not None and visual is not None and visual.exists(): + # Same GLB + alignment the sim world was cooked from, so the + # browser shows the world MuJoCo is stepping. + kwargs.update( + scene_path=str(visual), + scene_scale=package.alignment.scale, + scene_translation=package.alignment.translation, + scene_rotation_zyx_deg=package.alignment.rotation_zyx_deg, + scene_y_up=package.alignment.y_up, + ) + return BabylonViewerModule.blueprint(**kwargs) + + _babylon_bp = _babylon_viewer() + if _babylon_bp is not None: + from dimos.web.viewer.module import BabylonViewerModule as _BabylonMod + + _nav_stack = autoconnect(_nav_stack, _babylon_bp) + _remappings.append((_BabylonMod, "joint_state", "coordinator_joint_state")) else: from dimos.robot.unitree.g1.wholebody_connection import G1WholeBodyConnection diff --git a/dimos/web/lcm_bridge/README.md b/dimos/web/lcm_bridge/README.md new file mode 100644 index 0000000000..bf6036285b --- /dev/null +++ b/dimos/web/lcm_bridge/README.md @@ -0,0 +1,96 @@ +# LCM ↔ WebSocket bridge + +The dimos bus, in the browser. A subscribe-all bridge that forwards LCM +small-message packets to WebSocket clients and republishes packets clients +send back — so browser code publishes and subscribes like any other bus +peer, using [`@dimos/msgs`](https://jsr.io/@dimos/msgs) to encode/decode. + +This is the load-tested extraction of the Babylon scene viewer's `/lcm-ws` +endpoint, contributed as a concrete v0 for the Dimos Web discussion: + +- [#2710 Dimos Web Proposal](https://github.com/dimensionalOS/dimos/issues/2710) — + this is a working data point for "the bridge is a DimOS module" (question 2) + and the local/LAN case (question 3). +- [#2502 TS API Spec](https://github.com/dimensionalOS/dimos/issues/2502) — + the central `DimosWebsocket` server sketched there subsumes this; until it + exists, this module is the smallest thing that gives any blueprint a + browser-reachable bus. +- [#2708 Dimos Web SDK](https://github.com/dimensionalOS/dimos/issues/2708) — + the flow-control here (latest-wins buffering, per-channel rate caps) is + the operational evidence behind the per-stream QoS requirements. + +## Usage + +Standalone, in any blueprint: + +```python +from dimos.web.lcm_bridge.module import LcmWebSocketBridgeModule + +autoconnect( + my_robot_blueprint, + LcmWebSocketBridgeModule.blueprint( + port=9669, + channel_rate_hz={"/global_map": 1.0}, # throttle the WS leg only + topic_blocklist=["/camera_image_raw*"], # never forward these + ), +) +``` + +Browser: + +```html + + +``` + +Embedded in an existing Starlette app (the Babylon viewer pattern): + +```python +from dimos.web.lcm_bridge.bridge import LcmWebSocketBridge + +bridge = LcmWebSocketBridge(channel_rate_hz={"/global_map": 1.0}) +bridge.start() +app = Starlette(routes=[*my_routes, *bridge.routes()]) +``` + +`GET /` on the standalone module returns JSON debug counters +(clients, forwarded, rate_capped, filtered, published_from_clients). + +Requires the `web` extra (`starlette` + `uvicorn`). + +## Flow control (why this isn't a naive forwarder) + +Learned running multi-MB pointclouds at 10 Hz into browser tabs: + +- **Latest-wins buffering** — one slot per (client, channel); when the bus + outpaces a client's socket, stale packets are overwritten, never queued. + The naive queue-per-packet design put browsers 10–15 s behind real time. +- **Single drain task per client** — exactly one in-flight send per socket; + a stalled client is closed (its JS auto-reconnects) instead of holding + buffers hostage. +- **Per-channel rate caps** — fnmatch pattern → max Hz, applied to the + WebSocket leg only; in-process bus consumers are unaffected. + +## Known limitation: LCM only + +The bridge taps the **LCM** bus. Runs using the Zenoh transport default +(`--transport zenoh` / `DIMOS_TRANSPORT`) publish nothing on LCM, so the +bridge forwards nothing — run with `--transport lcm` for now. A Zenoh +subscribe-all twin is the natural follow-up (payloads are the same +encoded bytes; only the tap changes), and is exactly the +transport-agnostic central bridge #2502 sketches. + +## Deliberately out of scope (v0) + +- **Per-connection QoS** (client-requested rates/filters) — that's the + `setQos` API in #2502; here filtering and caps are server config. +- **Fragmented LCM messages** (>~64 KB) — not bridged. Big payloads (video, + raw pointclouds) should be encoded/decimated server-side first, per #2708. +- **WebTransport / brokered remote access** — #2710's transport and broker + questions; this module is the same-host/LAN case. +- **Auth** — LAN-trust, like every other dimos listener today. diff --git a/dimos/web/lcm_bridge/bridge.py b/dimos/web/lcm_bridge/bridge.py new file mode 100644 index 0000000000..da6fbb1ab4 --- /dev/null +++ b/dimos/web/lcm_bridge/bridge.py @@ -0,0 +1,359 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LCM <-> WebSocket bridge: the dimos bus, in the browser. + +Both directions speak the standard LCM small-message wire format +``[magic u32 BE][seq u32 BE][channel utf-8][\\0][payload]``, so +``@dimos/msgs`` in the browser decodes bus traffic unchanged, and messages +the browser publishes are indistinguishable from any other bus peer's. +``static/lcm_client.js`` is the matching browser client. + +Extracted from the Babylon scene viewer's ``/lcm-ws`` endpoint, where this +design accumulated its flow control under real load (a Rust voxel mapper +publishing multi-MB pointclouds at 10 Hz into a browser tab): + +- **Latest-wins buffering.** One slot per (client, channel). When the bus + emits faster than a client's socket drains, the slot is overwritten and + old packets drop on the floor — the browser gets the freshest state at + its own drain rate. The naive alternative (queue one send per packet) + backed asyncio up with hundreds of pending sends and put the browser + 10-15 s behind real time. +- **One in-flight send per client.** A single drain task owns each socket's + send side; a stalled client times out and is closed so its browser-side + auto-reconnect fires, instead of holding buffers hostage. +- **Per-channel rate caps.** Server-side throttling of the WebSocket leg + only — bus consumers are unaffected. Note: a capped packet is dropped, + not deferred, so a channel that goes quiet right after a burst can leave + the client one update stale until the next publish. + +This is deliberately a server-level v0 of the QoS story sketched in the +Dimos Web proposals (#2708, #2710) and the TS API spec (#2502): rate caps +and allow/block lists are bridge configuration here, not yet per-connection +requests from the client. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping +from contextlib import suppress +from fnmatch import fnmatchcase +import struct +import threading +import time + +import lcm as lcmlib +from starlette.routing import WebSocketRoute +from starlette.websockets import WebSocket, WebSocketDisconnect + +from dimos.utils.logging_config import setup_logger + +logger = setup_logger() + +#: Magic prefix of the LCM small-message (non-fragmented) packet, "LC02". +LCM_MAGIC_SHORT = 0x4C433032 + +#: Fragmented LCM messages (>~64 KB) use a different magic and are not yet +#: bridged; big payloads (video, raw pointclouds) should be encoded or +#: decimated server-side before they reach the bus channel a browser reads. +_WS_SEND_TIMEOUT_S = 1.0 +_WS_CLOSE_TIMEOUT_S = 0.5 + + +def encode_packet(channel: str, payload: bytes, seq: int = 0) -> bytes: + """Frame ``payload`` as an LCM small-message packet for ``channel``.""" + return struct.pack(">II", LCM_MAGIC_SHORT, seq) + channel.encode("utf-8") + b"\x00" + payload + + +def decode_packet(packet: bytes) -> tuple[str, bytes] | None: + """Parse an LCM small-message packet into ``(channel, payload)``. + + Returns None for anything that isn't a well-formed small-message + packet (wrong magic, truncated header, unterminated channel). + """ + if len(packet) < 8: + return None + magic, _seq = struct.unpack(">II", packet[:8]) + if magic != LCM_MAGIC_SHORT: + return None + try: + null_idx = packet.index(b"\x00", 8) + except ValueError: + return None + channel = packet[8:null_idx].decode("utf-8", errors="replace") + return channel, packet[null_idx + 1 :] + + +def _topic_of(channel: str) -> str: + # dimos LCM channels are "#". + return channel.split("#", 1)[0] + + +class LcmWebSocketBridge: + """Bridges every LCM bus channel to browser WebSocket clients. + + Embeddable: a host Starlette app adds :meth:`routes` next to its own + (the Babylon viewer pattern), or + :class:`dimos.web.lcm_bridge.module.LcmWebSocketBridgeModule` runs it + as a standalone server. Call :meth:`start` before serving and + :meth:`stop` on shutdown. + + ``topic_allowlist``/``topic_blocklist`` take :mod:`fnmatch` patterns + matched against both the topic (``/odom``) and the full channel + (``/odom#geometry_msgs.PoseStamped``); the blocklist wins. Filtering + applies to the bus->browser direction only. ``channel_rate_hz`` maps + the same kind of pattern to a per-client forward-rate cap. + + ``on_bus_message`` is a tap called with ``(channel, payload)`` for + every bus message on the LCM reader thread — hosts use it to observe + traffic they'd otherwise need a second subscribe-all handle for. It + must not block. + """ + + def __init__( + self, + *, + lcm_url: str | None = None, + channel_rate_hz: Mapping[str, float] | None = None, + topic_allowlist: list[str] | None = None, + topic_blocklist: list[str] | None = None, + on_bus_message: Callable[[str, bytes], None] | None = None, + ws_path: str = "/lcm-ws", + ) -> None: + self._lcm_url = lcm_url + self._channel_rate_hz = dict(channel_rate_hz or {}) + self._topic_allowlist = list(topic_allowlist) if topic_allowlist is not None else None + self._topic_blocklist = list(topic_blocklist or []) + self._on_bus_message = on_bus_message + self._ws_path = ws_path + + self._lcm: lcmlib.LCM | None = None + self._subscription: object | None = None + self._handle_thread: threading.Thread | None = None + self._stop_event = threading.Event() + self._loop: asyncio.AbstractEventLoop | None = None + self._seq = 0 + + self._clients: set[WebSocket] = set() + # Per-client latest-packet-per-channel slot + wake event + drain + # task. The LCM thread overwrites slots (latest-wins); each drain + # task is the only sender on its socket, so sends never compete. + self._pending: dict[WebSocket, dict[str, bytes]] = {} + self._wake: dict[WebSocket, asyncio.Event] = {} + self._drain_tasks: dict[WebSocket, asyncio.Task[None]] = {} + self._last_forward_ts: dict[WebSocket, dict[str, float]] = {} + + # Per-channel filter/rate decisions are pattern matches; cache them + # by exact channel so the hot path is two dict lookups. + self._forward_cache: dict[str, bool] = {} + self._min_interval_cache: dict[str, float] = {} + + # Debug counters, exposed by the standalone module's status page. + self.forwarded = 0 + self.rate_capped = 0 + self.filtered = 0 + self.published_from_clients = 0 + + # ---- lifecycle ------------------------------------------------------- + + def start(self) -> None: + """Open the LCM handle and start the bus reader thread.""" + self._lcm = lcmlib.LCM(self._lcm_url) if self._lcm_url else lcmlib.LCM() + subscription = self._lcm.subscribe(".*", self._on_bus_msg) + subscription.set_queue_capacity(10000) + self._subscription = subscription + self._handle_thread = threading.Thread( + target=self._handle_loop, + name="lcm-ws-bridge", + daemon=True, + ) + self._handle_thread.start() + + def stop(self) -> None: + self._stop_event.set() + if self._handle_thread is not None and self._handle_thread.is_alive(): + self._handle_thread.join(timeout=2.0) + + def routes(self) -> list[WebSocketRoute]: + """Starlette routes to mount on the serving app.""" + return [WebSocketRoute(self._ws_path, self.websocket_endpoint)] + + @property + def lcm(self) -> lcmlib.LCM | None: + """The bridge's bus handle (None before :meth:`start`).""" + return self._lcm + + @property + def client_count(self) -> int: + return len(self._clients) + + # ---- bus -> browser -------------------------------------------------- + + def _handle_loop(self) -> None: + assert self._lcm is not None + while not self._stop_event.is_set(): + try: + self._lcm.handle_timeout(100) + except Exception: + logger.exception("lcm-ws bridge: bus handle failed") + + def _on_bus_msg(self, channel: str, data: bytes) -> None: + # Runs on the LCM reader thread. Framing and fan-out happen on the + # asyncio loop via call_soon_threadsafe so the per-client drain + # serialises naturally. + if self._on_bus_message is not None: + try: + self._on_bus_message(channel, data) + except Exception: + logger.exception("lcm-ws bridge: on_bus_message tap failed") + if not self._clients: + return + if not self._forward_allowed(channel): + self.filtered += 1 + return + loop = self._loop + if loop is None: + return + loop.call_soon_threadsafe(self._enqueue, channel, data) + + def _forward_allowed(self, channel: str) -> bool: + cached = self._forward_cache.get(channel) + if cached is not None: + return cached + topic = _topic_of(channel) + + def _matches(patterns: list[str]) -> bool: + return any( + fnmatchcase(topic, pattern) or fnmatchcase(channel, pattern) for pattern in patterns + ) + + allowed = not _matches(self._topic_blocklist) and ( + self._topic_allowlist is None or _matches(self._topic_allowlist) + ) + self._forward_cache[channel] = allowed + return allowed + + def _min_interval_s(self, channel: str) -> float: + cached = self._min_interval_cache.get(channel) + if cached is not None: + return cached + topic = _topic_of(channel) + interval = 0.0 + for pattern, rate_hz in self._channel_rate_hz.items(): + if rate_hz > 0 and (fnmatchcase(topic, pattern) or fnmatchcase(channel, pattern)): + interval = max(interval, 1.0 / rate_hz) + self._min_interval_cache[channel] = interval + return interval + + def _enqueue(self, channel: str, data: bytes) -> None: + # Runs on the asyncio loop thread. Synthesise the wire bytes once, + # then for each client overwrite its latest-packet slot for this + # channel and wake its drain task. + if not self._clients: + return + min_interval = self._min_interval_s(channel) + now = time.monotonic() if min_interval > 0.0 else 0.0 + packet = encode_packet(channel, data, self._seq) + self._seq = (self._seq + 1) & 0xFFFFFFFF + for websocket in tuple(self._clients): + pending = self._pending.get(websocket) + wake = self._wake.get(websocket) + if pending is None or wake is None: + continue + if min_interval > 0.0: + last = self._last_forward_ts.get(websocket, {}) + if now - last.get(channel, 0.0) < min_interval: + self.rate_capped += 1 + continue # rate-cap: drop this packet for this client + last[channel] = now + pending[channel] = packet + wake.set() + self.forwarded += 1 + + async def _drain_loop(self, websocket: WebSocket) -> None: + # One per connected client; the sole sender on its socket. Waits + # for the wake event, snapshots the pending dict, clears it, sends + # every captured packet. A send that stalls past the timeout (or + # errors) closes the client; the receive coroutine wakes on the + # disconnect and runs the normal cleanup, and the browser client's + # auto-reconnect takes it from there. + pending = self._pending.get(websocket) + wake = self._wake.get(websocket) + if pending is None or wake is None: + return + try: + while True: + await wake.wait() + wake.clear() + if not pending: + continue + snapshot = list(pending.values()) + pending.clear() + for packet in snapshot: + await asyncio.wait_for(websocket.send_bytes(packet), timeout=_WS_SEND_TIMEOUT_S) + except asyncio.CancelledError: + pass + except Exception: + logger.warning("lcm-ws bridge: drain failed, closing client", exc_info=True) + with suppress(Exception): + await asyncio.wait_for(websocket.close(), timeout=_WS_CLOSE_TIMEOUT_S) + + # ---- browser -> bus ---------------------------------------------------- + + def publish_packet(self, packet: bytes) -> None: + """Republish a browser-sent LCM small-message packet onto the bus.""" + if self._lcm is None: + return + decoded = decode_packet(packet) + if decoded is None: + logger.warning("lcm-ws bridge: dropping malformed packet from client") + return + channel, payload = decoded + self._lcm.publish(channel, payload) + self.published_from_clients += 1 + + # ---- endpoint ---------------------------------------------------------- + + async def websocket_endpoint(self, websocket: WebSocket) -> None: + await websocket.accept() + # The serving loop is whichever loop accepts clients; capture it + # here so the bridge needs no lifespan hook from the host app. + self._loop = asyncio.get_running_loop() + self._pending[websocket] = {} + self._wake[websocket] = asyncio.Event() + self._last_forward_ts[websocket] = {} + self._clients.add(websocket) + drain_task = asyncio.create_task(self._drain_loop(websocket)) + self._drain_tasks[websocket] = drain_task + logger.info("lcm-ws bridge: client connected", clients=len(self._clients)) + try: + while True: + packet = await websocket.receive_bytes() + self.publish_packet(packet) + except WebSocketDisconnect: + pass + except Exception: + logger.exception("lcm-ws bridge: receive failed") + finally: + self._clients.discard(websocket) + self._pending.pop(websocket, None) + self._wake.pop(websocket, None) + self._last_forward_ts.pop(websocket, None) + task = self._drain_tasks.pop(websocket, None) + if task is not None: + task.cancel() + with suppress(asyncio.CancelledError, Exception): + await task + logger.info("lcm-ws bridge: client disconnected", clients=len(self._clients)) diff --git a/dimos/web/lcm_bridge/module.py b/dimos/web/lcm_bridge/module.py new file mode 100644 index 0000000000..f7085da4b1 --- /dev/null +++ b/dimos/web/lcm_bridge/module.py @@ -0,0 +1,148 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Standalone server for the LCM <-> WebSocket bridge. + +Add it to any blueprint and every bus topic becomes reachable from a +browser tab on the same host/LAN: + + autoconnect( + my_robot_blueprint, + LcmWebSocketBridgeModule.blueprint(port=9669), + ) + +Browser side (see ``static/lcm_client.js``, served at ``/lcm_client.js``): + + + dimosLcm.subscribe("/odom", dimosMsgs.geometry_msgs.PoseStamped, cb) + dimosLcm.publish("/cmd_vel", new dimosMsgs.geometry_msgs.Twist(...)) + +``GET /`` serves a JSON status page (client count, forward/drop counters, +active filter config) for quick debugging. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +import threading +from typing import Any + +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import FileResponse, JSONResponse +from starlette.routing import Route +import uvicorn + +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.utils.logging_config import setup_logger +from dimos.web.lcm_bridge.bridge import LcmWebSocketBridge + +logger = setup_logger() + +_STATIC_DIR = Path(__file__).parent / "static" + + +class LcmWebSocketBridgeModule(Module): + """Serves :class:`LcmWebSocketBridge` on its own port. + + The module has no In/Out ports: it taps the LCM bus directly with a + subscribe-all handle, like the rerun bridge, so it composes into any + blueprint without wiring. Constructor arguments mirror the bridge's + (fnmatch patterns for filtering and rate caps); requires the ``web`` + extra (starlette + uvicorn). + """ + + def __init__( + self, + port: int = 9669, + host: str = "0.0.0.0", + channel_rate_hz: Mapping[str, float] | None = None, + topic_allowlist: list[str] | None = None, + topic_blocklist: list[str] | None = None, + lcm_url: str | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._port = port + self._host = host + self._bridge = LcmWebSocketBridge( + lcm_url=lcm_url, + channel_rate_hz=channel_rate_hz, + topic_allowlist=topic_allowlist, + topic_blocklist=topic_blocklist, + ) + self._uvicorn_server: uvicorn.Server | None = None + self._server_thread: threading.Thread | None = None + + @property + def bridge(self) -> LcmWebSocketBridge: + return self._bridge + + @property + def port(self) -> int: + return self._port + + @rpc + def start(self) -> None: + super().start() + self._bridge.start() + config = uvicorn.Config( + self._create_app(), host=self._host, port=self._port, log_level="warning" + ) + self._uvicorn_server = uvicorn.Server(config) + self._server_thread = threading.Thread( + target=self._uvicorn_server.run, + name="lcm-ws-bridge-server", + daemon=True, + ) + self._server_thread.start() + logger.info("LCM websocket bridge: ws://localhost:%s/lcm-ws", self._port) + + @rpc + def stop(self) -> None: + self._bridge.stop() + if self._uvicorn_server is not None: + self._uvicorn_server.should_exit = True + if self._server_thread is not None and self._server_thread.is_alive(): + self._server_thread.join(timeout=2.0) + super().stop() + + def _create_app(self) -> Starlette: + return Starlette( + routes=[ + Route("/", self._status), + Route("/lcm_client.js", self._client_js), + *self._bridge.routes(), + ] + ) + + async def _status(self, request: Request) -> JSONResponse: + bridge = self._bridge + return JSONResponse( + { + "clients": bridge.client_count, + "forwarded": bridge.forwarded, + "rate_capped": bridge.rate_capped, + "filtered": bridge.filtered, + "published_from_clients": bridge.published_from_clients, + "channel_rate_hz": bridge._channel_rate_hz, + "topic_allowlist": bridge._topic_allowlist, + "topic_blocklist": bridge._topic_blocklist, + } + ) + + async def _client_js(self, request: Request) -> FileResponse: + return FileResponse(_STATIC_DIR / "lcm_client.js", media_type="text/javascript") diff --git a/dimos/web/lcm_bridge/static/lcm_client.js b/dimos/web/lcm_bridge/static/lcm_client.js new file mode 100644 index 0000000000..7fbcffe2ad --- /dev/null +++ b/dimos/web/lcm_bridge/static/lcm_client.js @@ -0,0 +1,174 @@ +// Copyright 2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Browser-side LCM client. Talks to the LCM<->WebSocket bridge +// (dimos/web/lcm_bridge) using the same wire format the rest of the +// dimos bus uses, via @dimos/msgs. +// +// Pinned to specific versions so we don't silently inherit upstream +// schema/wire changes. + +import * as msgs from "https://esm.sh/jsr/@dimos/msgs@0.1.4"; +const { decode, decodeChannel, encodePacket } = msgs; + +const subscribers = new Map(); // channel string -> Set +const payloadSubscribers = new Map(); // channel string -> Set +let socket = null; +let reconnectTimer = null; +let bridgeUrl = null; +const listeners = new Set(); // status listeners + +function notifyStatus(status, ready) { + for (const cb of listeners) { + try { cb({ status, ready }); } catch (e) { console.error("[lcm] status listener", e); } + } +} + +function defaultBridgeUrl() { + const proto = window.location.protocol === "https:" ? "wss:" : "ws:"; + return `${proto}//${window.location.host}/lcm-ws`; +} + +function connect() { + if (reconnectTimer != null) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + bridgeUrl = bridgeUrl || defaultBridgeUrl(); + socket = new WebSocket(bridgeUrl); + socket.binaryType = "arraybuffer"; + + socket.onopen = () => notifyStatus("live", true); + socket.onerror = (e) => { + console.error("[lcm] socket error", e); + notifyStatus("error", false); + }; + socket.onclose = () => { + notifyStatus("reconnecting", false); + reconnectTimer = setTimeout(connect, 1000); + }; + socket.onmessage = (event) => { + if (!(event.data instanceof ArrayBuffer)) return; + let channel, payload; + try { + ({ channel, payload } = decodeChannel(new Uint8Array(event.data))); + } catch (err) { + console.error("[lcm] decode failed", err); + return; + } + + const subs = subscribers.get(channel); + if (subs && subs.size > 0) { + let data; + try { + data = decode(payload); + } catch (err) { + console.error(`[lcm] payload decode failed for ${channel}`, err); + return; + } + for (const cb of subs) { + try { cb(data, channel); } catch (e) { console.error(`[lcm] handler for ${channel}`, e); } + } + } + + const rawSubs = payloadSubscribers.get(channel); + if (!rawSubs || rawSubs.size === 0) return; + for (const cb of rawSubs) { + try { cb(payload, channel); } catch (e) { console.error(`[lcm] raw handler for ${channel}`, e); } + } + }; +} + +function channelOf(topic, msgClass) { + // Mirrors @dimos/lcm: actual LCM channel is "#" + return `${topic}#${msgClass._NAME}`; +} + +function subscribe(topic, msgClass, callback) { + const channel = channelOf(topic, msgClass); + let subs = subscribers.get(channel); + if (!subs) { + subs = new Set(); + subscribers.set(channel, subs); + } + subs.add(callback); + return () => { + const s = subscribers.get(channel); + if (s) { + s.delete(callback); + if (s.size === 0) subscribers.delete(channel); + } + }; +} + +function subscribePayload(topic, msgClass, callback) { + return subscribeChannel(channelOf(topic, msgClass), callback); +} + +// Raw-channel variant for dimos types @dimos/msgs can't decode. The +// callback receives the raw payload bytes. +function subscribeChannel(channel, callback) { + let subs = payloadSubscribers.get(channel); + if (!subs) { + subs = new Set(); + payloadSubscribers.set(channel, subs); + } + subs.add(callback); + return () => { + const s = payloadSubscribers.get(channel); + if (s) { + s.delete(callback); + if (s.size === 0) payloadSubscribers.delete(channel); + } + }; +} + +function publish(topic, message) { + const cls = message?.constructor; + if (!cls || !cls._NAME) { + console.error(`[lcm] publish: message has no _NAME`, message); + return false; + } + if (!socket || socket.readyState !== WebSocket.OPEN) { + console.warn(`[lcm] publish dropped (socket not open): ${topic}#${cls._NAME}`); + return false; + } + const channel = channelOf(topic, cls); + let packet; + try { + packet = encodePacket(channel, message); + } catch (err) { + console.error(`[lcm] encode failed for ${channel}`, err); + return false; + } + socket.send(packet); + return true; +} + +function onStatus(callback) { + listeners.add(callback); + return () => listeners.delete(callback); +} + +function start(url) { + if (url) bridgeUrl = url; + if (socket && socket.readyState === WebSocket.OPEN) return; + connect(); +} + +window.dimosMsgs = msgs; +window.dimosLcm = { subscribe, subscribePayload, subscribeChannel, publish, onStatus, start }; + +// Auto-start so host pages don't need to know about us. +start(); diff --git a/dimos/web/lcm_bridge/test_lcm_bridge.py b/dimos/web/lcm_bridge/test_lcm_bridge.py new file mode 100644 index 0000000000..c51f7b452c --- /dev/null +++ b/dimos/web/lcm_bridge/test_lcm_bridge.py @@ -0,0 +1,234 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the LCM <-> WebSocket bridge. + +The integration tests run the standalone module against a ``memq://`` +LCM provider (in-process queue, no multicast needed on CI) and a real +``websockets`` client, so both directions cross a real socket. +""" + +from __future__ import annotations + +import asyncio +import socket +import threading +import time +from typing import Any + +import pytest +import websockets.asyncio.client as ws_client + +from dimos.web.lcm_bridge.bridge import ( + LcmWebSocketBridge, + decode_packet, + encode_packet, +) +from dimos.web.lcm_bridge.module import LcmWebSocketBridgeModule + +_TEST_PORT = 13041 +_SERVER_STARTUP_TIMEOUT_S = 10.0 + + +# ---- wire format ---------------------------------------------------------- + + +def test_packet_roundtrip() -> None: + packet = encode_packet("/odom#geometry_msgs.PoseStamped", b"\x01\x02\x03", seq=7) + assert decode_packet(packet) == ("/odom#geometry_msgs.PoseStamped", b"\x01\x02\x03") + + +def test_decode_rejects_garbage() -> None: + assert decode_packet(b"") is None + assert decode_packet(b"\x00" * 8) is None # wrong magic + # Right magic but no channel terminator. + assert decode_packet(encode_packet("x", b"")[:-2]) is None + + +# ---- filtering ------------------------------------------------------------ + + +def _bridge(**kwargs: Any) -> LcmWebSocketBridge: + return LcmWebSocketBridge(lcm_url="memq://", **kwargs) + + +def test_no_filters_forwards_everything() -> None: + bridge = _bridge() + assert bridge._forward_allowed("/odom#geometry_msgs.PoseStamped") + + +def test_blocklist_matches_topic_and_channel() -> None: + bridge = _bridge(topic_blocklist=["/global_map", "/camera_*"]) + assert not bridge._forward_allowed("/global_map#sensor_msgs.PointCloud2") + assert not bridge._forward_allowed("/camera_image#sensor_msgs.Image") + assert bridge._forward_allowed("/odom#geometry_msgs.PoseStamped") + + +def test_allowlist_restricts_and_blocklist_wins() -> None: + bridge = _bridge(topic_allowlist=["/odom", "/tf"], topic_blocklist=["/tf"]) + assert bridge._forward_allowed("/odom#geometry_msgs.PoseStamped") + assert not bridge._forward_allowed("/tf#tf2_msgs.TFMessage") + assert not bridge._forward_allowed("/joint_state#sensor_msgs.JointState") + + +def test_rate_pattern_resolves_min_interval() -> None: + bridge = _bridge(channel_rate_hz={"/global_map": 2.0}) + assert bridge._min_interval_s("/global_map#sensor_msgs.PointCloud2") == pytest.approx(0.5) + assert bridge._min_interval_s("/odom#geometry_msgs.PoseStamped") == 0.0 + + +# ---- end to end ----------------------------------------------------------- + + +def _wait_for_port(port: int, timeout: float = _SERVER_STARTUP_TIMEOUT_S) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.2) + if sock.connect_ex(("127.0.0.1", port)) == 0: + return + time.sleep(0.05) + raise TimeoutError(f"server on port {port} did not come up") + + +@pytest.fixture() +def module() -> Any: + mod = LcmWebSocketBridgeModule( + port=_TEST_PORT, + host="127.0.0.1", + lcm_url="memq://", + channel_rate_hz={"/capped": 2.0}, + ) + mod.start() + _wait_for_port(_TEST_PORT) + yield mod + mod.stop() + + +class _WsClient: + """Sync facade over a websockets client on a private event loop.""" + + def __init__(self, url: str) -> None: + self._url = url + self._loop = asyncio.new_event_loop() + self._ws: Any = None + + def __enter__(self) -> _WsClient: + self._ws = self._loop.run_until_complete(ws_client.connect(self._url)) + return self + + def __exit__(self, *_: Any) -> None: + if self._ws is not None: + self._loop.run_until_complete(self._ws.close()) + self._loop.close() + + def send(self, data: bytes) -> None: + self._loop.run_until_complete(self._ws.send(data)) + + def recv(self, timeout: float = 3.0) -> bytes: + return self._loop.run_until_complete(asyncio.wait_for(self._ws.recv(), timeout)) + + def recv_channel(self, channel: str, timeout: float = 3.0) -> bytes: + """Receive until a packet for ``channel`` arrives; return its payload.""" + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"no packet on {channel}") + decoded = decode_packet(self.recv(timeout=remaining)) + if decoded is not None and decoded[0] == channel: + return decoded[1] + + def drain(self, window_s: float) -> list[bytes]: + """Collect every packet arriving within ``window_s``.""" + got: list[bytes] = [] + deadline = time.monotonic() + window_s + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return got + try: + got.append(self.recv(timeout=remaining)) + except TimeoutError: + return got + + +@pytest.fixture() +def client(module: LcmWebSocketBridgeModule) -> Any: + with _WsClient(f"ws://127.0.0.1:{_TEST_PORT}/lcm-ws") as ws: + # The bridge registers the client on accept; give the server loop + # a beat so bus messages published next actually fan out to it. + deadline = time.monotonic() + 2.0 + while module.bridge.client_count == 0 and time.monotonic() < deadline: + time.sleep(0.02) + assert module.bridge.client_count == 1 + yield ws + + +def test_bus_to_browser(module: LcmWebSocketBridgeModule, client: _WsClient) -> None: + """A bus publish arrives at the WebSocket client as an LC02 packet.""" + channel = "/test_topic#test_msgs.Blob" + assert module.bridge.lcm is not None + module.bridge.lcm.publish(channel, b"payload-bytes") + assert client.recv_channel(channel) == b"payload-bytes" + + +def test_browser_to_bus_and_multitab_echo( + module: LcmWebSocketBridgeModule, client: _WsClient +) -> None: + """A client publish lands on the bus, and the bridge's subscribe-all + fans it back out to connected clients (multi-tab sync).""" + received: list[tuple[str, bytes]] = [] + got_one = threading.Event() + assert module.bridge.lcm is not None + module.bridge.lcm.subscribe( + "/from_browser.*", lambda ch, data: (received.append((ch, data)), got_one.set()) + ) + + channel = "/from_browser#test_msgs.Blob" + client.send(encode_packet(channel, b"hello-from-tab")) + + assert got_one.wait(timeout=3.0), "browser publish never reached the bus" + assert received[0] == (channel, b"hello-from-tab") + assert client.recv_channel(channel) == b"hello-from-tab" + + +def test_rate_cap_drops_bursts(module: LcmWebSocketBridgeModule, client: _WsClient) -> None: + """A channel capped at 2 Hz forwards ~1 packet from a rapid burst.""" + channel = "/capped#test_msgs.Blob" + assert module.bridge.lcm is not None + for i in range(10): + module.bridge.lcm.publish(channel, bytes([i])) + packets = client.drain(window_s=0.4) + assert 1 <= len(packets) <= 2, f"expected the burst capped to 1-2 packets, got {len(packets)}" + + +def test_status_endpoint(module: LcmWebSocketBridgeModule) -> None: + import urllib.request + + with urllib.request.urlopen(f"http://127.0.0.1:{_TEST_PORT}/", timeout=3) as response: + assert response.status == 200 + body = response.read().decode() + assert "clients" in body and "forwarded" in body + + +def test_serves_browser_client(module: LcmWebSocketBridgeModule) -> None: + import urllib.request + + with urllib.request.urlopen( + f"http://127.0.0.1:{_TEST_PORT}/lcm_client.js", timeout=3 + ) as response: + assert response.status == 200 + body = response.read().decode() + assert "dimosLcm" in body diff --git a/dimos/web/viewer/README.md b/dimos/web/viewer/README.md new file mode 100644 index 0000000000..1d95575bb9 --- /dev/null +++ b/dimos/web/viewer/README.md @@ -0,0 +1,71 @@ +# DimOS Operator Console (Babylon viewer) + +Browser-based 3D viewer for any dimos system: scene mesh and/or gaussian +splat, the robot (FK from `joint_state` + `odom`), pointclouds, nav path, +camera frames, teleop (`/cmd_vel`) and click-to-navigate — in one page, +with no client install. + +This is the viewer-only build of the simulation scene viewer that has been +driving the sim work (and has been teleoperated across the internet). It +**renders, it never owns state**: every display topic and every +browser-authored message flows through the LCM<->WebSocket bridge +([`dimos/web/lcm_bridge`](../lcm_bridge/README.md)), so the browser is just +another peer on the bus. The browser-physics / entity-authority half of +the original module stays in the simulation tree. + +## Usage + +```python +from dimos.web.viewer.module import BabylonViewerModule + +autoconnect( + my_robot_blueprint, + BabylonViewerModule.blueprint( + mjcf_path=..., # optional: robot meshes + server-side FK + assets=..., # optional: {filename: bytes} for MJCF meshes + scene_path=..., # optional: visual scene .glb/.gltf + splat_path=..., # optional: gaussian splat .ply + ), +) +``` + +Open `http://:8091/`. Everything is optional — with no arguments you +get pointclouds, path, camera and teleop over a grid floor. + +What the page shows, and where it comes from: + +| Layer | Source | +| ---------------- | --------------------------------------------- | +| robot meshes | `/robot.json` (parsed from the MJCF once) | +| robot pose | server-side FK -> binary frames on `/ws` | +| scene / splat | `/assets/…` (content-hash versioned) | +| pointclouds | `/global_map` etc. via `/lcm-ws`, web worker | +| camera | JPEG `Image` packets via `/lcm-ws` | +| nav path, goals | `/nav_path`, `point_goal` via `/lcm-ws` | +| teleop | browser publishes `/cmd_vel` via `/lcm-ws` | + +`mujoco` (the `sim` extra) is only imported when `mjcf_path` is given — +the robot layer parses the same MJCF the simulator steps, so sim users +get a pixel-exact mirror for free. A URDF/pinocchio loader shared with +the rerun viewer is the natural follow-up for mujoco-free robot display. + +## Try it against the G1 sim + +```bash +dimos --transport lcm --simulation mujoco --scene-package dimos_office \ + run unitree-g1-groot-wbc +``` + +The console comes up at `:8091` next to the sim: the G1 mirrored from +live coordinator state inside the office scene. Enable Drive + WASD +walks it; the Lidar layer streams `/global_map`. `--transport lcm` +matters: display topics and teleop flow through the LCM bridge, which +does not yet tap the Zenoh bus (see the lcm_bridge README). + +## Notes + +- `DIMOS_BABYLON_OPEN=0` disables the auto-opened tab. +- `DIMOS_VIEWER_GLOBAL_MAP_HZ` (default 1.0) rate-caps the voxel map on + the websocket leg only; bus consumers are unaffected. +- The HUD's respawn / arm-slider / policy buttons are wired to the + simulation tree's authority module; in this build they are inert. diff --git a/dimos/web/viewer/geometry.py b/dimos/web/viewer/geometry.py new file mode 100644 index 0000000000..4265674efc --- /dev/null +++ b/dimos/web/viewer/geometry.py @@ -0,0 +1,77 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import mimetypes +from pathlib import Path + +import numpy as np +from scipy.spatial.transform import Rotation + +WS_MSG_CAMERA = 0x01 + + +def dimos_joint_to_mjcf(name: str) -> str: + parts = name.split("/", 1) + suffix = parts[1] if len(parts) > 1 else parts[0] + if suffix.endswith("_joint"): + return suffix + return f"{suffix}_joint" + + +def canonical_joint_name(name: str) -> str: + if "/" in name: + name = name.split("/", 1)[1] + if name.endswith("_joint"): + name = name[: -len("_joint")] + return name + + +def compose_scene_mesh_wxyz( + *, y_up: bool, rotation_zyx_deg: tuple[float, float, float] +) -> tuple[float, float, float, float]: + matrix = np.eye(3, dtype=np.float64) + if y_up: + matrix = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]], dtype=np.float64) + + rz, ry, rx = (np.deg2rad(angle) for angle in rotation_zyx_deg) + cz, sz = np.cos(rz), np.sin(rz) + cy, sy = np.cos(ry), np.sin(ry) + cx, sx = np.cos(rx), np.sin(rx) + rotate_z = np.array([[cz, -sz, 0], [sz, cz, 0], [0, 0, 1]], dtype=np.float64) + rotate_y = np.array([[cy, 0, sy], [0, 1, 0], [-sy, 0, cy]], dtype=np.float64) + rotate_x = np.array([[1, 0, 0], [0, cx, -sx], [0, sx, cx]], dtype=np.float64) + matrix = rotate_z @ rotate_y @ rotate_x @ matrix + + x, y, z, w = Rotation.from_matrix(matrix).as_quat() + return (float(w), float(x), float(y), float(z)) + + +def path_contains(parent: Path, child: Path) -> bool: + try: + child.resolve().relative_to(parent.resolve()) + except ValueError: + return False + return True + + +def media_type(path: Path) -> str | None: + match path.suffix.lower(): + case ".glb": + return "model/gltf-binary" + case ".gltf": + return "model/gltf+json" + case _: + return mimetypes.guess_type(path.name)[0] diff --git a/dimos/web/viewer/module.py b/dimos/web/viewer/module.py new file mode 100644 index 0000000000..ecb9b595c8 --- /dev/null +++ b/dimos/web/viewer/module.py @@ -0,0 +1,553 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Babylon-based browser viewer for any dimos system. + +Viewer-only cut of the simulation scene viewer: it renders, it never owns +state. The browser page (Babylon.js) draws a scene mesh and/or gaussian +splat, the robot (server-side FK from ``joint_state`` + ``odom``), +pointclouds, the nav path and camera frames, and offers teleop +(``/cmd_vel``) and click-to-navigate (``point_goal``) — every display +topic and browser-authored message flows through the LCM<->WebSocket +bridge (:mod:`dimos.web.lcm_bridge`), so the browser is just another bus +peer. The browser-physics / entity-authority half of the original module +stayed in the simulation tree; this one composes with real robots and +sims alike. + +Add to any blueprint:: + + BabylonViewerModule.blueprint( + mjcf_path=..., # optional: robot meshes + FK + scene_path=..., # optional: visual scene GLB/glTF + splat_path=..., # optional: gaussian splat + ) + +Then open ``http://:8091/``. All three arguments are optional — +with none you still get pointclouds, path, teleop and camera over a +grid floor. +""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager, suppress +import os +from pathlib import Path +import struct +import threading +import time +from typing import Any + +import numpy as np +from reactivex.disposable import Disposable +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import FileResponse, HTMLResponse, JSONResponse, Response +from starlette.routing import Mount, Route, WebSocketRoute +from starlette.staticfiles import StaticFiles +from starlette.websockets import WebSocket, WebSocketDisconnect +import uvicorn + +from dimos.core.core import rpc +from dimos.core.module import Module +from dimos.core.stream import In +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.utils.logging_config import setup_logger +from dimos.web.lcm_bridge import bridge as lcm_bridge +from dimos.web.viewer.geometry import ( + compose_scene_mesh_wxyz, + dimos_joint_to_mjcf, + media_type, + path_contains, +) + +logger = setup_logger() + +STATIC_DIR = Path(__file__).with_name("static") +_LCM_CLIENT_PATH = Path(lcm_bridge.__file__).with_name("static") / "lcm_client.js" + +_DEFAULT_BROADCAST_HZ = 30.0 +_DEFAULT_PORT = 8091 +_POSE_POSITION_EPSILON = 1e-5 +_POSE_QUATERNION_EPSILON = 1e-5 +# Binary websocket message tag. Robot body poses are the only binary frame +# left on /ws — everything else (pointclouds, camera JPEGs, path, teleop) +# flows through the LCM<->WS bridge. Once the browser does its own FK from +# /joint_state + /odom this goes too. +_WS_MSG_ROBOT_POSE = 0x03 +_WS_ROBOT_POSE_HEADER_BYTES = 16 +# Per-message WS send timeout. The browser tab momentarily failing to drain +# TCP (Chrome backgrounding, GC pause, App Nap) shouldn't permanently wedge +# the in-flight gate: time out, drop the client, let the JS reconnect. +_WS_SEND_TIMEOUT_S = 1.0 +_WS_CLOSE_TIMEOUT_S = 0.5 +_NO_CACHE_HEADERS = { + "Cache-Control": "no-store, no-cache, must-revalidate, max-age=0", + "Pragma": "no-cache", + "Expires": "0", +} + + +def _asset_token(path: Path) -> str: + stat = path.stat() + return f"{stat.st_mtime_ns:x}-{stat.st_size:x}" + + +def _versioned_asset_name(prefix: str, path: Path) -> str: + return f"{prefix}-{_asset_token(path)}{path.suffix.lower()}" + + +def _legacy_asset_name(prefix: str, path: Path) -> str: + return f"{prefix}{path.suffix.lower()}" + + +def _matches_asset_name(asset_name: str, prefix: str, path: Path) -> bool: + suffix = path.suffix.lower() + return asset_name == _legacy_asset_name(prefix, path) or ( + asset_name.startswith(f"{prefix}-") and asset_name.endswith(suffix) + ) + + +class BabylonViewerModule(Module): + # joint_state and odom are consumed server-side because forward + # kinematics runs here (-> _make_robot_pose_payload). The browser also + # sees both via /lcm-ws; once FK moves to the browser these go away. + joint_state: In[JointState] + odom: In[PoseStamped] + + def __init__( + self, + mjcf_path: str | Path | None = None, + *, + port: int = _DEFAULT_PORT, + assets: dict[str, bytes] | None = None, + mesh_dir: str | Path | None = None, + scene_path: str | Path | None = None, + splat_path: str | Path | None = None, + splat_alignment: dict[str, Any] | None = None, + scene_scale: float = 1.0, + scene_translation: tuple[float, float, float] = (0.0, 0.0, 0.0), + scene_rotation_zyx_deg: tuple[float, float, float] = (0.0, 0.0, 0.0), + scene_y_up: bool = True, + broadcast_hz: float = _DEFAULT_BROADCAST_HZ, + channel_rate_hz: dict[str, float] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + self._mjcf_path = Path(mjcf_path) if mjcf_path else None + self._assets = assets + # Deferred alternative to `assets`: a directory of mesh files read + # at start() so blueprint import stays cheap (and LFS-backed dirs + # download lazily). + self._mesh_dir = mesh_dir + self._port = port + self._scene_path = Path(scene_path) if scene_path else None + self._splat_path = Path(splat_path) if splat_path else None + self._splat_alignment = splat_alignment or {} + self._scene_scale = scene_scale + self._scene_translation = scene_translation + self._scene_rotation_zyx_deg = scene_rotation_zyx_deg + self._scene_y_up = scene_y_up + self._broadcast_dt = 1.0 / float(broadcast_hz) + + self._state_lock = threading.Lock() + self._latest_joints: dict[str, float] = {} + self._latest_base_pos: np.ndarray | None = None + self._latest_base_wxyz: np.ndarray | None = None + self._robot_pose_lock = threading.Lock() + self._last_robot_pose_values: np.ndarray | None = None + + self._robot: Any = None + self._uvicorn_server: uvicorn.Server | None = None + self._server_thread: threading.Thread | None = None + self._broadcast_thread: threading.Thread | None = None + self._stop_event = threading.Event() + self._server_loop: asyncio.AbstractEventLoop | None = None + self._clients: set[WebSocket] = set() + # Per-client send lock. Starlette's WebSocket is not safe for + # concurrent sends; without serialization the broadcast loop and + # the initial replay in _websocket can race on the same socket. + self._ws_send_locks: dict[WebSocket, asyncio.Lock] = {} + + # The bus-to-browser data plane. Default rate cap keeps the rust + # voxel mapper's multi-MB /global_map from eating the tab's budget; + # in-process consumers are unaffected. + _gm_hz = float(os.environ.get("DIMOS_VIEWER_GLOBAL_MAP_HZ", "1.0")) + rates: dict[str, float] = {} + if _gm_hz > 0: + rates["/global_map#sensor_msgs.PointCloud2"] = _gm_hz + rates.update(channel_rate_hz or {}) + self._ws_bridge = lcm_bridge.LcmWebSocketBridge(channel_rate_hz=rates) + + @rpc + def start(self) -> None: + super().start() + + if self._mjcf_path is not None: + # mujoco is only needed for the robot layer (MJCF parse + FK); + # a robotless viewer must not require the sim extra. + from dimos.web.viewer.robot_meshes import load_robot_meshes + + assets = self._assets + if assets is None and self._mesh_dir is not None: + assets = { + mesh.name: mesh.read_bytes() + for mesh in Path(self._mesh_dir).iterdir() + # skip .DS_Store / AppleDouble "._*" junk that rides + # along in LFS-extracted asset dirs + if mesh.is_file() and not mesh.name.startswith(".") + } + self._robot = load_robot_meshes(self._mjcf_path, assets=assets) + + config = uvicorn.Config( + self._create_app(), host="0.0.0.0", port=self._port, log_level="warning" + ) + self._uvicorn_server = uvicorn.Server(config) + self._server_thread = threading.Thread( + target=self._uvicorn_server.run, + name="babylon-viewer-server", + daemon=True, + ) + self._server_thread.start() + + # In a blueprint these ports arrive wired; standalone (tests, bare + # scripts) they may not be — serve anyway, the robot just holds its + # default pose until someone wires or publishes. + for port, callback, name in ( + (self.joint_state, self._on_joint_state, "joint_state"), + (self.odom, self._on_odom, "odom"), + ): + if getattr(port, "transport", None) is None: + logger.warning("BabylonViewer: %s port not wired; robot pose will be static", name) + continue + self.register_disposable(Disposable(port.subscribe(callback))) + + self._broadcast_thread = threading.Thread( + target=self._broadcast_loop, + name="babylon-viewer-broadcast", + daemon=True, + ) + self._broadcast_thread.start() + + self._ws_bridge.start() + + logger.info("Babylon viewer: http://localhost:%s/", self._port) + if os.environ.get("DIMOS_BABYLON_OPEN", "1").lower() not in ("0", "false", "no"): + import webbrowser + + try: + webbrowser.open(f"http://localhost:{self._port}/") + except Exception: # headless environments have no opener + pass + + @rpc + def stop(self) -> None: + self._stop_event.set() + self._ws_bridge.stop() + if self._uvicorn_server is not None: + self._uvicorn_server.should_exit = True + if self._broadcast_thread and self._broadcast_thread.is_alive(): + self._broadcast_thread.join(timeout=2.0) + if self._server_thread and self._server_thread.is_alive(): + self._server_thread.join(timeout=2.0) + super().stop() + + # ---- HTTP surface ---------------------------------------------------- + + def _create_app(self) -> Starlette: + @asynccontextmanager + async def _lifespan(_app: Starlette) -> Any: + self._server_loop = asyncio.get_running_loop() + yield + + return Starlette( + routes=[ + Route("/", self._index), + Route("/config.json", self._config), + Route("/robot.json", self._robot_json), + Route("/arms.json", self._arms_json), + Route("/viewer_debug", self._viewer_debug, methods=["POST"]), + Route("/assets/{asset_name:path}", self._asset), + # The browser client ships with the bridge package; serve it + # under /static so index.html needs no special-casing. Listed + # before the Mount so it wins the match. + Route("/static/lcm_client.js", self._lcm_client_js), + Mount("/static", app=StaticFiles(directory=STATIC_DIR), name="static"), + WebSocketRoute("/ws", self._websocket), + *self._ws_bridge.routes(), + ], + lifespan=_lifespan, + ) + + async def _lcm_client_js(self, request: Request) -> FileResponse: + return FileResponse( + _LCM_CLIENT_PATH, media_type="text/javascript", headers=_NO_CACHE_HEADERS + ) + + async def _index(self, request: Request) -> HTMLResponse: + html = (STATIC_DIR / "index.html").read_text() + for asset_name in ("style.css", "ui.js", "app.js"): + asset_path = STATIC_DIR / asset_name + if not asset_path.exists(): + continue + token = _asset_token(asset_path) + html = html.replace( + f'"/static/{asset_name}"', + f'"/static/{asset_name}?v={token}"', + ) + return HTMLResponse(html, headers=_NO_CACHE_HEADERS) + + async def _config(self, request: Request) -> JSONResponse: + scene_file = None + scene_bytes = 0 + scene_wxyz = compose_scene_mesh_wxyz( + y_up=self._scene_y_up, + rotation_zyx_deg=self._scene_rotation_zyx_deg, + ) + if self._scene_path is not None and self._scene_path.exists(): + scene_file = _versioned_asset_name("scene", self._scene_path) + scene_bytes = self._scene_path.stat().st_size + splat_file = None + splat_bytes = 0 + if self._splat_path is not None and self._splat_path.exists(): + splat_file = _versioned_asset_name("splat", self._splat_path) + splat_bytes = self._splat_path.stat().st_size + # Exactly the keys the page reads — this build renders, it never + # simulates, so there is no browser-physics block. + return JSONResponse( + { + "sceneFile": scene_file, + "sceneBytes": scene_bytes, + "splatFile": splat_file, + "splatBytes": splat_bytes, + "splatAlignment": self._splat_alignment, + "sceneScale": self._scene_scale, + "scenePosition": list(self._scene_translation), + "sceneWxyz": list(scene_wxyz), + # "external" = any entity authority lives outside the browser; + # the page's ENTITY MIRROR stub keys off this. + "entityAuthority": "external", + }, + headers=_NO_CACHE_HEADERS, + ) + + async def _arms_json(self, request: Request) -> JSONResponse: + # Arm-slider RPC surface is a sim-tree feature; the viewer-only + # build reports no controllable joints so the HUD hides the panel. + return JSONResponse({"joints": []}, headers=_NO_CACHE_HEADERS) + + async def _viewer_debug(self, request: Request) -> JSONResponse: + try: + message = await request.json() + except Exception: + return JSONResponse({"ok": False}, status_code=400, headers=_NO_CACHE_HEADERS) + label = message.get("label", "viewer") + payload = message.get("payload") + if isinstance(payload, dict): + # Client-controlled keys must not be splatted into structlog + # kwargs (an "event" key would collide with the message itself). + logger.info("BabylonViewer debug", label=label, payload=payload) + return JSONResponse({"ok": True}, headers=_NO_CACHE_HEADERS) + + async def _robot_json(self, request: Request) -> JSONResponse: + robot = self._robot + if robot is None: + return JSONResponse({"bodyNames": [], "geoms": []}, headers=_NO_CACHE_HEADERS) + + geoms: list[dict[str, Any]] = [] + for index, geom in enumerate(robot.geoms): + geoms.append( + { + "id": index, + "body": geom.body_name, + "vertices": geom.vertices.astype(np.float32).reshape(-1).tolist(), + "indices": geom.faces.astype(np.int32).reshape(-1).tolist(), + "position": geom.local_pos.astype(np.float32).tolist(), + "wxyz": geom.local_wxyz.astype(np.float32).tolist(), + "rgba": [float(value) for value in geom.rgba], + } + ) + return JSONResponse( + {"bodyNames": robot.body_names, "geoms": geoms}, + headers=_NO_CACHE_HEADERS, + ) + + async def _asset(self, request: Request) -> Response: + asset_name = request.path_params["asset_name"] + + if self._splat_path is not None and _matches_asset_name( + asset_name, "splat", self._splat_path + ): + return FileResponse( + self._splat_path, + media_type="application/octet-stream", + headers=_NO_CACHE_HEADERS, + ) + + if self._scene_path is not None and _matches_asset_name( + asset_name, "scene", self._scene_path + ): + return FileResponse( + self._scene_path, + media_type=media_type(self._scene_path), + headers=_NO_CACHE_HEADERS, + ) + + # .gltf scenes reference sibling files (textures, .bin buffers). + if self._scene_path is not None and self._scene_path.suffix.lower() == ".gltf": + candidate = self._scene_path.parent / asset_name + if path_contains(self._scene_path.parent, candidate) and candidate.exists(): + return FileResponse( + candidate, + media_type=media_type(candidate), + headers=_NO_CACHE_HEADERS, + ) + + return Response("asset not found", status_code=404) + + # ---- /ws: robot pose broadcast + JSON control ------------------------ + + async def _websocket(self, websocket: WebSocket) -> None: + # Display topics and browser-authored messages (cmd_vel, point_goal, + # clicked_point) flow through /lcm-ws. /ws carries the robot_pose + # binary frame plus a JSON control channel the viewer-only build + # mostly ignores. + await websocket.accept() + self._ws_send_locks[websocket] = asyncio.Lock() + self._clients.add(websocket) + logger.info("BabylonViewer: websocket connected", clients=len(self._clients)) + try: + robot_pose_payload = self._make_robot_pose_payload(force=True) + if robot_pose_payload is not None: + await self._send_bytes_locked(websocket, robot_pose_payload) + while True: + message = await websocket.receive_json() + self._handle_client_message(message) + except WebSocketDisconnect: + pass + finally: + self._clients.discard(websocket) + self._ws_send_locks.pop(websocket, None) + logger.info("BabylonViewer: websocket disconnected", clients=len(self._clients)) + + def _handle_client_message(self, message: dict[str, Any]) -> None: + message_type = message.get("type") + # Respawn / arm / coordinator RPCs belong to the simulation tree's + # authority build. Log-and-ignore keeps old HUD buttons harmless. + logger.debug("BabylonViewer: ignoring control message", type=message_type) + + # ---- server-side FK -------------------------------------------------- + + def _broadcast_loop(self) -> None: + while not self._stop_event.is_set(): + loop = self._server_loop + if loop is not None and self._clients: + robot_pose_payload = self._make_robot_pose_payload() + if robot_pose_payload is not None: + asyncio.run_coroutine_threadsafe( + self._broadcast_robot_pose(robot_pose_payload), + loop, + ) + time.sleep(self._broadcast_dt) + + async def _broadcast_robot_pose(self, robot_pose_payload: bytes) -> None: + dead: list[WebSocket] = [] + for websocket in tuple(self._clients): + try: + await self._send_bytes_locked(websocket, robot_pose_payload) + except Exception: + dead.append(websocket) + await self._drop_clients(dead) + + def _make_robot_pose_payload(self, *, force: bool = False) -> bytes | None: + robot = self._robot + if robot is None: + return None + from dimos.web.viewer.robot_meshes import apply_state + + with self._state_lock: + joints = dict(self._latest_joints) + base_pos = None if self._latest_base_pos is None else self._latest_base_pos.copy() + base_wxyz = None if self._latest_base_wxyz is None else self._latest_base_wxyz.copy() + + with self._robot_pose_lock: + apply_state( + robot, + base_pos=base_pos, + base_wxyz=base_wxyz, + joint_positions=joints, + ) + + body_count = len(robot.body_names) + poses = np.empty((body_count, 7), dtype=np.float32) + poses[:, 0:3] = robot.data.xpos[:body_count].astype(np.float32, copy=False) + poses[:, 3:7] = robot.data.xquat[:body_count].astype(np.float32, copy=False) + + previous = self._last_robot_pose_values + if not force and previous is not None and previous.shape == poses.shape: + position_delta = np.max(np.abs(poses[:, 0:3] - previous[:, 0:3])) + quaternion_delta = np.max(np.abs(poses[:, 3:7] - previous[:, 3:7])) + if ( + position_delta <= _POSE_POSITION_EPSILON + and quaternion_delta <= _POSE_QUATERNION_EPSILON + ): + return None + + self._last_robot_pose_values = poses.copy() + + header = struct.pack( + ">B3xId", + _WS_MSG_ROBOT_POSE, + body_count, + time.time(), + ) + assert len(header) == _WS_ROBOT_POSE_HEADER_BYTES + return header + np.ascontiguousarray(poses).tobytes() + + def _on_joint_state(self, msg: JointState) -> None: + with self._state_lock: + self._latest_joints = { + dimos_joint_to_mjcf(name): float(position) + for name, position in zip(msg.name, msg.position, strict=False) + } + + def _on_odom(self, msg: PoseStamped) -> None: + with self._state_lock: + self._latest_base_pos = np.array([msg.x, msg.y, msg.z], dtype=np.float64) + self._latest_base_wxyz = np.array( + [ + msg.orientation.w, + msg.orientation.x, + msg.orientation.y, + msg.orientation.z, + ], + dtype=np.float64, + ) + + # ---- send helpers ------------------------------------------------------ + + async def _send_bytes_locked(self, websocket: WebSocket, payload: bytes) -> None: + lock = self._ws_send_locks.get(websocket) + if lock is None: + raise RuntimeError("WebSocket no longer registered") + async with lock: + await asyncio.wait_for(websocket.send_bytes(payload), timeout=_WS_SEND_TIMEOUT_S) + + async def _drop_clients(self, dead: list[WebSocket]) -> None: + for websocket in dead: + self._clients.discard(websocket) + self._ws_send_locks.pop(websocket, None) + with suppress(Exception): + await asyncio.wait_for(websocket.close(), timeout=_WS_CLOSE_TIMEOUT_S) diff --git a/dimos/web/viewer/robot_meshes.py b/dimos/web/viewer/robot_meshes.py new file mode 100644 index 0000000000..2612dcffb9 --- /dev/null +++ b/dimos/web/viewer/robot_meshes.py @@ -0,0 +1,227 @@ +# Copyright 2025-2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""MuJoCo-based robot mesh extractor and FK helper for browser viewers. + +Loads an MJCF (the same one the sim subprocess loads), pulls out visual +mesh geoms with their parent-body indices and local poses, and runs FK +on demand to give world poses for every body in the model. + +Lets render modules display the same robot the simulation is stepping, +without forcing a separate URDF or duplicating geometry. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +import mujoco +import numpy as np + +if TYPE_CHECKING: + from pathlib import Path + + +# dimos joint names look like ``g1/left_hip_pitch`` (slash-separated +# hardware id + snake_case suffix — the canonical convention since +# Mustafa's #1954 G1 coordinator integration unified naming with the +# Unitree SDK). MJCF joint names look like ``left_hip_pitch_joint``. +# Strip the hardware prefix and append ``_joint``. +def dimos_joint_to_mjcf(name: str) -> str: + parts = name.split("/", 1) + suffix = parts[1] if len(parts) > 1 else parts[0] + if suffix.endswith("_joint"): + return suffix + return f"{suffix}_joint" + + +@dataclass +class GeomInstance: + """A single visual mesh geom, parented to a body.""" + + body_name: str + vertices: np.ndarray # (V, 3) + faces: np.ndarray # (F, 3) + local_pos: np.ndarray # (3,) + local_wxyz: np.ndarray # (4,) + rgba: tuple[float, float, float, float] + + +@dataclass +class RobotMeshes: + model: mujoco.MjModel + data: mujoco.MjData + geoms: list[GeomInstance] + # joint-name (in MJCF order) -> qpos address. Used to splice incoming + # joint_state values into the right slots of qpos. + qpos_addr_by_mjcf_name: dict[str, int] + # body_id -> body name for browser entity paths. + body_names: list[str] + + +def load_robot_meshes( + mjcf_path: str | Path, + *, + visual_groups: tuple[int, ...] = (0, 1, 2), + assets: dict[str, bytes] | None = None, +) -> RobotMeshes: + """Parse the MJCF, pull visual mesh geoms into Python arrays. + + ``visual_groups`` defaults to MuJoCo's convention where group 0-2 are + visual and group 3+ are collision. Most menagerie / dimos models + follow this; if a model uses different groups, override it. + + ``assets`` is an optional ``{filename: bytes}`` map for mesh files + referenced by bare name in the MJCF (e.g. menagerie meshes). + Pass ``dimos.simulation.backend.mujoco.assets.get_assets()`` for G1. + When omitted, meshes are resolved from disk relative to ``mjcf_path`` + (the MJCF's own ``meshdir`` attribute, if present, applies normally). + """ + if assets is None: + # Disk-based: mujoco resolves relative to the + # MJCF's meshdir. Works for any robot that ships meshes on disk. + model = mujoco.MjModel.from_xml_path(str(mjcf_path)) + else: + with open(mjcf_path) as f: + xml = f.read() + model = mujoco.MjModel.from_xml_string(xml, assets) + data = mujoco.MjData(model) + + geoms: list[GeomInstance] = [] + for gid in range(model.ngeom): + if int(model.geom_group[gid]) not in visual_groups: + continue + gtype = int(model.geom_type[gid]) + body_id = int(model.geom_bodyid[gid]) + body_name = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, body_id) or f"body_{body_id}" + rgba = tuple(float(x) for x in model.geom_rgba[gid]) + local_pos = np.array(model.geom_pos[gid], dtype=np.float32).copy() + local_wxyz = np.array(model.geom_quat[gid], dtype=np.float32).copy() + size = model.geom_size[gid] + + # Mesh: pull vertices/faces from the model. + if gtype == mujoco.mjtGeom.mjGEOM_MESH: + mesh_id = int(model.geom_dataid[gid]) + if mesh_id < 0: + continue + v_start = int(model.mesh_vertadr[mesh_id]) + v_count = int(model.mesh_vertnum[mesh_id]) + f_start = int(model.mesh_faceadr[mesh_id]) + f_count = int(model.mesh_facenum[mesh_id]) + vertices = np.array( + model.mesh_vert[v_start : v_start + v_count], + dtype=np.float32, + ).copy() + faces = np.array( + model.mesh_face[f_start : f_start + f_count], + dtype=np.int32, + ).copy() + # Box: tessellate as 8 verts + 12 triangles, half-sizes from + # geom_size[0..2]. Lets us render primitives + # (manip_table, manip_cube, scene-editor exports) without a + # mesh asset. + elif gtype == mujoco.mjtGeom.mjGEOM_BOX: + hx, hy, hz = float(size[0]), float(size[1]), float(size[2]) + vertices = np.array( + [ + [-hx, -hy, -hz], + [hx, -hy, -hz], + [hx, hy, -hz], + [-hx, hy, -hz], + [-hx, -hy, hz], + [hx, -hy, hz], + [hx, hy, hz], + [-hx, hy, hz], + ], + dtype=np.float32, + ) + # Outward-facing CCW triangles (verified by cross-product). + faces = np.array( + [ + [0, 2, 1], + [0, 3, 2], # -Z (bottom) + [4, 5, 6], + [4, 6, 7], # +Z (top) + [0, 1, 5], + [0, 5, 4], # -Y + [1, 2, 6], + [1, 6, 5], # +X + [2, 3, 7], + [2, 7, 6], # +Y + [3, 0, 4], + [3, 4, 7], # -X + ], + dtype=np.int32, + ) + else: + # Sphere, cylinder, plane, etc. — skip for now. Only manip + # rigs and scene-editor exports use boxes; everything else + # the dimos sims care about is a mesh. + continue + + geoms.append( + GeomInstance( + body_name=body_name, + vertices=vertices, + faces=faces, + local_pos=local_pos, + local_wxyz=local_wxyz, + rgba=rgba, # type: ignore[arg-type] + ) + ) + + qpos_addr_by_mjcf_name: dict[str, int] = {} + for jid in range(model.njnt): + jname = mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_JOINT, jid) + if jname is None: + continue + qpos_addr_by_mjcf_name[jname] = int(model.jnt_qposadr[jid]) + + body_names = [ + mujoco.mj_id2name(model, mujoco.mjtObj.mjOBJ_BODY, bid) or f"body_{bid}" + for bid in range(model.nbody) + ] + + return RobotMeshes( + model=model, + data=data, + geoms=geoms, + qpos_addr_by_mjcf_name=qpos_addr_by_mjcf_name, + body_names=body_names, + ) + + +def apply_state( + rm: RobotMeshes, + *, + base_pos: np.ndarray | None = None, + base_wxyz: np.ndarray | None = None, + joint_positions: dict[str, float] | None = None, +) -> None: + """Splice base + joint values into qpos and run FK in-place. + + After this returns, ``rm.data.xpos`` and ``rm.data.xquat`` hold each + body's world pose. Cheap (~50 us for G1). + """ + if base_pos is not None: + rm.data.qpos[0:3] = base_pos + if base_wxyz is not None: + rm.data.qpos[3:7] = base_wxyz + if joint_positions: + for name, q in joint_positions.items(): + adr = rm.qpos_addr_by_mjcf_name.get(name) + if adr is not None: + rm.data.qpos[adr] = q + mujoco.mj_kinematics(rm.model, rm.data) diff --git a/dimos/web/viewer/static/app.js b/dimos/web/viewer/static/app.js new file mode 100644 index 0000000000..81c8e375a2 --- /dev/null +++ b/dimos/web/viewer/static/app.js @@ -0,0 +1,1829 @@ +// Copyright 2025-2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const canvas = document.getElementById("renderCanvas"); +const statusEl = document.getElementById("status"); +const ui = window.DimosViewerUI || {}; +const staticVersionToken = (() => { + try { + return new URL(document.currentScript?.src || window.location.href).searchParams.get("v") || ""; + } catch { + return ""; + } +})(); +const engine = new BABYLON.Engine(canvas, true, { + preserveDrawingBuffer: false, + stencil: false, + antialias: false, +}); +const scene = new BABYLON.Scene(engine); +scene.useRightHandedSystem = true; +scene.clearColor = new BABYLON.Color4(0.055, 0.063, 0.078, 1); +scene.skipPointerMovePicking = true; +scene.autoClear = true; +scene.autoClearDepthAndStencil = true; + +const camera = new BABYLON.ArcRotateCamera( + "camera", + -Math.PI / 2, + Math.PI / 3, + 8, + new BABYLON.Vector3(0, 0, 1), + scene, +); +camera.upVector = new BABYLON.Vector3(0, 0, 1); +camera.minZ = 0.01; +camera.maxZ = 100000; +camera.wheelPrecision = 40; +camera.attachControl(canvas, true); + +new BABYLON.HemisphericLight("skyLight", new BABYLON.Vector3(0.2, 0.4, 1), scene); +const sun = new BABYLON.DirectionalLight("sun", new BABYLON.Vector3(-0.4, -0.6, -1), scene); +sun.position = new BABYLON.Vector3(20, 30, 40); +scene.environmentIntensity = 0.85; +try { + scene.environmentTexture = BABYLON.CubeTexture.CreateFromPrefilteredData( + "https://assets.babylonjs.com/environments/environmentSpecular.env", + scene, + ); +} catch (error) { + console.warn("environment map unavailable", error); +} + +const bodyNodes = new Map(); +const bodyNodeList = []; +const bodyNameList = []; +const sceneMeshes = []; +const sceneMeshSet = new Set(); +const robotMeshes = []; +const maxAutoSceneBytes = 2 * 1024 * 1024 * 1024; + +const params = new URLSearchParams(window.location.search); +const useRobotMesh = params.get("robot") !== "proxy"; +const sceneMode = params.get("scene") || "auto"; +const showPerfHud = params.get("perf") === "1"; +const enablePointcloudDebug = params.get("debugpc") === "1"; +let sceneConfig = null; +let sceneLoadStarted = false; +let sceneVisible = true; +let pathMesh = null; +let lidarMesh = null; +let splatMesh = null; +let splatLoadStarted = false; +let splatVisible = false; +let lidarMaterial = null; +let lidarPointCount = 0; +let lidarBufferCapacity = 0; +let lidarVisible = true; +const lidarPointSizePx = 5.0; +let clickMode = null; +let navGoalMarker = null; +let pointGoalMarker = null; +let graspGoalMarker = null; +let spawnMarker = null; +let latestRootPosition = null; +let sceneDepthEnabled = true; +let sceneWireEnabled = false; +let forceVisibleEnabled = false; +let driveEnabled = false; +let lastDriveSendTime = 0; +let lastDriveSignature = ""; +let proxyMaterial = null; +const pressedKeys = new Set(); +const driveSendPeriod = 0.08; +const driveLinearSpeed = 0.35; +const driveStrafeSpeed = 0.25; +const driveAngularSpeed = 0.8; +const robotPoseHeaderBytes = 16; +const stateQueueMaxLength = 8; +const stateImmediateDeltaMs = 100; +const stateMaxLagMs = 100; +const stateEarlyThresholdMs = 3; +const statePacingDamping = 0.95; +const robotRootBodyNames = new Set(["pelvis", "torso_link", "body_1", "base"]); +const posePositionEpsilon = 1e-5; +const poseQuaternionEpsilon = 1e-5; +const queuedStateFrames = []; +const lastAppliedRobotPoses = []; +let robotRootBodyIndex = -1; +const robotPoseComposeScale = BABYLON.Vector3.One(); +const robotPoseDecomposeScale = BABYLON.Vector3.One(); +const robotPosePositionScratch = BABYLON.Vector3.Zero(); +const robotPoseQuaternionScratch = BABYLON.Quaternion.Identity(); +const robotPoseMatrixScratch = BABYLON.Matrix.Identity(); +const robotPoseRootMatrixScratch = BABYLON.Matrix.Identity(); +const robotPoseRootInverseMatrixScratch = BABYLON.Matrix.Identity(); +const stateTiming = { + previousSourceMs: null, + lastIdealJsMs: null, + jsMinusSourceMs: Number.POSITIVE_INFINITY, +}; +const streamRef = { worker: null, ready: false }; +const pointcloudWorkerRef = { + worker: null, + inflight: false, + queued: null, + lastDropped: 0, + inflightSinceMs: 0, +}; +const POINTCLOUD_INFLIGHT_TIMEOUT_MS = 3000; +let lastPathVersion = null; +let pendingPointcloudFrame = null; +let lastPointcloudDebugMs = 0; +const debugLabelLastMs = new Map(); +const perfCounters = { + lastReportMs: performance.now(), + frames: 0, + poseFrames: 0, + poseBodiesUpdated: 0, + poseBodiesSkipped: 0, + pointcloudFrames: 0, + pointcloudBufferUpdates: 0, + pointcloudRecreates: 0, + pointcloudDecodeMs: 0, + pointcloudBuildMs: 0, + pointcloudUploadMs: 0, +}; + +const vec3 = (values) => new BABYLON.Vector3(values[0], values[1], values[2]); +const quatWxyz = (values) => + new BABYLON.Quaternion(values[1], values[2], values[3], values[0]); + +function finiteNumber(value, fallback) { + const number = Number(value); + return Number.isFinite(number) ? number : fallback; +} + +function registerSceneMesh(mesh) { + sceneMeshes.push(mesh); + sceneMeshSet.add(mesh); +} + +function setStatus(message) { + if (ui.setStatus) { + ui.setStatus(message); + return; + } + statusEl.textContent = message; +} + +function updatePerfCounters() { + perfCounters.frames += 1; + const now = performance.now(); + const elapsedMs = now - perfCounters.lastReportMs; + if (elapsedMs < 1000) return; + + const elapsedSeconds = elapsedMs / 1000; + const summary = [ + `fps=${(perfCounters.frames / elapsedSeconds).toFixed(1)}`, + `pose=${perfCounters.poseFrames}/s`, + `bodies=${perfCounters.poseBodiesUpdated}/${perfCounters.poseBodiesSkipped}`, + `pc=${perfCounters.pointcloudFrames}/s`, + `pcbuf=${perfCounters.pointcloudBufferUpdates}/${perfCounters.pointcloudRecreates}`, + `pcms=${perfCounters.pointcloudDecodeMs.toFixed(1)}/${perfCounters.pointcloudBuildMs.toFixed(1)}/${perfCounters.pointcloudUploadMs.toFixed(1)}`, + `pcdrop=${pointcloudWorkerRef.lastDropped}`, + `q=${queuedStateFrames.length}`, + ].join(" "); + statusEl.title = summary; + if (showPerfHud) statusEl.textContent = summary; + + perfCounters.lastReportMs = now; + perfCounters.frames = 0; + perfCounters.poseFrames = 0; + perfCounters.poseBodiesUpdated = 0; + perfCounters.poseBodiesSkipped = 0; + perfCounters.pointcloudFrames = 0; + perfCounters.pointcloudBufferUpdates = 0; + perfCounters.pointcloudRecreates = 0; + perfCounters.pointcloudDecodeMs = 0; + perfCounters.pointcloudBuildMs = 0; + perfCounters.pointcloudUploadMs = 0; +} + +function maybeSendPointcloudDebug(now = performance.now()) { + if (!enablePointcloudDebug) return; + if (now - lastPointcloudDebugMs < 1000) return; + lastPointcloudDebugMs = now; + postViewerDebug("pointcloud", { + status: statusEl.textContent, + title: statusEl.title, + ready: Boolean(window.__viewerReady), + lidarVisible, + hasLidarMesh: Boolean(lidarMesh), + lidarPointCount, + pcFrames: perfCounters.pointcloudFrames, + pcBufferUpdates: perfCounters.pointcloudBufferUpdates, + pcRecreates: perfCounters.pointcloudRecreates, + pcDecodeMs: perfCounters.pointcloudDecodeMs, + pcBuildMs: perfCounters.pointcloudBuildMs, + pcUploadMs: perfCounters.pointcloudUploadMs, + pcDropped: pointcloudWorkerRef.lastDropped, + pcInflight: pointcloudWorkerRef.inflight, + pcQueued: Boolean(pointcloudWorkerRef.queued), + pcPendingFrame: Boolean(pendingPointcloudFrame), + }); +} + +function postViewerDebug(label, payload, throttleMs = 1000) { + if (!enablePointcloudDebug) return; + const now = performance.now(); + const last = debugLabelLastMs.get(label) || 0; + if (throttleMs > 0 && now - last < throttleMs) return; + debugLabelLastMs.set(label, now); + fetch("/viewer_debug", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ label, payload }), + keepalive: true, + }).catch(() => {}); +} + +function setButtonActive(id, active) { + if (ui.setButtonActive) { + ui.setButtonActive(id, active); + return; + } + document.getElementById(id).dataset.active = String(active); +} + +function setRenderableVisible(node, visible) { + if (!node) return; + if ("isVisible" in node) node.isVisible = visible; + if ("visibility" in node) node.visibility = visible ? 1 : 0; +} + +function setSceneVisibility(visible) { + sceneVisible = visible; + for (const mesh of sceneMeshes) setRenderableVisible(mesh, visible); + setButtonActive("toggleScene", visible); +} + +function setRobotVisibility(visible) { + for (const mesh of robotMeshes) mesh.setEnabled(visible); + setButtonActive("toggleRobot", visible); +} + +function setLidarVisibility(visible) { + lidarVisible = visible; + if (lidarMesh) lidarMesh.setEnabled(visible); + setButtonActive("toggleLidar", visible); +} + +function setSplatVisibility(visible) { + splatVisible = visible; + setButtonActive("toggleSplat", visible); + if (visible && !splatMesh && !splatLoadStarted && sceneConfig) { + loadSplat(sceneConfig); + return; + } + if (splatMesh) splatMesh.setEnabled(visible); +} + +async function loadSplat(config) { + if (splatLoadStarted || !config || !config.splatFile) return; + splatLoadStarted = true; + setStatus("loading splat"); + try { + // splat is parented to the same scene root the mesh uses, so the package's + // scene alignment (scale / translation / rotation) is applied exactly once + // in the browser and the per-splat alignment stays source-frame relative. + const root = new BABYLON.TransformNode("splatRoot", scene); + root.position = vec3(config.scenePosition); + root.scaling = new BABYLON.Vector3(config.sceneScale, config.sceneScale, config.sceneScale); + root.rotationQuaternion = quatWxyz(config.sceneWxyz); + + const ply = "/assets/" + config.splatFile; + const GS = BABYLON.GaussianSplattingMesh; + if (!GS) { + console.error("BABYLON.GaussianSplattingMesh not available — Babylon too old?"); + setStatus("splat unsupported"); + return; + } + const gs = new GS("splat", null, scene); + await gs.loadFileAsync(ply); + gs.parent = root; + + const a = config.splatAlignment || {}; + const s = Number(a.scale ?? 1.0); + gs.scaling = new BABYLON.Vector3(s, s, s); + const t = a.translation || [0, 0, 0]; + gs.position = new BABYLON.Vector3(t[0], t[1], t[2]); + // SplatAlignment.rotation_zyx is intrinsic ZYX in degrees: R = Rz @ Ry @ Rx. + const [rzDeg, ryDeg, rxDeg] = a.rotation_zyx || [0, 0, 0]; + const rz = (rzDeg * Math.PI) / 180; + const ry = (ryDeg * Math.PI) / 180; + const rx = (rxDeg * Math.PI) / 180; + const qx = BABYLON.Quaternion.RotationAxis(new BABYLON.Vector3(1, 0, 0), rx); + const qy = BABYLON.Quaternion.RotationAxis(new BABYLON.Vector3(0, 1, 0), ry); + const qz = BABYLON.Quaternion.RotationAxis(new BABYLON.Vector3(0, 0, 1), rz); + gs.rotationQuaternion = qz.multiply(qy).multiply(qx); + if (a.y_up) { + // Compose an extra Y-up -> Z-up rotation (-90° around X) on splat-local + // before the ZYX above. Equivalent to inserting Rx(-90°) on the right. + const qYupToZup = BABYLON.Quaternion.RotationAxis( + new BABYLON.Vector3(1, 0, 0), + -Math.PI / 2, + ); + gs.rotationQuaternion = gs.rotationQuaternion.multiply(qYupToZup); + } + + splatMesh = gs; + splatMesh.setEnabled(splatVisible); + setStatus("splat loaded"); + } catch (err) { + console.error("splat load failed", err); + setStatus("splat load failed"); + splatLoadStarted = false; + } +} + +function sceneMaterials() { + const visited = new Set(); + const materials = []; + for (const mesh of sceneMeshes) { + const material = mesh.material; + if (!material || visited.has(material.uniqueId)) continue; + visited.add(material.uniqueId); + materials.push(material); + } + return materials; +} + +function ensureMaterialDefaults(material) { + material.metadata = material.metadata || {}; + if (material.metadata.dimosDefaults) return material.metadata.dimosDefaults; + material.metadata.dimosDefaults = { + alpha: material.alpha, + backFaceCulling: material.backFaceCulling, + disableDepthWrite: material.disableDepthWrite, + forceDepthWrite: material.forceDepthWrite, + transparencyMode: material.transparencyMode, + needDepthPrePass: material.needDepthPrePass, + metallic: material.metallic, + roughness: material.roughness, + environmentIntensity: material.environmentIntensity, + }; + return material.metadata.dimosDefaults; +} + +function editMaterial(material, apply) { + if (material.unfreeze) material.unfreeze(); + apply(material, ensureMaterialDefaults(material)); + if (material.freeze) material.freeze(); +} + +function configureStaticSceneMaterial(material) { + editMaterial(material, (mat) => { + mat.disableDepthWrite = false; + mat.forceDepthWrite = true; + mat.needDepthPrePass = false; + }); +} + +function setClickMode(mode) { + clickMode = clickMode === mode ? null : mode; + setButtonActive("navClick", clickMode === "nav"); + setButtonActive("pointClick", clickMode === "point"); + setButtonActive("graspClick", clickMode === "grasp"); + setButtonActive("spawnClick", clickMode === "spawn"); + if (clickMode === "nav") setStatus("click nav target"); + if (clickMode === "point") setStatus("click point target"); + if (clickMode === "grasp") setStatus("click object to grasp"); + if (clickMode === "spawn") setStatus("click spawn point"); + if (clickMode === null) setStatus("live"); +} + +function markerMaterial(name, color) { + const material = new BABYLON.StandardMaterial(name, scene); + material.diffuseColor = color; + material.emissiveColor = color.scale(0.75); + material.specularColor = BABYLON.Color3.Black(); + material.disableLighting = true; + return material; +} + +const navMarkerMaterial = markerMaterial( + "navGoalMaterial", + new BABYLON.Color3(0.06, 0.82, 1.0), +); +const pointMarkerMaterial = markerMaterial( + "pointGoalMaterial", + new BABYLON.Color3(1.0, 0.18, 0.78), +); +const graspMarkerMaterial = markerMaterial( + "graspGoalMaterial", + new BABYLON.Color3(1.0, 0.72, 0.05), +); +const spawnMarkerMaterial = markerMaterial( + "spawnMarkerMaterial", + new BABYLON.Color3(0.14, 1.0, 0.42), +); + +function placeMarker(existingMarker, name, position, material, diameter) { + if (existingMarker) existingMarker.dispose(); + const marker = BABYLON.MeshBuilder.CreateSphere( + name, + { diameter, segments: 16 }, + scene, + ); + marker.position = position; + marker.material = material; + marker.isPickable = false; + return marker; +} + +function updateKeyboardCamera() { + if (driveEnabled) return; + const deltaSeconds = Math.min(engine.getDeltaTime() / 1000, 0.05); + const speed = (pressedKeys.has("shift") ? 8.0 : 2.7) * deltaSeconds; + const up = new BABYLON.Vector3(0, 0, 1); + const forward = camera.getForwardRay().direction; + forward.z = 0; + if (forward.lengthSquared() < 1e-8) return; + forward.normalize(); + const right = BABYLON.Vector3.Cross(forward, up).normalize(); + const move = BABYLON.Vector3.Zero(); + + if (pressedKeys.has("w")) move.addInPlace(forward); + if (pressedKeys.has("s")) move.subtractInPlace(forward); + if (pressedKeys.has("d")) move.addInPlace(right); + if (pressedKeys.has("a")) move.subtractInPlace(right); + if (pressedKeys.has("e")) move.addInPlace(up); + if (pressedKeys.has("q")) move.subtractInPlace(up); + + if (move.lengthSquared() === 0) return; + move.normalize().scaleInPlace(speed); + camera.target.addInPlace(move); +} + +function sendSocketPayload(payload) { + if (!streamRef.worker || !streamRef.ready) return false; + streamRef.worker.postMessage({ type: "send_json", payload }); + return true; +} + +function currentDriveTwist() { + const speedScale = pressedKeys.has("shift") ? 1.8 : 1.0; + let linearX = 0.0; + let linearY = 0.0; + let angularZ = 0.0; + + if (pressedKeys.has("w")) linearX += driveLinearSpeed * speedScale; + if (pressedKeys.has("s")) linearX -= driveLinearSpeed * speedScale; + if (pressedKeys.has("q")) linearY += driveStrafeSpeed * speedScale; + if (pressedKeys.has("e")) linearY -= driveStrafeSpeed * speedScale; + if (pressedKeys.has("a")) angularZ += driveAngularSpeed * speedScale; + if (pressedKeys.has("d")) angularZ -= driveAngularSpeed * speedScale; + + return { + linear: [linearX, linearY, 0.0], + angular: [0.0, 0.0, angularZ], + }; +} + +function sendDriveCommand(force = false) { + if (!driveEnabled && !force) return; + const now = performance.now() / 1000; + if (!force && now - lastDriveSendTime < driveSendPeriod) return; + + const twist = force + ? { linear: [0.0, 0.0, 0.0], angular: [0.0, 0.0, 0.0] } + : currentDriveTwist(); + const signature = JSON.stringify(twist); + const isZero = + twist.linear.every((value) => Math.abs(value) < 1e-6) && + twist.angular.every((value) => Math.abs(value) < 1e-6); + if (!force && isZero && signature === lastDriveSignature) return; + + const M = window.dimosMsgs; + const lcm = window.dimosLcm; + if (M && lcm) { + try { + const t = new M.geometry_msgs.Twist({ + linear: new M.geometry_msgs.Vector3({ + x: twist.linear[0], y: twist.linear[1], z: twist.linear[2], + }), + angular: new M.geometry_msgs.Vector3({ + x: twist.angular[0], y: twist.angular[1], z: twist.angular[2], + }), + }); + lcm.publish("/cmd_vel", t); + } catch (err) { + console.warn("[lcm] cmd_vel publish failed", err); + return; + } + } + lastDriveSendTime = now; + lastDriveSignature = signature; +} + +function setDriveEnabled(enabled) { + driveEnabled = enabled; + setButtonActive("toggleDrive", enabled); + setStatus(enabled ? "drive: WASD turn/move, QE strafe" : "live"); + if (!enabled) sendDriveCommand(true); +} + +// ─── ENTITY MIRROR (stub) ──────────────────────────────────────────────── +// +// The simulation tree's authority viewer can mirror dynamic scene entities: +// MuJoCo publishes /entity_state_batch (plus an entity-descriptor replay on +// /ws) and the browser spawns kinematic meshes it re-poses every tick. That +// subsystem is deliberately absent from this viewer-only build — this page +// renders robots, scenes and pointclouds; it owns and simulates nothing. +// +// To bring entity mirroring here, port from the simulation tree's Babylon +// module (dimos/simulation/bridges/babylon on the sim refactor branch): +// 1. entity-descriptor replay on /ws → spawn/despawn handlers + the +// primitive/mesh entity builders, +// 2. an /entity_state_batch subscription → pose mirroring, +// 3. per-entity visibility hooks in setSceneVisibility. +// The server half of the replay was dropped from the slim module and needs +// re-adding at the same time. +let entityAuthorityExternal = false; +function configureEntityMirror(config) { + entityAuthorityExternal = config.entityAuthority === "external"; + if (!entityAuthorityExternal) return; + whenLcmReady(() => { + // Acknowledge-only: log once so an operator can see the sim IS + // publishing entities that this build does not draw yet. + let seen = false; + window.dimosLcm.subscribeChannel( + "/entity_state_batch#pimsim.EntityStateBatch", + () => { + if (seen) return; + seen = true; + console.info( + "[entities] entity_state_batch is on the bus — mirroring is stubbed in this build (see ENTITY MIRROR in app.js)", + ); + }, + ); + }); +} + +function setSceneDepthWrite(enabled) { + sceneDepthEnabled = enabled; + for (const material of sceneMaterials()) { + editMaterial(material, (mat) => { + mat.disableDepthWrite = !enabled; + mat.needDepthPrePass = false; + }); + } + setButtonActive("toggleDepth", enabled); +} + +function setSceneWireframe(enabled) { + sceneWireEnabled = enabled; + for (const material of sceneMaterials()) { + editMaterial(material, (mat) => { + mat.wireframe = enabled; + }); + } + setButtonActive("toggleWire", enabled); +} + +function setForceVisible(enabled) { + forceVisibleEnabled = enabled; + for (const material of sceneMaterials()) { + editMaterial(material, (mat, defaults) => { + if (!enabled) { + mat.alpha = defaults.alpha; + mat.backFaceCulling = defaults.backFaceCulling; + mat.disableDepthWrite = false; + mat.forceDepthWrite = true; + mat.transparencyMode = defaults.transparencyMode; + mat.needDepthPrePass = false; + return; + } + mat.backFaceCulling = false; + mat.alpha = 1; + mat.disableDepthWrite = false; + mat.forceDepthWrite = true; + mat.needDepthPrePass = false; + mat.transparencyMode = BABYLON.Material.MATERIAL_OPAQUE; + if (mat.albedoColor) { + mat.metallic = 0; + mat.roughness = 0.9; + mat.environmentIntensity = 1; + } + if (mat.diffuseColor) { + mat.diffuseColor = mat.diffuseColor || new BABYLON.Color3(0.8, 0.8, 0.8); + } + }); + } + setButtonActive("forceVisible", enabled); +} + +function computeMeshBounds(meshes) { + const min = new BABYLON.Vector3(Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY, Number.POSITIVE_INFINITY); + const max = new BABYLON.Vector3(Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY, Number.NEGATIVE_INFINITY); + let count = 0; + for (const mesh of meshes) { + if (!mesh.getTotalVertices || mesh.getTotalVertices() === 0) continue; + mesh.computeWorldMatrix(true); + // Do NOT refreshBoundingInfo() here: it recomputes bounds from the raw + // vertex buffer, which for KHR_mesh_quantization GLBs (the babylon cook + // profile quantizes) yields raw integer coords (0..16383) — the camera + // then auto-frames a phantom ~18 km scene. The loader's bounding info is + // already dequantized and correct; the world-matrix refresh above is all + // that's needed. + const box = mesh.getBoundingInfo().boundingBox; + min.x = Math.min(min.x, box.minimumWorld.x); + min.y = Math.min(min.y, box.minimumWorld.y); + min.z = Math.min(min.z, box.minimumWorld.z); + max.x = Math.max(max.x, box.maximumWorld.x); + max.y = Math.max(max.y, box.maximumWorld.y); + max.z = Math.max(max.z, box.maximumWorld.z); + count += 1; + } + if (count === 0) return null; + const center = min.add(max).scale(0.5); + const extent = max.subtract(min); + return { min, max, center, extent, count }; +} + +function fitCameraToMeshes(meshes) { + const bounds = computeMeshBounds(meshes); + if (!bounds) return; + const { center, extent, count } = bounds; + camera.setTarget(center); + camera.radius = Math.max(2, extent.length() * 0.55); + setStatus(`scene ${count} meshes`); +} + +function focusRobot() { + if (!latestRootPosition) return; + camera.setTarget(latestRootPosition); + // Clamp INTO a useful orbit: the scene auto-frame can leave the camera + // tens of metres out (outside the building, roof occluding everything), + // and only-growing the radius kept it there. + camera.radius = Math.min(8, Math.max(4, camera.radius)); +} + +async function loadConfig() { + const response = await fetch("/config.json", { cache: "no-store" }); + return await response.json(); +} + +async function loadSceneAsset(config) { + if (sceneLoadStarted) return; + sceneLoadStarted = true; + if (!config.sceneFile) return; + if (config.sceneBytes > maxAutoSceneBytes) { + setStatus("scene exceeds browser load guard"); + return; + } + setStatus("loading scene"); + const root = new BABYLON.TransformNode("sceneRoot", scene); + root.position = vec3(config.scenePosition); + root.scaling = new BABYLON.Vector3(config.sceneScale, config.sceneScale, config.sceneScale); + root.rotationQuaternion = quatWxyz(config.sceneWxyz); + + engine.stopRenderLoop(renderFrame); + let result = null; + try { + result = await BABYLON.SceneLoader.ImportMeshAsync(null, "/assets/", config.sceneFile, scene); + } finally { + engine.runRenderLoop(renderFrame); + } + + for (const light of result.lights || []) { + light.dispose(); + } + for (const camera of result.cameras || []) { + camera.dispose(); + } + for (const mesh of result.meshes) { + if (mesh.parent === null) mesh.parent = root; + mesh.isPickable = true; + mesh.metadata = { dimosSceneMesh: true }; + if (mesh.getTotalVertices && mesh.getTotalVertices() > 0) registerSceneMesh(mesh); + if (mesh.material) configureStaticSceneMaterial(mesh.material); + if (mesh.getTotalVertices && mesh.getTotalVertices() > 0) { + mesh.computeWorldMatrix(true); + const hasThinInstances = !!mesh.thinInstanceCount && mesh.thinInstanceCount > 0; + if (hasThinInstances) { + // EXT_mesh_gpu_instancing meshes land as Babylon "thin instances". + // refreshBoundingInfo() only sees the prototype geometry at the GLB + // origin, so as the camera pans Babylon frustum-culls the entire + // mesh whenever (0, 0, 0) leaves the frustum - scattered instances + // pop out even though they're in view. thinInstanceRefreshBoundingInfo + // expands the bbox to enclose every instance; alwaysSelectAsActiveMesh + // is the belt-and-suspenders fallback. 47k instanced draws is trivial + // GPU work, so killing CPU-side culling on instanced meshes is cheap. + mesh.thinInstanceRefreshBoundingInfo(true); + mesh.alwaysSelectAsActiveMesh = true; + } else { + mesh.refreshBoundingInfo(true); + } + mesh.freezeWorldMatrix(); + // Instanced bbox depends on instance buffer updates we may apply + // later; leave the sync hook intact for those. + mesh.doNotSyncBoundingInfo = !hasThinInstances; + } + } + root.freezeWorldMatrix(); + setSceneDepthWrite(sceneDepthEnabled); + setSceneWireframe(sceneWireEnabled); + setForceVisible(forceVisibleEnabled); + fitCameraToMeshes(sceneMeshes); +} + +function pickScenePoint() { + const ray = scene.createPickingRay( + scene.pointerX, + scene.pointerY, + BABYLON.Matrix.Identity(), + camera, + ); + const pick = scene.pickWithRay(ray, (mesh) => sceneMeshSet.has(mesh)); + if (pick && pick.hit && pick.pickedPoint) return pick.pickedPoint; + if (Math.abs(ray.direction.z) < 1e-6) return null; + const distance = -ray.origin.z / ray.direction.z; + if (distance <= 0) return null; + return ray.origin.add(ray.direction.scale(distance)); +} + +async function loadRobot() { + setStatus("loading robot"); + const response = await fetch("/robot.json", { cache: "no-store" }); + const payload = await response.json(); + bodyNodeList.length = 0; + bodyNameList.length = 0; + lastAppliedRobotPoses.length = 0; + robotRootBodyIndex = -1; + for (const bodyName of payload.bodyNames) { + bodyNameList.push(bodyName); + bodyNodeList.push(ensureBodyNode(bodyName)); + if (robotRootBodyIndex < 0 && robotRootBodyNames.has(bodyName)) { + robotRootBodyIndex = bodyNodeList.length - 1; + } + } + if (!useRobotMesh) return; + + for (const geom of payload.geoms) { + const mesh = new BABYLON.Mesh(`robot:${geom.id}`, scene); + const normals = []; + BABYLON.VertexData.ComputeNormals(geom.vertices, geom.indices, normals); + const vertexData = new BABYLON.VertexData(); + vertexData.positions = geom.vertices; + vertexData.indices = geom.indices; + vertexData.normals = normals; + vertexData.applyToMesh(mesh); + + const material = new BABYLON.StandardMaterial(`robotMat:${geom.id}`, scene); + material.diffuseColor = new BABYLON.Color3(geom.rgba[0], geom.rgba[1], geom.rgba[2]); + material.specularColor = new BABYLON.Color3(0.18, 0.18, 0.18); + material.alpha = geom.rgba[3] > 0 ? geom.rgba[3] : 1; + material.backFaceCulling = false; + mesh.material = material; + + mesh.parent = bodyNodes.get(geom.body); + mesh.position = vec3(geom.position); + mesh.rotationQuaternion = quatWxyz(geom.wxyz); + mesh.isPickable = false; + robotMeshes.push(mesh); + } +} + +function proxyDiameter(bodyName) { + if (bodyName === "world") return 0; + if (bodyName.includes("pelvis") || bodyName.includes("torso")) return 0.22; + if (bodyName.includes("hip") || bodyName.includes("shoulder")) return 0.14; + if (bodyName.includes("knee") || bodyName.includes("elbow")) return 0.11; + if (bodyName.includes("ankle") || bodyName.includes("wrist")) return 0.09; + return 0.075; +} + +function ensureBodyNode(bodyName) { + let node = bodyNodes.get(bodyName); + if (node) return node; + node = new BABYLON.TransformNode(`body:${bodyName}`, scene); + node.rotationQuaternion = BABYLON.Quaternion.Identity(); + bodyNodes.set(bodyName, node); + if (!useRobotMesh) { + const diameter = proxyDiameter(bodyName); + if (diameter > 0) { + if (!proxyMaterial) { + proxyMaterial = new BABYLON.StandardMaterial("robotProxyMat", scene); + proxyMaterial.diffuseColor = new BABYLON.Color3(0.95, 0.62, 0.24); + proxyMaterial.specularColor = new BABYLON.Color3(0.22, 0.22, 0.22); + } + const marker = BABYLON.MeshBuilder.CreateSphere( + `robotProxy:${bodyName}`, + { diameter, segments: 8 }, + scene, + ); + marker.material = proxyMaterial; + marker.parent = node; + marker.isPickable = false; + robotMeshes.push(marker); + } + } + return node; +} + +function poseChanged(poses, offset, previous) { + if (!previous) return true; + return ( + Math.abs(poses[offset] - previous[0]) > posePositionEpsilon || + Math.abs(poses[offset + 1] - previous[1]) > posePositionEpsilon || + Math.abs(poses[offset + 2] - previous[2]) > posePositionEpsilon || + Math.abs(poses[offset + 3] - previous[3]) > poseQuaternionEpsilon || + Math.abs(poses[offset + 4] - previous[4]) > poseQuaternionEpsilon || + Math.abs(poses[offset + 5] - previous[5]) > poseQuaternionEpsilon || + Math.abs(poses[offset + 6] - previous[6]) > poseQuaternionEpsilon + ); +} + +function rememberAppliedPose(bodyIndex, poses, offset) { + let previous = lastAppliedRobotPoses[bodyIndex]; + if (!previous) { + previous = new Float32Array(7); + lastAppliedRobotPoses[bodyIndex] = previous; + } + previous[0] = poses[offset]; + previous[1] = poses[offset + 1]; + previous[2] = poses[offset + 2]; + previous[3] = poses[offset + 3]; + previous[4] = poses[offset + 4]; + previous[5] = poses[offset + 5]; + previous[6] = poses[offset + 6]; +} + +function packedPoseToMatrixToRef(poses, offset, result) { + robotPosePositionScratch.copyFromFloats( + poses[offset], + poses[offset + 1], + poses[offset + 2], + ); + robotPoseQuaternionScratch.copyFromFloats( + poses[offset + 4], + poses[offset + 5], + poses[offset + 6], + poses[offset + 3], + ); + BABYLON.Matrix.ComposeToRef( + robotPoseComposeScale, + robotPoseQuaternionScratch, + robotPosePositionScratch, + result, + ); +} + +function copyPackedPoseToNode(node, poses, offset) { + node.position.copyFromFloats(poses[offset], poses[offset + 1], poses[offset + 2]); + if (!node.rotationQuaternion) { + node.rotationQuaternion = BABYLON.Quaternion.Identity(); + } + node.rotationQuaternion.copyFromFloats( + poses[offset + 4], + poses[offset + 5], + poses[offset + 6], + poses[offset + 3], + ); +} + +function copyMatrixPoseToNode(node, matrix) { + if (!node.rotationQuaternion) { + node.rotationQuaternion = BABYLON.Quaternion.Identity(); + } + matrix.decompose(robotPoseDecomposeScale, node.rotationQuaternion, node.position); +} + +function updateLatestRootPosition(position) { + if (!latestRootPosition) latestRootPosition = BABYLON.Vector3.Zero(); + latestRootPosition.copyFrom(position); +} + +function updatePath(path, pathVersion) { + const hasVersion = Number.isFinite(pathVersion); + if (hasVersion && pathVersion === lastPathVersion) return; + if (hasVersion) lastPathVersion = pathVersion; + + if (pathMesh) { + pathMesh.dispose(); + pathMesh = null; + } + if (!path || path.length <= 1) return; + + pathMesh = BABYLON.MeshBuilder.CreateLines( + "navPath", + { points: path.map((point) => vec3([point[0], point[1], point[2] + 0.08])) }, + scene, + ); + pathMesh.color = new BABYLON.Color3(0.15, 0.95, 0.68); + pathMesh.isPickable = false; +} + +function applyRobotPose(payload) { + const count = Math.min(payload.count, bodyNodeList.length); + + let updated = 0; + let skipped = 0; + for (let bodyIndex = 0; bodyIndex < count; bodyIndex += 1) { + const node = bodyNodeList[bodyIndex]; + const bodyName = bodyNameList[bodyIndex]; + const offset = bodyIndex * 7; + + if (poseChanged(payload.poses, offset, lastAppliedRobotPoses[bodyIndex])) { + copyPackedPoseToNode(node, payload.poses, offset); + rememberAppliedPose(bodyIndex, payload.poses, offset); + updated += 1; + } else { + skipped += 1; + } + if (robotRootBodyNames.has(bodyName)) { + updateLatestRootPosition(node.position); + } + } + + if (!latestRootPosition && count > 1) { + updateLatestRootPosition(bodyNodeList[1].position); + } + perfCounters.poseFrames += 1; + perfCounters.poseBodiesUpdated += updated; + perfCounters.poseBodiesSkipped += skipped; +} + +function queueRobotPose(payload) { + const nowMs = performance.now(); + const sourceMs = Number.isFinite(payload.time) ? payload.time * 1000 : nowMs; + stateTiming.jsMinusSourceMs = Math.min( + nowMs - sourceMs, + stateTiming.jsMinusSourceMs, + ); + + const previousSourceMs = stateTiming.previousSourceMs ?? sourceMs; + const sourceDeltaMs = sourceMs - previousSourceMs; + stateTiming.previousSourceMs = sourceMs; + + let targetMs = nowMs; + const sourceLagMs = nowMs - stateTiming.jsMinusSourceMs - sourceMs; + if ( + stateTiming.lastIdealJsMs === null || + sourceDeltaMs <= 0 || + sourceDeltaMs > stateImmediateDeltaMs || + sourceLagMs > stateMaxLagMs + ) { + stateTiming.lastIdealJsMs = nowMs; + } else { + const idealNextMs = stateTiming.lastIdealJsMs + sourceDeltaMs; + const timeUntilIdealMs = idealNextMs - nowMs; + if (timeUntilIdealMs > stateEarlyThresholdMs) { + targetMs = nowMs + timeUntilIdealMs * statePacingDamping; + stateTiming.lastIdealJsMs += sourceDeltaMs * statePacingDamping; + } else { + stateTiming.lastIdealJsMs = nowMs; + } + } + + queuedStateFrames.push({ targetMs, payload }); + while (queuedStateFrames.length > stateQueueMaxLength) { + queuedStateFrames.shift(); + } +} + +function applyQueuedState() { + const nowMs = performance.now(); + let frame = null; + while (queuedStateFrames.length > 0 && queuedStateFrames[0].targetMs <= nowMs) { + frame = queuedStateFrames.shift(); + } + if (!frame) return; + applyRobotPose(frame.payload); +} + +function createLidarMaterial() { + if (lidarMaterial) return lidarMaterial; + + const shaders = BABYLON.Effect.ShadersStore; + shaders.lidarPointVertexShader = ` + precision highp float; + attribute vec3 position; + attribute vec4 color; + uniform mat4 worldViewProjection; + uniform float pointSize; + varying vec4 vColor; + + void main(void) { + gl_Position = worldViewProjection * vec4(position, 1.0); + gl_PointSize = pointSize; + vColor = color; + } + `; + shaders.lidarPointFragmentShader = ` + precision highp float; + varying vec4 vColor; + + void main(void) { + vec2 delta = gl_PointCoord - vec2(0.5); + float radiusSquared = dot(delta, delta); + if (radiusSquared > 0.25) discard; + float edgeAlpha = smoothstep(0.25, 0.16, radiusSquared); + gl_FragColor = vec4(vColor.rgb, vColor.a * edgeAlpha); + } + `; + lidarMaterial = new BABYLON.ShaderMaterial( + "lidarMaterial", + scene, + { vertex: "lidarPoint", fragment: "lidarPoint" }, + { + attributes: ["position", "color"], + uniforms: ["worldViewProjection", "pointSize"], + needAlphaBlending: true, + }, + ); + lidarMaterial.pointsCloud = true; + lidarMaterial.setFloat("pointSize", lidarPointSizePx); + lidarMaterial.backFaceCulling = false; + return lidarMaterial; +} + +function updatePointCloud(payload) { + const count = payload.count || 0; + if (count === 0 || !payload.positions || !payload.colors) return; + perfCounters.pointcloudFrames += 1; + const uploadStart = performance.now(); + + // Grow-only watermark: the voxel mapper's point count drifts upward + // almost every emit as the robot explores. Without padding to a stable + // capacity, every emit would dispose+recreate the Babylon mesh, which + // is a black frame between dispose and create — the visible flicker. + // Pad to next power of two above count, unused slots get NaN positions + // so the rasterizer culls them and we don't render junk at point 0. + if (count > lidarBufferCapacity) { + let cap = Math.max(8192, lidarBufferCapacity || 1); + while (cap < count) cap *= 2; + lidarBufferCapacity = cap; + if (lidarMesh) { + lidarMesh.dispose(); + lidarMesh = null; + lidarPointCount = 0; + } + } + + const cap = lidarBufferCapacity; + const srcPositions = payload.positions instanceof Float32Array + ? payload.positions + : Float32Array.from(payload.positions); + const positions = new Float32Array(cap * 3); + positions.set(srcPositions.subarray(0, count * 3)); + for (let i = count * 3; i < cap * 3; i += 1) positions[i] = NaN; + + const srcColors = payload.colors instanceof Float32Array + ? payload.colors + : Float32Array.from(payload.colors); + const colors = new Float32Array(cap * 4); + colors.set(srcColors.subarray(0, count * 4)); + // padded color slots stay zero (transparent); doesn't matter since the + // NaN-positioned vertices get culled before rasterization anyway. + + if (lidarMesh && lidarPointCount === cap) { + lidarMesh.updateVerticesData( + BABYLON.VertexBuffer.PositionKind, + positions, + true, + false, + ); + lidarMesh.updateVerticesData( + BABYLON.VertexBuffer.ColorKind, + colors, + true, + false, + ); + lidarMesh.setEnabled(lidarVisible); + perfCounters.pointcloudBufferUpdates += 1; + perfCounters.pointcloudUploadMs += performance.now() - uploadStart; + postViewerDebug("pointcloud-update", { count, cap, mode: "buffer-update" }, 500); + return; + } + + const nextMesh = new BABYLON.Mesh("lidarCloud", scene); + const vertexData = new BABYLON.VertexData(); + vertexData.positions = positions; + vertexData.colors = colors; + vertexData.applyToMesh(nextMesh, true); + nextMesh.hasVertexAlpha = true; + nextMesh.alwaysSelectAsActiveMesh = true; + nextMesh.isPickable = false; + + nextMesh.material = createLidarMaterial(); + nextMesh.setEnabled(lidarVisible); + + lidarMesh = nextMesh; + lidarPointCount = cap; + perfCounters.pointcloudRecreates += 1; + perfCounters.pointcloudUploadMs += performance.now() - uploadStart; + postViewerDebug("pointcloud-update", { count, cap, mode: "recreate" }, 500); +} + +function queuePointcloudFrame(payload) { + pendingPointcloudFrame = payload; +} + +function applyQueuedPointcloud() { + if (!pendingPointcloudFrame) return; + const payload = pendingPointcloudFrame; + pendingPointcloudFrame = null; + updatePointCloud(payload); +} + +function flushQueuedPointcloudPayload() { + const worker = pointcloudWorkerRef.worker; + const queued = pointcloudWorkerRef.queued; + if (!worker || pointcloudWorkerRef.inflight || !queued) return; + pointcloudWorkerRef.queued = null; + pointcloudWorkerRef.inflight = true; + worker.postMessage( + { + type: "payload", + buffer: queued.buffer, + byteOffset: queued.byteOffset, + byteLength: queued.byteLength, + }, + [queued.buffer], + ); +} + +function enqueuePointcloudPayload(payload) { + const worker = pointcloudWorkerRef.worker; + if (!worker) return; + + // Watchdog: if a previous send has been "inflight" past the timeout the + // worker crashed, its ESM import failed, or it dropped a message. We + // can't see worker module-load failures (Chrome fires no onerror for + // those), so force-clear here instead of wedging every subsequent emit + // into the dropped counter forever. + if ( + pointcloudWorkerRef.inflight + && pointcloudWorkerRef.inflightSinceMs > 0 + && performance.now() - pointcloudWorkerRef.inflightSinceMs > POINTCLOUD_INFLIGHT_TIMEOUT_MS + ) { + console.warn("[pointcloud worker] inflight timeout, force-clearing"); + postViewerDebug("pointcloud-worker-timeout", { + stuckForMs: performance.now() - pointcloudWorkerRef.inflightSinceMs, + }, 0); + pointcloudWorkerRef.inflight = false; + pointcloudWorkerRef.inflightSinceMs = 0; + } + + const queued = { + buffer: payload.buffer, + byteOffset: payload.byteOffset, + byteLength: payload.byteLength, + }; + + if (pointcloudWorkerRef.inflight) { + pointcloudWorkerRef.queued = queued; + worker.postMessage({ type: "dropped", count: 1 }); + return; + } + + pointcloudWorkerRef.inflight = true; + pointcloudWorkerRef.inflightSinceMs = performance.now(); + worker.postMessage( + { + type: "payload", + buffer: queued.buffer, + byteOffset: queued.byteOffset, + byteLength: queued.byteLength, + }, + [queued.buffer], + ); +} + +function connectPointcloudWorker() { + if (pointcloudWorkerRef.worker) return; + const suffix = staticVersionToken ? `?v=${encodeURIComponent(staticVersionToken)}` : ""; + const worker = new Worker(`/static/pointcloud_worker.js${suffix}`, { type: "module" }); + pointcloudWorkerRef.worker = worker; + worker.onmessage = (event) => { + const message = event.data || {}; + if (message.type === "error") { + console.error("[pointcloud worker]", message.message); + postViewerDebug("pointcloud-worker-error", { message: String(message.message || "unknown") }, 0); + pointcloudWorkerRef.inflight = false; + flushQueuedPointcloudPayload(); + return; + } + if (message.type === "empty") { + postViewerDebug("pointcloud-worker-empty", {}, 500); + pointcloudWorkerRef.inflight = false; + pointcloudWorkerRef.inflightSinceMs = 0; + flushQueuedPointcloudPayload(); + return; + } + if (message.type !== "pointcloud") return; + + pointcloudWorkerRef.inflight = false; + pointcloudWorkerRef.inflightSinceMs = 0; + pointcloudWorkerRef.lastDropped = Number(message.stats?.dropped || 0); + perfCounters.pointcloudDecodeMs += Number(message.stats?.decodeMs || 0); + perfCounters.pointcloudBuildMs += Number(message.stats?.buildMs || 0); + postViewerDebug("pointcloud-worker", { + count: Number(message.count || 0), + dropped: Number(message.stats?.dropped || 0), + decodeMs: Number(message.stats?.decodeMs || 0), + buildMs: Number(message.stats?.buildMs || 0), + }, 500); + queuePointcloudFrame({ + count: message.count, + positions: new Float32Array(message.positions), + colors: new Float32Array(message.colors), + }); + flushQueuedPointcloudPayload(); + }; + worker.onerror = (event) => { + console.error("[pointcloud worker] crash", event); + postViewerDebug("pointcloud-worker-crash", { + message: String(event.message || "worker crash"), + filename: String(event.filename || ""), + lineno: Number(event.lineno || 0), + colno: Number(event.colno || 0), + }, 0); + pointcloudWorkerRef.inflight = false; + pointcloudWorkerRef.inflightSinceMs = 0; + flushQueuedPointcloudPayload(); + }; + worker.onmessageerror = (event) => { + console.error("[pointcloud worker] messageerror", event); + pointcloudWorkerRef.inflight = false; + pointcloudWorkerRef.inflightSinceMs = 0; + flushQueuedPointcloudPayload(); + }; +} + +// `window.__viewerReady` becomes true once the WS to the Python module is +// up and (if requested) the scene assets have settled. Headless test +// harnesses (Playwright) poll this before sending cmd_vel etc. +let __viewerSceneReady = false; +async function evaluateViewerReady() { + if (!streamRef.ready) return; + if (!__viewerSceneReady) return; + window.__viewerReady = true; +} +function markViewerSceneReady() { + __viewerSceneReady = true; + evaluateViewerReady(); +} + +// ─── LCM <-> WS bridge subscriptions ──────────────────────────────────── +// +// the viewer module subscribes to every LCM channel and +// forwards raw packets to /lcm-ws. @dimos/msgs decodes them in the +// lcm_client.js module worker. Here we just dispatch decoded messages +// into the existing Babylon renderers — same hooks the old binary frames +// fed into, no rendering changes. + +let recBadgeEl = null; +function setRecordingBadge(active) { + if (!recBadgeEl) { + recBadgeEl = document.createElement("div"); + recBadgeEl.textContent = "● REC"; + recBadgeEl.style.cssText = + "display:none;position:fixed;top:14px;right:14px;z-index:50;" + + "padding:6px 14px;font:700 15px system-ui;color:#fff;" + + "background:#e52626;border-radius:8px;pointer-events:none;"; + document.body.appendChild(recBadgeEl); + // Blink via opacity toggle — cheap, no CSS file changes. + setInterval(() => { + if (recBadgeEl.style.display !== "none") { + recBadgeEl.style.opacity = recBadgeEl.style.opacity === "0.35" ? "1" : "0.35"; + } + }, 500); + } + recBadgeEl.style.display = active ? "block" : "none"; + recBadgeEl.style.opacity = "1"; +} + +function whenLcmReady(callback) { + if (window.dimosLcm && window.dimosMsgs) { + callback(); + return; + } + setTimeout(() => whenLcmReady(callback), 50); +} + +function subscribeLcmTopics() { + const M = window.dimosMsgs; + const lcm = window.dimosLcm; + if (!M || !lcm) return; + + let pathVersion = 0; + + lcm.subscribePayload("/global_map", M.sensor_msgs.PointCloud2, (payload) => { + enqueuePointcloudPayload(payload); + }); + + // Episode-recorder state → blinking REC badge. + lcm.subscribe("/recording", M.std_msgs.Bool, (msg) => { + setRecordingBadge(Boolean(msg.data)); + }); + + lcm.subscribe("/nav_path", M.nav_msgs.Path, (msg) => { + pathVersion += 1; + const pts = (msg.poses || []).map((ps) => [ + ps.pose.position.x, + ps.pose.position.y, + ps.pose.position.z, + ]); + updatePath(pts, pathVersion); + }); + + lcm.subscribe("/coordinator/joint_state", M.sensor_msgs.JointState, (msg) => { + if (!msg.name || !msg.position) return; + const joints = {}; + for (let i = 0; i < msg.name.length && i < msg.position.length; i += 1) { + // Match the canonicalisation the old _make_state_payload did so + // the slider HUD keys still line up. + let k = msg.name[i]; + const slash = k.indexOf("/"); + if (slash >= 0) k = k.slice(slash + 1); + if (k.endsWith("_joint")) k = k.slice(0, -"_joint".length); + joints[k] = Number(msg.position[i]); + } + _updateSlidersFromState(joints); + }); + + lcm.subscribe("/camera_image", M.sensor_msgs.Image, (msg) => { + dispatchLcmCameraFrame("camera", msg); + }); + lcm.subscribe("/workspace_image", M.sensor_msgs.Image, (msg) => { + dispatchLcmCameraFrame("workspace", msg); + }); +} + +function connectStreamWorker() { + const suffix = staticVersionToken ? `?v=${encodeURIComponent(staticVersionToken)}` : ""; + const worker = new Worker(`/static/stream_worker.js${suffix}`); + streamRef.worker = worker; + streamRef.ready = false; + worker.onmessage = (event) => { + const message = event.data || {}; + switch (message.type) { + case "status": + streamRef.ready = Boolean(message.ready); + setStatus(message.status); + if (streamRef.ready) evaluateViewerReady(); + break; + case "state": + // Authority-mode control frames (entity spawn/despawn, sim respawn) + // are ignored in the viewer-only build — see the ENTITY MIRROR stub. + break; + case "robot_pose": + queueRobotPose({ + count: message.count, + time: message.time, + poses: new Float32Array( + message.buffer, + robotPoseHeaderBytes, + message.count * 7, + ), + }); + break; + case "error": + console.warn(message.message); + break; + default: + break; + } + }; + worker.onerror = (event) => { + console.error(event); + streamRef.ready = false; + setStatus("stream worker error"); + }; + worker.postMessage({ type: "connect" }); + return worker; +} + +function updateCameraFrame(cameraName, buffer, jpegOffset) { + if (ui.updateCameraFrame) { + ui.updateCameraFrame(cameraName, buffer, jpegOffset); + return; + } + console.warn("camera frame dropped: DimosViewerUI.updateCameraFrame unavailable", cameraName); +} + +// /camera_image and /workspace_image flow through the bridge as JPEG- +// encoded sensor_msgs.Image (publisher uses JpegLcmTransport). msg.data +// is the JPEG bytes when encoding === "jpeg"; createObjectURL on a +// Blob is the cheapest browser path to display. +function dispatchLcmCameraFrame(name, msg) { + if (!msg || !msg.data || !msg.data.byteLength) return; + if (msg.encoding && msg.encoding !== "jpeg") { + console.warn(`[camera] ${name}: expected jpeg encoding, got ${msg.encoding}`); + return; + } + updateCameraFrame(name, msg.data.buffer, msg.data.byteOffset); +} + +function installClickPublisher() { + scene.onPointerObservable.add((pointerInfo) => { + if (pointerInfo.type !== BABYLON.PointerEventTypes.POINTERPICK) return; + const event = pointerInfo.event; + if (event.target !== canvas) return; + + const publishNav = clickMode === "nav" || event.shiftKey; + const publishPoint = clickMode === "point" || event.altKey; + const publishGrasp = clickMode === "grasp"; + const publishSpawn = clickMode === "spawn"; + if (!publishNav && !publishPoint && !publishGrasp && !publishSpawn) return; + + if (!streamRef.ready) return; + + if (publishNav || publishSpawn) { + const point = pickScenePoint(); + if (!point) return; + if (publishSpawn) { + spawnMarker = placeMarker( + spawnMarker, + "spawnMarker", + new BABYLON.Vector3(point.x, point.y, point.z + 0.12), + spawnMarkerMaterial, + 0.28, + ); + sendSocketPayload({ + type: "respawn_at", + point: [point.x, point.y, point.z], + }); + setClickMode(null); + setStatus("spawn requested"); + return; + } + navGoalMarker = placeMarker( + navGoalMarker, + "navGoalMarker", + new BABYLON.Vector3(point.x, point.y, point.z + 0.08), + navMarkerMaterial, + 0.22, + ); + publishLcmPointStamped("/clicked_point", point); + setClickMode(null); + setStatus("nav target sent"); + return; + } + + // Grasp targets the object itself. Entity meshes are isPickable=false and + // sit in front of the scene geometry, so the DEFAULT pick selects the + // wall/floor behind the object. Pick with a predicate that matches entity + // meshes (the invisible per-entity collider carries the merged geometry at + // the live pose) so the click lands on the object — then /grasp_goal lets + // the planner resolve the nearest scene object and reach its grasp pose. + if (publishGrasp) { + // Walk the parent chain: an entity is either the named `entity:` + // collider/primitive or a visible child mesh parented under it. + const entityIdOf = (m) => { + for (let n = m; n; n = n.parent) { + if (n.metadata && n.metadata.entityId) return n.metadata.entityId; + if (typeof n.name === "string" && n.name.startsWith("entity:")) { + return n.name.slice("entity:".length); + } + } + return null; + }; + const hit = scene.pick(scene.pointerX, scene.pointerY, (m) => entityIdOf(m) !== null); + if (!hit || !hit.hit || !hit.pickedPoint) { + setStatus("no object here — click directly on an object to grasp"); + return; // keep grasp mode active so the next click can retry + } + graspGoalMarker = placeMarker( + graspGoalMarker, + "graspGoalMarker", + hit.pickedPoint, + graspMarkerMaterial, + 0.16, + ); + publishLcmPointStamped("/grasp_goal", hit.pickedPoint); + setClickMode(null); + setStatus(`grasp target sent (${entityIdOf(hit.pickedMesh) || "object"})`); + return; + } + + const pick = pointerInfo.pickInfo; + let point = null; + if (pick && pick.hit && pick.pickedPoint) { + point = pick.pickedPoint; + } else { + const ray = scene.createPickingRay( + scene.pointerX, + scene.pointerY, + BABYLON.Matrix.Identity(), + camera, + ); + if (Math.abs(ray.direction.z) < 1e-6) return; + const distance = (1.0 - ray.origin.z) / ray.direction.z; + if (distance <= 0) return; + point = ray.origin.add(ray.direction.scale(distance)); + } + pointGoalMarker = placeMarker( + pointGoalMarker, + "pointGoalMarker", + point, + pointMarkerMaterial, + 0.16, + ); + publishLcmPointStamped("/point_goal", point); + setClickMode(null); + setStatus("point target sent"); + }); +} + +function publishLcmPointStamped(topic, point) { + const M = window.dimosMsgs; + const lcm = window.dimosLcm; + if (!M || !lcm) return; + try { + const ts = Date.now() / 1000; + const sec = Math.floor(ts); + const nanosec = Math.floor((ts - sec) * 1e9); + const stamped = new M.geometry_msgs.PointStamped({ + header: new M.std_msgs.Header({ + // "world" to match /odom + the nav stack; + // click goals (/point_goal, /clicked_point, /grasp_goal) ride this frame. + stamp: new M.builtin_interfaces.Time({ sec, nanosec }), + frame_id: "world", + }), + point: new M.geometry_msgs.Point({ x: point.x, y: point.y, z: point.z }), + }); + lcm.publish(topic, stamped); + } catch (err) { + console.warn(`[lcm] ${topic} publish failed`, err); + } +} + +document.getElementById("toggleScene").onclick = () => { + const visible = document.getElementById("toggleScene").dataset.active !== "true"; + setSceneVisibility(visible); +}; +document.getElementById("toggleRobot").onclick = () => { + const visible = document.getElementById("toggleRobot").dataset.active !== "true"; + setRobotVisibility(visible); +}; +document.getElementById("toggleDrive").onclick = () => setDriveEnabled(!driveEnabled); +document.getElementById("respawnRobot").onclick = () => { + sendDriveCommand(true); + sendSocketPayload({ type: "respawn" }); + setStatus("respawn requested"); +}; +document.getElementById("toggleLidar").onclick = () => setLidarVisibility(!lidarVisible); +const _toggleSplat = document.getElementById("toggleSplat"); +if (_toggleSplat) _toggleSplat.onclick = () => setSplatVisibility(!splatVisible); +document.getElementById("toggleCamera").onclick = () => { + const btn = document.getElementById("toggleCamera"); + const active = btn.dataset.active !== "true"; + btn.dataset.active = active ? "true" : "false"; + if (ui.setPanelActive) ui.setPanelActive("cameraPanel", active); + else document.getElementById("cameraPanel").dataset.active = active ? "true" : "false"; +}; +document.getElementById("navClick").onclick = () => setClickMode("nav"); +document.getElementById("pointClick").onclick = () => setClickMode("point"); +document.getElementById("graspClick").onclick = () => setClickMode("grasp"); +document.getElementById("spawnClick").onclick = () => setClickMode("spawn"); +document.getElementById("toggleDepth").onclick = () => setSceneDepthWrite(!sceneDepthEnabled); +document.getElementById("toggleWire").onclick = () => setSceneWireframe(!sceneWireEnabled); +document.getElementById("forceVisible").onclick = () => setForceVisible(!forceVisibleEnabled); +document.getElementById("focusRobot").onclick = focusRobot; +document.getElementById("loadScene").onclick = () => { + if (!sceneConfig) return; + (async () => { + try { + await loadSceneAsset(sceneConfig); + } catch (error) { + console.error(error); + setStatus("scene load failed"); + } + })(); +}; + +// --- Policy arm / dry-run toggles --- +// Initial dataset.active reflects the coordinator's defaults for the +// typical real-hardware blueprint (unarmed, dry-run on). If the +// blueprint configured different defaults the button is still a plain +// toggle — click it once to sync. +document.getElementById("policyArm").onclick = () => { + const btn = document.getElementById("policyArm"); + const engaged = btn.dataset.active !== "true"; + if (!sendSocketPayload({ type: "set_activated", engaged })) return; + btn.dataset.active = engaged ? "true" : "false"; + setStatus(engaged ? "policy armed" : "policy disarmed"); +}; +document.getElementById("policyDryRun").onclick = () => { + const btn = document.getElementById("policyDryRun"); + const enabled = btn.dataset.active !== "true"; + if (!sendSocketPayload({ type: "set_dry_run", enabled })) return; + btn.dataset.active = enabled ? "true" : "false"; + setStatus(enabled ? "dry-run on" : "dry-run off (live)"); +}; + +// --- Arm slider panel --- +// Toggle visibility +document.getElementById("armsToggle").onclick = () => { + const btn = document.getElementById("armsToggle"); + const active = btn.dataset.active !== "true"; + btn.dataset.active = active ? "true" : "false"; + if (ui.setPanelActive) ui.setPanelActive("armsPanel", active); + else document.getElementById("armsPanel").dataset.active = active ? "true" : "false"; +}; + +// Release: stop publishing arm commands (hand control back to MC) +document.getElementById("armsRelease").onclick = () => { + if (sendSocketPayload({ type: "release_arms" })) { + setStatus("arms released"); + } +}; + +// Build the slider list from /arms.json. Each slider sends an +// {type: arm_joint, name, position} message on input (throttled). +function _humanLabel(name) { + // strip "left_"/"right_" prefix for column-internal display + return name + .replace(/^left_/, "") + .replace(/^right_/, "") + .replace(/_/g, " "); +} +// Track which slider the user is currently dragging so we don't +// overwrite its value from incoming joint-state updates. +const _armSliders = {}; // joint_name -> {slider, val, dragging} +let _armSendThrottle = {}; +function _throttledSendJoint(name, position) { + const now = performance.now(); + const last = _armSendThrottle[name] || 0; + if (now - last < 30) return; // ~33 Hz max per slider + _armSendThrottle[name] = now; + sendSocketPayload({ type: "arm_joint", name, position }); +} +function _buildSlider(joint) { + const row = document.createElement("div"); + row.className = "arm-slider-row"; + + const labelTop = document.createElement("div"); + labelTop.className = "joint-name"; + labelTop.textContent = _humanLabel(joint.name); + row.appendChild(labelTop); + + const val = document.createElement("div"); + val.className = "joint-val"; + val.textContent = " … "; + row.appendChild(val); + + const slider = document.createElement("input"); + slider.type = "range"; + slider.min = String(joint.min); + slider.max = String(joint.max); + slider.step = "0.005"; + // Default to midpoint until we get a real reading — visually obvious + // it's not real yet (the value cell shows "..." until first update). + slider.value = String((joint.min + joint.max) / 2); + slider.disabled = true; // enabled once a joint_state arrives + + const entry = { slider, val, dragging: false, ready: false }; + _armSliders[joint.name] = entry; + + slider.addEventListener("pointerdown", () => { entry.dragging = true; }); + slider.addEventListener("pointerup", () => { entry.dragging = false; }); + slider.addEventListener("pointercancel", () => { entry.dragging = false; }); + + slider.oninput = () => { + const pos = parseFloat(slider.value); + val.textContent = pos.toFixed(3); + _throttledSendJoint(joint.name, pos); + }; + slider.onchange = () => { + // Final exact value on release (in case the throttle dropped it) + const pos = parseFloat(slider.value); + sendSocketPayload({ type: "arm_joint", name: joint.name, position: pos }); + }; + row.appendChild(slider); + + const range = document.createElement("div"); + range.className = "joint-range"; + const lo = document.createElement("span"); + lo.textContent = joint.min.toFixed(2); + const hi = document.createElement("span"); + hi.textContent = joint.max.toFixed(2); + range.appendChild(lo); + range.appendChild(hi); + row.appendChild(range); + + return row; +} + +function _updateSlidersFromState(joints) { + if (!joints) return; + for (const [name, value] of Object.entries(joints)) { + const entry = _armSliders[name]; + if (!entry) continue; + if (!entry.ready) { + // First sample → enable the slider, set it to actual position. + entry.ready = true; + entry.slider.disabled = false; + } + if (entry.dragging) continue; // don't fight the user + // Only set value when the slider is idle, so the user sees ground truth. + entry.slider.value = String(value); + entry.val.textContent = Number(value).toFixed(3); + } +} + +(async () => { + try { + const resp = await fetch("/arms.json"); + const data = await resp.json(); + const leftCol = document.getElementById("leftArmSliders"); + const rightCol = document.getElementById("rightArmSliders"); + for (const j of data.joints || []) { + const row = _buildSlider(j); + if (j.name.startsWith("left_")) leftCol.appendChild(row); + else if (j.name.startsWith("right_")) rightCol.appendChild(row); + } + } catch (e) { + console.error("Failed to load /arms.json:", e); + } +})(); + +(async () => { + try { + const config = await loadConfig(); + sceneConfig = config; + configureEntityMirror(config); + connectPointcloudWorker(); + await loadRobot(); + connectStreamWorker(); + whenLcmReady(subscribeLcmTopics); + installClickPublisher(); + setStatus("live"); + if (sceneMode !== "0" && sceneMode !== "manual") { + window.setTimeout(async () => { + try { + await loadSceneAsset(config); + markViewerSceneReady(); + } catch (error) { + console.error(error); + setStatus("scene load failed"); + } + }, 0); + } else { + // No scene to load (manual / empty) — ready as soon as the WS is up. + markViewerSceneReady(); + } + } catch (error) { + console.error(error); + setStatus("load failed"); + } +})(); + +window.addEventListener("keydown", (event) => { + const key = event.key.toLowerCase(); + if (key === "shift") pressedKeys.add("shift"); + if (key === " ") { + if (driveEnabled) sendDriveCommand(true); + event.preventDefault(); + return; + } + if (!["w", "a", "s", "d", "q", "e"].includes(key)) return; + pressedKeys.add(key); + event.preventDefault(); +}); + +window.addEventListener("keyup", (event) => { + const key = event.key.toLowerCase(); + if (key === "shift") pressedKeys.delete("shift"); + pressedKeys.delete(key); +}); + +window.addEventListener("blur", () => { + // Clear any keys the user was holding when the window lost focus + // so we don't keep walking on alt-tab. Only publish a stop if drive + // was actually engaged — otherwise we'd send a spurious zero Twist + // on every page refresh, which clobbers whatever cmd_vel source + // is currently driving the robot (Quest joysticks, an agent, etc.) + // for one tick and shows up as the robot going briefly damp. + pressedKeys.clear(); + if (driveEnabled) sendDriveCommand(true); +}); + +function renderFrame() { + applyQueuedState(); + applyQueuedPointcloud(); + updateKeyboardCamera(); + sendDriveCommand(false); + maybeSendPointcloudDebug(); + scene.render(); + updatePerfCounters(); +} + +engine.runRenderLoop(renderFrame); +window.addEventListener("resize", () => engine.resize()); diff --git a/dimos/web/viewer/static/index.html b/dimos/web/viewer/static/index.html new file mode 100644 index 0000000000..aa347cb8bc --- /dev/null +++ b/dimos/web/viewer/static/index.html @@ -0,0 +1,189 @@ + + + + + + + DimOS Operator Console + + + + + + +
+
+
DIMENSIONAL
+
DIMOS OPERATOR CONSOLE
+
V0.4
+
+ +
+
+
Robot
+
G1 / Go2
+
+
+
Scene
+
Auto
+
+
+
Mode
+
Viewer
+
+
+
Stream
+
Boot
+
+
+
Physics
+
Mirror
+
+
+
Operator
+
Local
+
+ +
+ +
+
+ LIVE SIM VIEWPORT + starting +
+
+ +
+
BABYLON
+
LCM BRIDGE
+
MIRROR MODE
+
+
WHAT OPERATOR SEES
+ +
+
ACTIVE TASK: Manual / policy bridge
+
SPEED 0.0 M/S
+
VIDEO LIVE
+
CONTROL LOCAL
+
+
+
+ + + +
+
Drive Controls
+
+
+
W
+
+
A
+
S
+
D
+
+ +
+ +
+
+
Operator Actions
+
LIVE CONTROLS
+
+
+ + + + + + + + + + +
+
+ +
+
+
Viewport Layers
+
DISPLAY
+
+
+ + + + + + + +
+
+ +
+
+ Arm joints + direct joint command +
+
+
+
Left
+
+
+
+
Right
+
+
+
+
+
+ + + + + + diff --git a/dimos/web/viewer/static/pointcloud_worker.js b/dimos/web/viewer/static/pointcloud_worker.js new file mode 100644 index 0000000000..2849b65664 --- /dev/null +++ b/dimos/web/viewer/static/pointcloud_worker.js @@ -0,0 +1,142 @@ +// Copyright 2025-2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { sensor_msgs } from "https://esm.sh/jsr/@dimos/msgs@0.1.4"; + +let droppedFrames = 0; + +function turboColor(t) { + const x = Math.max(0, Math.min(1, t)); + const r = 34.61 + x * (1172.33 - x * (10793.56 - x * (33300.12 - x * (38394.49 - x * 14825.05)))); + const g = 23.31 + x * (557.33 + x * (1225.33 - x * (3574.96 - x * (1073.77 + x * 707.56)))); + const b = 27.2 + x * (3211.1 - x * (15327.97 - x * (27814 - x * (22569.18 - x * 6838.66)))); + return [ + Math.max(0, Math.min(255, r)), + Math.max(0, Math.min(255, g)), + Math.max(0, Math.min(255, b)), + ]; +} + +function findPointCloudField(fields, name) { + for (const f of fields) if (f.name === name) return f; + return null; +} + +function pointCloud2ToRenderPayload(msg) { + const fx = findPointCloudField(msg.fields, "x"); + const fy = findPointCloudField(msg.fields, "y"); + const fz = findPointCloudField(msg.fields, "z"); + if (!fx || !fy || !fz) return null; + + const step = msg.point_step; + const raw = msg.data instanceof Uint8Array ? msg.data : Uint8Array.from(msg.data || []); + if (!step || !raw.byteLength) return null; + const count = Math.floor(raw.byteLength / step); + if (count === 0) return null; + + const view = new DataView(raw.buffer, raw.byteOffset, raw.byteLength); + const littleEndian = !msg.is_bigendian; + const positions = new Float32Array(count * 3); + const colors = new Float32Array(count * 4); + const frgb = findPointCloudField(msg.fields, "rgb"); + + let zMin = Infinity; + let zMax = -Infinity; + for (let i = 0; i < count; i += 1) { + const o = i * step; + const x = view.getFloat32(o + fx.offset, littleEndian); + const y = view.getFloat32(o + fy.offset, littleEndian); + const z = view.getFloat32(o + fz.offset, littleEndian); + const pi = i * 3; + const ci = i * 4; + if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) { + positions[pi + 0] = 0; + positions[pi + 1] = 0; + positions[pi + 2] = 0; + colors[ci + 0] = 0; + colors[ci + 1] = 0; + colors[ci + 2] = 0; + colors[ci + 3] = 0; + continue; + } + positions[pi + 0] = x; + positions[pi + 1] = y; + positions[pi + 2] = z; + colors[ci + 3] = 0.94; + if (z < zMin) zMin = z; + if (z > zMax) zMax = z; + } + + if (frgb) { + for (let i = 0; i < count; i += 1) { + const o = i * step + frgb.offset; + const ci = i * 4; + colors[ci + 0] = view.getUint8(o + 2) / 255; + colors[ci + 1] = view.getUint8(o + 1) / 255; + colors[ci + 2] = view.getUint8(o + 0) / 255; + } + } else { + const denom = (zMax - zMin) || 1.0; + for (let i = 0; i < count; i += 1) { + const t = (positions[i * 3 + 2] - zMin) / denom; + const [r, g, b] = turboColor(t); + const ci = i * 4; + colors[ci + 0] = r / 255; + colors[ci + 1] = g / 255; + colors[ci + 2] = b / 255; + } + } + + return { count, positions, colors }; +} + +self.onmessage = (event) => { + const message = event.data || {}; + if (message.type === "dropped") { + droppedFrames += Number(message.count || 0); + return; + } + if (message.type !== "payload") return; + + try { + const payload = new Uint8Array(message.buffer, message.byteOffset, message.byteLength); + const decodeStart = performance.now(); + const msg = sensor_msgs.PointCloud2.decode(payload); + const decodeMs = performance.now() - decodeStart; + + const buildStart = performance.now(); + const renderPayload = pointCloud2ToRenderPayload(msg); + const buildMs = performance.now() - buildStart; + if (!renderPayload) { + self.postMessage({ type: "empty" }); + return; + } + + self.postMessage( + { + type: "pointcloud", + count: renderPayload.count, + positions: renderPayload.positions.buffer, + colors: renderPayload.colors.buffer, + stats: { decodeMs, buildMs, dropped: droppedFrames }, + }, + [renderPayload.positions.buffer, renderPayload.colors.buffer], + ); + } catch (error) { + self.postMessage({ + type: "error", + message: error instanceof Error ? `${error.name}: ${error.message}` : String(error), + }); + } +}; diff --git a/dimos/web/viewer/static/stream_worker.js b/dimos/web/viewer/static/stream_worker.js new file mode 100644 index 0000000000..b633ddabfd --- /dev/null +++ b/dimos/web/viewer/static/stream_worker.js @@ -0,0 +1,98 @@ +// Copyright 2025-2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 0x02 (pointcloud) and 0x01 (camera) used to live here too; both moved +// to /lcm-ws — /global_map as sensor_msgs.PointCloud2, /camera_image as +// JpegLcm-encoded sensor_msgs.Image. Only the robot_pose binary frame +// remains on /ws pending the browser-FK migration. +const wsMsgRobotPose = 0x03; +const robotPoseHeaderBytes = 16; + +let socket = null; +let reconnectTimer = null; + +function postStatus(status, ready) { + self.postMessage({ type: "status", status, ready }); +} + +function websocketUrl() { + const protocol = self.location.protocol === "https:" ? "wss:" : "ws:"; + return `${protocol}//${self.location.host}/ws`; +} + +function connect() { + if (reconnectTimer !== null) { + clearTimeout(reconnectTimer); + reconnectTimer = null; + } + + socket = new WebSocket(websocketUrl()); + socket.binaryType = "arraybuffer"; + socket.onopen = () => postStatus("live", true); + socket.onerror = () => postStatus("socket error", false); + socket.onclose = () => { + postStatus("reconnecting", false); + reconnectTimer = setTimeout(connect, 1000); + }; + socket.onmessage = (event) => { + if (typeof event.data === "string") { + try { + const payload = JSON.parse(event.data); + self.postMessage({ type: "state", payload }); + } catch (error) { + self.postMessage({ type: "error", message: String(error) }); + } + return; + } + handleBinaryMessage(event.data); + }; +} + +function handleBinaryMessage(buffer) { + const view = new DataView(buffer); + const msgType = view.getUint8(0); + + if (msgType === wsMsgRobotPose) { + if (buffer.byteLength < robotPoseHeaderBytes) return; + const count = view.getUint32(4, false); + const poseLength = count * 7; + const poseByteLength = poseLength * 4; + if (buffer.byteLength < robotPoseHeaderBytes + poseByteLength) return; + self.postMessage( + { + type: "robot_pose", + count, + time: view.getFloat64(8, false), + buffer, + }, + [buffer], + ); + return; + } + + // Any other tag is silently dropped — there are no remaining /ws binary + // frames besides robot_pose. +} + +self.onmessage = (event) => { + const message = event.data || {}; + if (message.type === "connect") { + connect(); + return; + } + if (message.type === "send_json") { + if (!socket || socket.readyState !== WebSocket.OPEN) return; + socket.send(JSON.stringify(message.payload)); + } +}; diff --git a/dimos/web/viewer/static/style.css b/dimos/web/viewer/static/style.css new file mode 100644 index 0000000000..acf70af352 --- /dev/null +++ b/dimos/web/viewer/static/style.css @@ -0,0 +1,776 @@ +/* + * Copyright 2025-2026 Dimensional Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import url("https://fonts.googleapis.com/css2?family=Inter+Tight:wght@400;500;600;700;800&family=Roboto+Mono:wght@400;500;600;700&display=swap"); + +:root { + color-scheme: dark; + --bg: #111414; + --panel: #1b1e1f; + --panel-2: #202425; + --panel-3: #303536; + --line: #3b4142; + --line-soft: rgb(176 225 240 / 18%); + --cyan: #b0e1f0; + --teal: #18cbb9; + --text: #f2f6f7; + --muted: #9da7aa; + --dim: #687174; + --danger: #ff6b61; +} + +* { + box-sizing: border-box; +} + +html, +body { + width: 100%; + height: 100%; + margin: 0; + overflow: hidden; + background: #0f1111; + color: var(--text); + font-family: + "Inter Tight", Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, + "Segoe UI", sans-serif; +} + +button, +input, +textarea { + font: inherit; +} + +button { + min-height: 34px; + border: 1px solid var(--line); + background: var(--panel-2); + color: var(--text); + cursor: pointer; + text-align: left; + transition: background 120ms ease, border-color 120ms ease, color 120ms ease, opacity 120ms ease; +} + +button:hover { + border-color: rgb(176 225 240 / 42%); + background: #263031; +} + +button:active { + transform: translateY(1px); +} + +button[data-active="true"] { + border-color: rgb(24 203 185 / 80%); + background: rgb(24 203 185 / 15%); + color: var(--cyan); +} + +button[data-active="false"] { + opacity: 0.58; +} + +.console-frame { + position: relative; + width: 100vw; + height: 100vh; + min-width: 1024px; + min-height: 720px; + overflow: hidden; + background: + linear-gradient(rgb(176 225 240 / 5%) 1px, transparent 1px), + linear-gradient(90deg, rgb(176 225 240 / 5%) 1px, transparent 1px), + radial-gradient(circle at 58% 42%, rgb(24 203 185 / 7%), transparent 360px), + var(--bg); + background-size: 72px 72px, 72px 72px, auto, auto; +} + +.console-top { + position: absolute; + left: 48px; + right: 48px; + top: 24px; + height: 33px; + display: grid; + grid-template-columns: 1fr auto 1fr; + align-items: start; + border-bottom: 1px solid var(--line); + font-family: "Roboto Mono", ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 10px; +} + +.brand { + color: var(--cyan); + font-weight: 800; + text-shadow: 0 0 16px rgb(176 225 240 / 35%); +} + +.console-title { + color: var(--muted); +} + +.console-version { + color: var(--muted); + text-align: right; +} + +.module { + position: absolute; + overflow: visible; + border: 1px solid var(--line); + background: rgb(27 30 31 / 94%); +} + +.module::before { + content: ""; + position: absolute; + inset: -4px; + z-index: 20; + pointer-events: none; + background: + linear-gradient(var(--text), var(--text)) 0 0 / 7px 7px no-repeat, + linear-gradient(var(--text), var(--text)) 100% 0 / 7px 7px no-repeat, + linear-gradient(var(--text), var(--text)) 0 100% / 7px 7px no-repeat, + linear-gradient(var(--text), var(--text)) 100% 100% / 7px 7px no-repeat; +} + +.label, +.tag, +.window-bar, +.badge, +.log-list, +.view-label, +.live-status, +.key, +.panel-header, +.session-line, +#status { + font-family: "Roboto Mono", ui-monospace, SFMono-Regular, Menlo, monospace; +} + +.label { + color: var(--muted); + font-size: 10px; + text-transform: uppercase; +} + +.value { + margin-top: 9px; + color: var(--text); + font-size: 17px; + line-height: 1; + white-space: nowrap; +} + +.cyan { + color: var(--cyan); +} + +.teal { + color: var(--teal); +} + +.top-stats { + position: absolute; + left: 48px; + right: 48px; + top: 88px; + height: 66px; + display: grid; + grid-template-columns: 160px 122px 130px 124px 126px 134px 130px 1fr; + gap: 4px; +} + +.top-stats .module { + position: relative; + inset: auto; +} + +.stat { + height: 66px; + padding: 10px 12px; +} + +.stop-button { + height: 66px; + display: grid; + place-items: center; + border-color: rgb(255 107 97 / 62%); + background: rgb(255 107 97 / 18%); + color: #fff; + font-size: 13px; + font-weight: 800; + text-align: center; +} + +.proof-window { + left: 48px; + top: 168px; + /* The 930px width / 490px height caps were tuned for ~1440p layouts and + pinned the viewport to a tiny frame on larger displays. Let it grow + into the available space: right sidebar is 394px wide with a 48px right + margin and ~70px gap (510px total), bottom controls strip is ~284px + tall, header is 168px (matches the ``top``). */ + width: calc(100vw - 510px); + height: calc(100vh - 284px - 168px); + min-width: 620px; + min-height: 360px; + overflow: hidden; + background: #121719; +} + +.window-bar { + position: absolute; + left: 0; + right: 0; + top: 0; + height: 32px; + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + padding: 0 12px; + border-bottom: 1px solid var(--line); + background: #363a3b; + color: var(--cyan); + font-size: 11px; +} + +#status { + min-width: 120px; + color: var(--muted); + text-align: right; +} + +.video-surface { + position: absolute; + left: 0; + right: 0; + top: 32px; + bottom: 0; + overflow: hidden; + background: + linear-gradient(rgb(176 225 240 / 5%) 1px, transparent 1px), + linear-gradient(90deg, rgb(176 225 240 / 5%) 1px, transparent 1px), + linear-gradient(145deg, #20292b 0%, #121719 54%, #070909 100%); + background-size: 48px 48px, 48px 48px, auto; +} + +#renderCanvas { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + display: block; + outline: none; + touch-action: none; +} + +.viewport-tags { + position: absolute; + left: 20px; + top: 20px; + display: flex; + gap: 10px; + pointer-events: none; +} + +.tag, +.view-label { + height: 33px; + display: grid; + align-items: center; + border: 1px solid var(--line-soft); + background: rgb(17 20 20 / 72%); + color: var(--text); + font-size: 10px; +} + +.tag { + min-width: 145px; + padding: 0 10px; +} + +.view-label { + position: absolute; + top: 20px; + right: 20px; + width: 165px; + justify-content: center; + color: var(--muted); + pointer-events: none; +} + +.reticle { + position: absolute; + left: 50%; + top: 53%; + z-index: 3; + width: 76px; + height: 76px; + margin-left: -38px; + margin-top: -38px; + border: 1px solid rgb(176 225 240 / 72%); + pointer-events: none; +} + +.reticle::before, +.reticle::after { + content: ""; + position: absolute; + background: rgb(176 225 240 / 72%); +} + +.reticle::before { + left: 50%; + top: -18px; + bottom: -18px; + width: 1px; +} + +.reticle::after { + top: 50%; + left: -18px; + right: -18px; + height: 1px; +} + +.live-status { + position: absolute; + left: 20px; + right: 20px; + bottom: 16px; + min-height: 56px; + display: grid; + grid-template-columns: 1fr auto auto auto; + gap: 28px; + align-items: center; + padding: 10px 14px; + border: 1px solid var(--line); + background: rgb(27 30 31 / 90%); + color: var(--muted); + font-size: 10px; + text-transform: uppercase; + pointer-events: none; +} + +.live-status strong { + color: var(--text); + font-family: "Inter Tight", Inter, ui-sans-serif, system-ui, sans-serif; + font-size: 15px; + font-weight: 800; + text-transform: none; +} + +.active { + color: var(--cyan); +} + +.right-stack { + position: absolute; + right: 48px; + top: 168px; + width: 394px; + /* Hard 676px cap removed - on tall displays the sidebar floated mid-screen. + Match the viewport's vertical extent. */ + height: calc(100vh - 284px - 168px); + display: grid; + grid-template-rows: 168px minmax(150px, 1fr) 168px; + gap: 16px; +} + +.right-stack .module { + position: relative; + inset: auto; + width: 100%; +} + +.patrol { + padding: 13px 14px; +} + +.session-line { + display: grid; + grid-template-columns: 1fr auto; + align-items: baseline; + gap: 12px; + height: 34px; + border-bottom: 1px solid rgb(176 225 240 / 12%); + color: var(--muted); + font-size: 12px; +} + +.session-line:first-of-type { + margin-top: 12px; +} + +.session-line strong { + color: var(--text); + font-weight: 500; +} + +.camera-feed { + min-height: 150px; + overflow: hidden; +} + +.camera-feed[data-active="false"], +.camera-feed[data-has-frame="false"].secondary { + display: none; +} + +.panel-header { + height: 32px; + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + padding: 0 12px; + border-bottom: 1px solid var(--line); + background: #363a3b; + color: var(--cyan); + font-size: 11px; + text-transform: uppercase; +} + +.panel-header button { + min-height: 22px; + padding: 0 8px; + font-size: 10px; +} + +.panel-note { + color: var(--muted); + font-size: 10px; +} + +#cameraImg, +#workspaceImg { + width: 100%; + height: calc(100% - 32px); + min-height: 120px; + display: block; + object-fit: cover; + background: #000; +} + +#cameraPanel[data-has-frame="false"] #cameraImg { + display: none; +} + +#cameraEmpty { + height: calc(100% - 32px); + min-height: 150px; + display: grid; + place-items: center; + color: var(--muted); + font-family: "Roboto Mono", ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 11px; + text-transform: uppercase; +} + +#cameraPanel[data-has-frame="true"] #cameraEmpty { + display: none; +} + +.log-panel { + padding: 14px; +} + +.log-list { + margin-top: 15px; + display: grid; + gap: 10px; + color: var(--muted); + font-size: 10px; +} + +.log-item { + display: grid; + grid-template-columns: 48px 1fr; + gap: 8px; +} + +.controls-drive { + left: 48px; + bottom: 48px; + width: 220px; + height: 168px; + padding: 14px; +} + +.keys { + position: absolute; + left: 67px; + top: 52px; + width: 126px; + display: grid; + grid-template-columns: repeat(3, 38px); + gap: 6px; +} + +.key { + width: 38px; + height: 38px; + display: grid; + place-items: center; + border: 1px solid var(--line); + background: #111515; + color: var(--cyan); + font-size: 12px; +} + +.key.blank { + border-color: transparent; + background: transparent; +} + +#toggleDrive { + position: absolute; + left: 14px; + right: 14px; + bottom: 14px; + text-align: center; +} + +.command-panel { + left: 284px; + bottom: 48px; + width: 360px; + height: 168px; + padding: 14px; +} + +.view-panel { + left: 660px; + width: 318px; + bottom: 48px; + height: 168px; + padding: 14px; +} + +.control-head { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; +} + +.badge { + height: 24px; + display: grid; + align-items: center; + padding: 0 9px; + border: 1px solid rgb(176 225 240 / 22%); + color: var(--cyan); + font-size: 10px; +} + +.action-grid, +.view-grid { + position: absolute; + left: 14px; + right: 14px; + top: 44px; + bottom: 14px; + display: grid; + gap: 6px; + align-content: start; +} + +.action-grid { + grid-template-columns: repeat(4, 1fr); +} + +.view-grid { + grid-template-columns: repeat(3, 1fr); +} + +.action-grid button, +.view-grid button { + min-height: 30px; + padding: 5px 7px; + font-size: 11px; + line-height: 1.05; + text-align: center; +} + +.arms-panel { + right: 48px; + top: 88px; + width: 420px; + max-width: calc(100vw - 96px); + max-height: calc(100vh - 176px); + overflow-y: auto; + z-index: 8; +} + +#armsPanel[data-active="false"] { + display: none; +} + +#armsColumns { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 14px; + padding: 14px 16px 16px; +} + +.arm-col-title { + margin-bottom: 6px; + padding: 4px 0; + background: rgb(255 255 255 / 4%); + color: rgb(255 255 255 / 80%); + font-size: 12px; + font-weight: 600; + text-align: center; +} + +.arm-sliders { + display: flex; + flex-direction: column; + gap: 8px; +} + +.arm-slider-row { + display: grid; + grid-template-columns: 1fr auto; + gap: 2px 6px; + align-items: center; +} + +.arm-slider-row .joint-name { + overflow: hidden; + color: rgb(255 255 255 / 75%); + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.arm-slider-row .joint-val { + min-width: 48px; + color: rgb(176 225 240 / 90%); + font-family: ui-monospace, "SF Mono", Menlo, monospace; + font-size: 10px; + font-variant-numeric: tabular-nums; + text-align: right; +} + +.arm-slider-row input[type="range"] { + grid-column: 1 / span 2; + width: 100%; + height: 4px; + appearance: none; + background: rgb(255 255 255 / 8%); + border-radius: 2px; + outline: none; +} + +.arm-slider-row input[type="range"]::-webkit-slider-thumb { + width: 12px; + height: 12px; + appearance: none; + background: var(--teal); + border: none; + border-radius: 50%; + cursor: pointer; +} + +.arm-slider-row input[type="range"]::-moz-range-thumb { + width: 12px; + height: 12px; + background: var(--teal); + border: none; + border-radius: 50%; + cursor: pointer; +} + +.arm-slider-row .joint-range { + grid-column: 1 / span 2; + display: flex; + justify-content: space-between; + color: rgb(255 255 255 / 35%); + font-size: 9px; + font-variant-numeric: tabular-nums; +} + +@media (max-width: 1180px) { + body { + overflow: auto; + } + + .console-frame { + min-width: 0; + min-height: 1120px; + overflow: visible; + } + + .console-top, + .top-stats, + .proof-window, + .right-stack, + .controls-drive, + .command-panel, + .view-panel { + position: relative; + left: auto; + right: auto; + top: auto; + bottom: auto; + width: calc(100vw - 32px); + min-width: 0; + margin: 16px; + } + + .console-top { + grid-template-columns: 1fr; + gap: 4px; + height: auto; + padding-bottom: 12px; + } + + .console-title, + .console-version { + text-align: left; + } + + .top-stats { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + height: auto; + } + + .proof-window { + height: 58vh; + } + + .right-stack { + display: grid; + grid-template-rows: none; + } + + .command-panel, + .view-panel, + .controls-drive { + height: auto; + min-height: 168px; + } + + .action-grid, + .view-grid { + position: static; + margin-top: 16px; + grid-template-columns: repeat(2, 1fr); + } +} diff --git a/dimos/web/viewer/static/ui.js b/dimos/web/viewer/static/ui.js new file mode 100644 index 0000000000..eeb8f80e7c --- /dev/null +++ b/dimos/web/viewer/static/ui.js @@ -0,0 +1,146 @@ +// Copyright 2025-2026 Dimensional Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +(() => { + const startedAt = Date.now(); + const cameraTargets = { + primary: { + img: document.getElementById("cameraImg"), + label: document.getElementById("cameraLabel"), + panel: document.getElementById("cameraPanel"), + lastUrl: null, + }, + workspace: { + img: document.getElementById("workspaceImg"), + label: document.getElementById("workspaceLabel"), + panel: document.getElementById("workspacePanel"), + lastUrl: null, + }, + }; + + const logEl = document.getElementById("sessionLog"); + const statusEl = document.getElementById("status"); + const streamLabel = document.getElementById("streamLabel"); + const driveStateLabel = document.getElementById("driveStateLabel"); + const clickModeLabel = document.getElementById("clickModeLabel"); + const entityStateLabel = document.getElementById("entityStateLabel"); + const videoLabel = document.getElementById("videoLabel"); + const runtimeLabel = document.getElementById("runtimeLabel"); + + function timeLabel(date = new Date()) { + return date.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }); + } + + function appendLog(message) { + if (!logEl || !message) return; + const item = document.createElement("div"); + item.className = "log-item"; + const ts = document.createElement("span"); + ts.textContent = timeLabel(); + const body = document.createElement("span"); + body.textContent = message; + item.append(ts, body); + logEl.prepend(item); + while (logEl.children.length > 6) { + logEl.lastElementChild?.remove(); + } + } + + let lastStatus = ""; + function setStatus(message) { + const text = String(message || ""); + if (statusEl) statusEl.textContent = text; + if (streamLabel) streamLabel.textContent = text || "Idle"; + if (text && text !== lastStatus) { + appendLog(text); + lastStatus = text; + } + } + + function setButtonActive(id, active) { + const button = document.getElementById(id); + if (!button) return; + button.dataset.active = active ? "true" : "false"; + if (id === "toggleDrive" && driveStateLabel) { + driveStateLabel.textContent = active ? "Enabled" : "Disabled"; + } + if ( + (id === "navClick" || id === "pointClick" || id === "graspClick" || id === "spawnClick") && + clickModeLabel + ) { + if (active) { + clickModeLabel.textContent = button.textContent.trim(); + return; + } + const anyActive = ["navClick", "pointClick", "graspClick", "spawnClick"].some( + (buttonId) => document.getElementById(buttonId)?.dataset.active === "true", + ); + if (!anyActive) clickModeLabel.textContent = "None"; + } + } + + function isButtonActive(id) { + return document.getElementById(id)?.dataset.active === "true"; + } + + function setPanelActive(id, active) { + const panel = document.getElementById(id); + if (panel) panel.dataset.active = active ? "true" : "false"; + } + + function updateCameraFrame(cameraName, buffer, jpegOffset) { + const jpegBytes = new Uint8Array(buffer, jpegOffset); + const blob = new Blob([jpegBytes], { type: "image/jpeg" }); + const url = URL.createObjectURL(blob); + const target = cameraName === "workspace" ? cameraTargets.workspace : cameraTargets.primary; + if (target.img) { + target.img.src = url; + if (target.lastUrl) URL.revokeObjectURL(target.lastUrl); + target.lastUrl = url; + } + if (target.label) target.label.textContent = cameraName; + if (target.panel) target.panel.dataset.hasFrame = "true"; + if (videoLabel) { + videoLabel.textContent = "LIVE"; + videoLabel.className = "active"; + } + } + + function setEntityStatus(message) { + if (entityStateLabel) entityStateLabel.textContent = message; + } + + function tickRuntime() { + if (!runtimeLabel) return; + const elapsed = Math.max(0, Date.now() - startedAt); + const totalSeconds = Math.floor(elapsed / 1000); + const hours = String(Math.floor(totalSeconds / 3600)).padStart(2, "0"); + const minutes = String(Math.floor((totalSeconds % 3600) / 60)).padStart(2, "0"); + const seconds = String(totalSeconds % 60).padStart(2, "0"); + runtimeLabel.textContent = `${hours}:${minutes}:${seconds}`; + } + + tickRuntime(); + window.setInterval(tickRuntime, 1000); + + window.DimosViewerUI = { + appendLog, + isButtonActive, + setButtonActive, + setEntityStatus, + setPanelActive, + setStatus, + updateCameraFrame, + }; +})(); diff --git a/pyproject.toml b/pyproject.toml index 62518a4777..8dcb0480d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,6 +224,8 @@ agents = [ web = [ "fastapi>=0.115.6", "sse-starlette>=2.2.1", + # Direct dep of dimos.web.lcm_bridge (fastapi pins a compatible version). + "starlette>=0.40", "uvicorn>=0.34.0", "jinja2>=3.1.6", "ffmpeg-python", @@ -623,4 +625,6 @@ ignore = [ "uv.lock", "*/package-lock.json", "dimos/web/dimos_interface/themes.json", + # The operator console frontend; splitting it would be artificial. + "dimos/web/viewer/static/app.js", ] diff --git a/uv.lock b/uv.lock index 493e30ce45..728d370dbf 100644 --- a/uv.lock +++ b/uv.lock @@ -1610,6 +1610,7 @@ all = [ { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "tensorboard" }, { name = "timm" }, { name = "torchreid" }, @@ -1650,6 +1651,7 @@ base = [ { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, { name = "uvicorn" }, @@ -1748,6 +1750,7 @@ unitree = [ { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, { name = "unitree-webrtc-connect" }, @@ -1781,6 +1784,7 @@ unitree-dds = [ { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, { name = "unitree-sdk2py-dimos" }, @@ -1799,6 +1803,7 @@ web = [ { name = "jinja2" }, { name = "soundfile" }, { name = "sse-starlette" }, + { name = "starlette" }, { name = "uvicorn" }, ] @@ -2046,6 +2051,7 @@ requires-dist = [ { name = "soundfile", marker = "extra == 'web'" }, { name = "sqlite-vec", specifier = ">=0.1.6" }, { name = "sse-starlette", marker = "extra == 'web'", specifier = ">=2.2.1" }, + { name = "starlette", marker = "extra == 'web'", specifier = ">=0.40" }, { name = "structlog", specifier = ">=25.5.0,<26" }, { name = "tensorboard", marker = "extra == 'misc'", specifier = "==2.20.0" }, { name = "terminaltexteffects", specifier = "==0.12.2" },