diff --git a/.gitmodules b/.gitmodules index 6bd7ad0c..f9ec1a7a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,6 @@ [submodule "packages/python-server-sdk"] path = packages/python-server-sdk url = https://github.com/fishjam-cloud/python-server-sdk.git +[submodule "api/composition"] + path = api/composition + url = git@github.com:fishjam-cloud/foundry.git diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 00000000..e6a839e3 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,49 @@ +// Rule reference: https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +// +// Prettier is the formatter of record in this repo, so every rule it already +// governs is turned off here. Leaving them on produces thousands of warnings +// that contradict `yarn format`. +{ + "config": { + "default": true, + + // Prose is deliberately left unwrapped: `.prettierrc` sets no `proseWrap`, + // so Prettier defaults to "preserve" and never reflows paragraphs. Wrapping + // would also make one-word edits reflow whole paragraphs in diffs. + "MD013": false, + + // MDX is JSX: , , and friends are not stray inline HTML. + "MD033": false, + + // Docusaurus frontmatter carries the title, so files legitimately start + // without a heading and may repeat the title as an H1. + "MD025": false, + "MD041": false, + + // House style: explanation pages open with an italic one-line subtitle. + "MD036": false, + + // Repeated headings are fine under different parents ("Clean up", + // "Prerequisites"), but not as siblings. + "MD024": { "siblings_only": true }, + + // Whitespace, list, and emphasis style: all owned by Prettier. + "MD004": false, + "MD007": false, + "MD012": false, + "MD022": false, + "MD031": false, + "MD032": false, + "MD046": false, + "MD048": false, + "MD050": false, + }, + "ignores": [ + // Generated by TypeDoc from SDK sources. + "docs/api/**", + // Snapshots of released versions; fix issues in `docs/` instead. + "versioned_docs/**", + "node_modules/**", + "build/**", + ], +} diff --git a/api/composition b/api/composition new file mode 160000 index 00000000..c6858eaf --- /dev/null +++ b/api/composition @@ -0,0 +1 @@ +Subproject commit c6858eaf840133de6edcd1831291f92b324d1504 diff --git a/docs/_common/compositions/idle-cleanup.mdx b/docs/_common/compositions/idle-cleanup.mdx new file mode 100644 index 00000000..96a4e307 --- /dev/null +++ b/docs/_common/compositions/idle-cleanup.mdx @@ -0,0 +1,6 @@ +:::warning[A composition bills for the whole time it exists] +With that guard off, cleanup waits for the outputs to fall silent too, so one that keeps +publishing is never cleaned up for you. Delete it as soon as you are finished, and do not +leave one running after a test. See +[Cost and lifecycle](../../explanation/compositions#cost-and-lifecycle). +::: diff --git a/docs/api/reference.md b/docs/api/reference.md index e10073e1..7eed625b 100644 --- a/docs/api/reference.md +++ b/docs/api/reference.md @@ -6,7 +6,7 @@ type: reference Describes APIs for direct interaction with Fishjam. -Fishjam publishes documentation for the Sandbox API and Fishjam Server APIs. +Fishjam publishes documentation for the Sandbox API, Fishjam Server APIs, and the Composition API. ## Sandbox API @@ -51,3 +51,15 @@ the first message that must be sent is an `AuthRequest`, with a valid Management Token. Next, you can should subscribe to notifications by sending `SubscribeRequest` event with `SERVER_NOTIFICATION` event type. + +## Compositions + +[Compositions](../explanation/compositions) are managed through the Composition API. All requests go to `https://rtc.fishjam.io`. + +### REST API + +[Composition REST API Reference](/api/compositions) + +Calls authenticate with your Management Token, from the [**Fishjam developer panel**](https://fishjam.io/app). + +Request bodies are snake_case and closed: a field the endpoint does not define is rejected with `422` rather than ignored, so a misspelling fails loudly instead of silently doing nothing. diff --git a/docs/explanation/compositions.mdx b/docs/explanation/compositions.mdx new file mode 100644 index 00000000..2a97a2b6 --- /dev/null +++ b/docs/explanation/compositions.mdx @@ -0,0 +1,112 @@ +--- +type: explanation +title: Compositions +sidebar_position: 4.5 +description: Understand compositions, the Fishjam feature that mixes multiple live media streams into a single composed output in real time. +--- + +# Compositions + +_Understanding real-time stream composition in Fishjam_ + +A **composition** mixes multiple live media streams into a single output stream in real time. You send inputs (WebRTC, RTMP, or MP4), describe how they should be laid out, and it produces one composed stream that it publishes to a destination of your choosing, all without running any rendering infrastructure yourself. + +Compositions are built on [Smelter](https://smelter.dev), the source-available real-time video compositing engine by [Software Mansion](https://swmansion.com), and are a native part of Fishjam: they authenticate with the same Fishjam credentials and can compose the peers of a [Fishjam room](./rooms) directly. + +## What you can build + +- **Multi-party layouts**: arrange the cameras of a conference or livestream into grids, side-by-sides, or picture-in-picture. +- **Branded streams**: overlay logos, captions, lower-thirds, and backgrounds on top of live video. +- **Cross-protocol bridging**: take WebRTC inputs and republish the composed result over RTMP, for example broadcasting a conference to YouTube or Twitch, or the other way round. + +## Core concepts + +A **composition** is a single running compositing session. You register three kinds of things into it: + +- **Inputs**: the live media sources being composed (WebRTC via WHIP/WHEP, RTMP, or MP4). +- **Outputs**: where the composed result is sent (WebRTC via WHIP, or RTMP). Each output carries a **scene** that describes the layout. +- **Renderers**: shared assets such as images and fonts you can place in a scene. + +A composition produces video, it does not serve it. There is nowhere to point a player at a composition, so it always needs an output aimed at a destination your viewers can connect to instead: a [livestream](./livestreams) they watch over WHEP, or an RTMP service such as YouTube. + +An output's scene can either be described directly in the API or rendered by a **template**: a React component, written with the layout components and the [`@fishjam-cloud/composition`](./../how-to/compositions/write-and-deploy-a-template) hooks, that updates the layout live as the room changes. + +## Cost and lifecycle + +A running composition holds a live rendering session for as long as it exists, and you are billed for that time whether or not anyone is watching. + +Two defaults keep that in check. A composition auto-starts, and it cleans itself up after five minutes in which none of its inputs carry any media. + +`cleanup_without_inputs: false` turns that guard off. It tightens the condition so cleanup needs both the inputs and the outputs to go quiet, which is what you want when inputs arrive late, such as a room whose peers have not joined yet, or when a composition legitimately has no inputs, such as an output that renders only text. The cost is that a composition whose output keeps publishing is no longer cleaned up for you, so anything created that way is yours to delete, and a forgotten one bills until you do. + +Delete a composition as soon as you are done with it: + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" +``` + +## Scenes + +Every video output carries a **scene**: a tree of components that describes how inputs, text, and images are arranged into the composed frame. Audio outputs carry an **audio scene** that describes which inputs are mixed together. + +### The video scene tree + +A video scene has a single `root` component. Each component has a `type` that determines how it lays out its children. + +```json +{ + "root": { + "type": "tiles", + "children": [ + { "type": "input_stream", "input_id": "camera_1" }, + { "type": "input_stream", "input_id": "camera_2" } + ] + } +} +``` + +The available component types are: + +| `type` | Purpose | +| -------------- | -------------------------------------------------------------------------------- | +| `input_stream` | Renders one registered input. Identified by `input_id`. | +| `view` | A container you position and style; the basic building block for custom layouts. | +| `tiles` | Automatically arranges its children into a grid. | +| `rescaler` | Fits a single child into a target area, preserving aspect ratio. | +| `text` | Renders a text string. | +| `image` | Renders a registered image. Identified by `image_id`. | + +Components nest freely: a `tiles` of `rescaler`s wrapping `input_stream`s, a `view` with a `text` caption over an `input_stream`, and so on. The styling and full property set of each component come from Smelter itself; the [Smelter HTTP API reference](https://smelter.dev/http-api/overview) documents every component and its properties. + +### The audio scene + +An audio scene lists the inputs to mix and, optionally, their relative volume: + +```json +{ + "inputs": [ + { "input_id": "camera_1" }, + { "input_id": "camera_2", "volume": 0.5 } + ] +} +``` + +`volume` defaults to `1.0`. Only the inputs you list are audible in the output. + +### Setting a scene + +You provide the initial scene when you register an output, under `video.initial` (a video scene) and `audio.initial` (an audio scene). See [Choose inputs and outputs](./../how-to/compositions/inputs-and-outputs) for the full output shape. + +### Changing a scene over time + +A scene is not fixed for the life of an output. You can replace it while the composition is running, immediately or at a chosen point on the composition timeline. + +Either you push those updates yourself, or you hand the job to a **template**: a React component that receives the live room state and re-renders as the room changes. See [Choose inputs and outputs](./../how-to/compositions/inputs-and-outputs) to send an update, or [Write and deploy a template](./../how-to/compositions/write-and-deploy-a-template) to build one. + +## Where to go next + +- [Compositions tutorial](./../tutorials/compositions): create your first composition end to end. +- [Write and deploy a template](./../how-to/compositions/write-and-deploy-a-template): build a React layout with the composition SDK. +- [Compose a Fishjam room](./../how-to/compositions/compose-a-fishjam-room): turn a room's peers into one composed stream. +- [Composition API](./../api/reference#compositions): the full REST surface. diff --git a/docs/how-to/compositions/_category_.json b/docs/how-to/compositions/_category_.json new file mode 100644 index 00000000..f0331dc1 --- /dev/null +++ b/docs/how-to/compositions/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Compositions", + "position": 5, + "link": { + "type": "generated-index", + "description": "Compose rooms and live streams into new outputs with templates, events, and multiple transports." + } +} diff --git a/docs/how-to/compositions/compose-a-fishjam-room.mdx b/docs/how-to/compositions/compose-a-fishjam-room.mdx new file mode 100644 index 00000000..e80d5003 --- /dev/null +++ b/docs/how-to/compositions/compose-a-fishjam-room.mdx @@ -0,0 +1,188 @@ +--- +type: how-to +sidebar_position: 2 +description: Forward a Fishjam room into a composition and render its peers with a live React template. +--- + +import IdleCleanup from "../../_common/compositions/idle-cleanup.mdx"; + +# Compose a Fishjam room + +A composition can include the peers of a [Fishjam room](./../../explanation/rooms). You forward the room's tracks to the composition with a single Fishjam API call: Fishjam then pushes each participant's media into the composition as inputs, and hooks from `@fishjam-cloud/composition` let a [template](./write-and-deploy-a-template) render one tile per participant and push the result to a [livestream](./../../explanation/livestreams). + +```text +Fishjam room (peers) ──forwarded──▶ composition (template) ──WHIP──▶ Fishjam livestream ──WHEP──▶ [viewers] +``` + +## Prerequisites + +- A [livestream](./../../explanation/livestreams) (or any other WHIP/RTMP destination) to publish the composed stream to. +- A template project scaffolded with the composition CLI (see [Write and deploy a template](./write-and-deploy-a-template)). + +The [Composition API](./../../api/reference#compositions) lives on `https://rtc.fishjam.io`, while rooms and livestreams live on the [Fishjam Server API](./../../api/reference#server). Both take the same Management Token: + +```bash +export COMPOSITION_URL="https://rtc.fishjam.io" +export FISHJAM_URL="https://fishjam.io/api/v1/connect/" +export TOKEN="" +``` + +## Step 1: Create a room and invite peers + +Compositions consume h264 video, so the room has to enforce that codec. It is the default, but set it explicitly so a change of default cannot break the composition later: + +```bash +curl -X POST "$FISHJAM_URL/room" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "roomType": "conference", "videoCodec": "h264" }' +``` + +The room id comes back under `data.room.id`. Save it: + +```bash +export ROOM_ID="" +``` + +Every participant needs their own peer token. Create one per person: + +```bash +curl -X POST "$FISHJAM_URL/room/$ROOM_ID/peer" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "type": "webrtc", "options": {} }' +``` + +Hand each `data.token` to a client and have it join, using [Connect to a room](./../client/connecting) or the [React quick start](./../../tutorials/react-quick-start). The composition renders whoever is publishing, so get at least one peer in with a camera on before you expect a picture. + +## Step 2: Write a room-aware template + +Hooks from `@fishjam-cloud/composition` give the template live room state; it re-renders automatically as participants join, leave, mute, or speak. + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { InputStream, Rescaler, Text, Tiles, View } from "@swmansion/smelter"; +import { usePeers, useSpeakingState } from "@fishjam-cloud/composition"; +import type { PeerWithStreams } from "@fishjam-cloud/composition"; + +type PeerMetadata = { displayName?: string }; + +function PeerTile({ peer }: { peer: PeerWithStreams }) { + const camera = peer.cameraStream; + const cameraOn = camera?.video && !camera.video.paused; + const speaking = useSpeakingState(peer.id) === "speech"; + const name = peer.metadata.peer?.displayName ?? peer.id; + + return ( + + {cameraOn ? ( + + + + ) : ( + + {name} + + )} + + ); +} + +export default function App() { + const peers = usePeers(); + const connected = peers.filter((peer) => peer.streams.length > 0); + + return ( + + + {connected.map((peer) => ( + + ))} + + + ); +} +``` + +| Hook | Returns | +| -------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `usePeers()` | All peers in the forwarded room, each with its streams (`cameraStream`, `screenShareStream`, `customStreams`). | +| `usePeer(peerId)` | A single peer, or `undefined`. | +| `useRoom()` | The forwarded room `{ id }`, or `undefined` before a room is forwarded. | +| `useSpeakingState(peerId)` | `"speech"` or `"silence"` for active-speaker highlighting. | + +The key link is `stream.inputId`: you pass it to `` to render that participant's forwarded track. A peer's streams fill in once its media actually starts flowing into the composition. + +Build the bundle as usual with `npm run build`. + +## Step 3: Create the composition and register the templated output + +Create the composition with auto-start off, because the room's inputs only appear once forwarding starts. +Setting `cleanup_without_inputs` to `false` tightens the cleanup condition, so that it now takes both the inputs and the outputs going quiet rather than the inputs alone. Cleanup therefore fires less readily, and the composition survives the wait for the first peer to publish. + + + +```bash +curl -X POST "$COMPOSITION_URL/api/composition" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "autostart": false, "cleanup_without_inputs": false }' +``` + +Save the `composition_id` from the response, every call below uses it: + +```bash +export COMPOSITION="" +``` + +Register a templated `whip_client` output that pushes to your livestream's WHIP endpoint. Create the livestream and its streamer token first, as in [Step 3 of the tutorial](./../../tutorials/compositions#step-3-create-a-livestream-to-publish-to). The output configuration and the template bundle go together in one multipart request: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/template" \ + -H "Authorization: Bearer $TOKEN" \ + -F 'config={ + "type": "whip_client", + "endpoint_url": "", + "bearer_token": "", + "video": { "resolution": { "width": 1280, "height": 720 }, "initial": { "root": { "type": "view" } } }, + "audio": { "initial": { "inputs": [] } } + };type=application/json' \ + -F "template=@dist/App.js" +``` + +## Step 4: Forward the room into the composition + +One call to the [Fishjam Server API](./../../api/reference#server) wires everything up: + +```bash +curl -X POST "$FISHJAM_URL/room/$ROOM_ID/track_forwardings" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d "{ \"compositionURL\": \"$COMPOSITION_URL/api/composition/$COMPOSITION\", \"selector\": \"all\" }" +``` + +Everything else happens automatically: Fishjam links the room to the composition, registers an input for every forwarded track, and streams the media in. Your template's `usePeers()` fills with the room's peers as their media starts flowing. You never register room inputs by hand. + +A room and a composition pair up one to one, in both directions. Repeating the call with the same `compositionURL` is a no-op, but pointing the room at a second composition, or a second room at this composition, fails. To compose two rooms, give each its own composition. + +## Step 5: Start the composition + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/start" \ + -H "Authorization: Bearer $TOKEN" +``` + +Viewers can now watch the composed grid through the livestream's WHEP endpoint. + +## Step 6: Clean up + +Delete the composition when you are done. Forwarding stops on the Fishjam side when the room itself stops, so delete the room too once you no longer need it. There is no separate call to remove a forwarding from a live room. + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" + +curl -X DELETE "$FISHJAM_URL/room/$ROOM_ID" \ + -H "Authorization: Bearer $TOKEN" +``` diff --git a/docs/how-to/compositions/drive-a-template-with-events.mdx b/docs/how-to/compositions/drive-a-template-with-events.mdx new file mode 100644 index 00000000..fd011930 --- /dev/null +++ b/docs/how-to/compositions/drive-a-template-with-events.mdx @@ -0,0 +1,105 @@ +--- +type: how-to +sidebar_position: 3 +description: Send custom events to a running composition to update a template's on-screen state at runtime. +--- + +import IdleCleanup from "../../_common/compositions/idle-cleanup.mdx"; + +# Drive a template with events + +Once a [template](./write-and-deploy-a-template) is running, you can push **custom events** to it from your backend to change what it shows, such as toggling a live badge or updating a caption, without re-uploading the template. + +## Handle events in the template + +Inside a template, subscribe to events with the `eventBus`. Each `on` call returns an unsubscribe function. Event names and payloads are entirely defined by your application. + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { useEffect, useState } from "react"; +import { Text, View } from "@swmansion/smelter"; +import { eventBus } from "@fishjam-cloud/composition"; + +export default function App() { + const [caption, setCaption] = useState(null); + const [live, setLive] = useState(false); + + useEffect(() => { + const unsubs = [ + eventBus.on<{ text: string }>("SET_CAPTION", ({ text }) => + setCaption(text), + ), + eventBus.on("CLEAR_CAPTION", () => setCaption(null)), + eventBus.on<{ live: boolean }>("SET_LIVE", ({ live }) => setLive(live)), + ]; + return () => unsubs.forEach((off) => off()); + }, []); + + return ( + + {live && ● LIVE} + {caption && {caption}} + + ); +} +``` + +## Deploy it + +Now create the composition. A caption-only template registers no inputs, and a composition with no input media cleans itself up after five minutes, so create this one with `cleanup_without_inputs` set to `false`: + +```bash +export COMPOSITION_URL="https://rtc.fishjam.io" +export TOKEN="" + +curl -X POST "$COMPOSITION_URL/api/composition" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "cleanup_without_inputs": false }' +``` + +Save the id from the response: + +```bash +export COMPOSITION="" +``` + + + +Build the bundle and register it as a templated output, the same single multipart request to `POST …/output/{output_id}/template` as in [Write and deploy a template](./write-and-deploy-a-template). + +That output has to publish somewhere, so create a livestream and take its streamer token first, as in [Step 3 of the compositions tutorial](./../../tutorials/compositions#step-3-create-a-livestream-to-publish-to). Watching it works the same way as there. + +## Send events from your backend + +Send an event with `POST /api/composition/{composition_id}/event`. The `event_name` is matched against your `eventBus.on(...)` subscriptions, and `data` is delivered as the handler's argument. + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "event_name": "SET_CAPTION", "data": { "text": "Welcome to the stream" } }' +``` + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/event" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "event_name": "SET_LIVE", "data": { "live": true } }' +``` + +The template re-renders as soon as the event arrives. + +:::note +Event names and payloads are an application-defined contract between your backend and your template. There is no canonical set of events; `SET_LIVE` and `SET_CAPTION` above are just examples. Pick names and payload shapes that fit your app, and keep them in sync on both sides. +::: + +## Clean up + +Delete the composition when you have finished testing, so it stops billing: + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" +``` diff --git a/docs/how-to/compositions/inputs-and-outputs.mdx b/docs/how-to/compositions/inputs-and-outputs.mdx new file mode 100644 index 00000000..a99d5791 --- /dev/null +++ b/docs/how-to/compositions/inputs-and-outputs.mdx @@ -0,0 +1,139 @@ +--- +type: how-to +sidebar_position: 4 +description: Choose the right input and output protocols for a composition, from WebRTC to RTMP and MP4. +--- + +# Choose inputs and outputs + +A composition pulls media in through **inputs** and pushes the composed result out through **outputs**. Each is a tagged object whose `type` selects the protocol. This guide summarizes the available types and when to use each. + +## Inputs + +Register an input with `POST /api/composition/{composition_id}/input/{input_id}/register` and a body whose `type` is one of: + +| `type` | Use it when | Key fields | +| ------------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `whip_server` | A WebRTC client should publish **to** the composition. | `bearer_token` (optional; generated and returned if omitted), `video` (optional, default `true`). | +| `whep_client` | The composition should **pull** a WebRTC stream from a WHEP endpoint. | `endpoint_url` (required), `bearer_token` (optional), `video` (optional). | +| `rtmp_server` | An encoder (OBS, hardware) should push RTMP in. | `app` (required), `stream_key` (required). | +| `mp4` | You want to compose a file, optionally looped. | `url` (required), `loop` (optional). | + +For `whip_server`, the register response returns the `bearer_token` a publisher uses to authenticate against the input's WHIP endpoint, together with the route to push to: + +```json +{ "bearer_token": "", "endpoint_route": "/whip/camera_1" } +``` + +That token is the only one accepted on the publish endpoint. Your Management Token is rejected there. + +`endpoint_route` is relative to the composition, so the full address is `https://rtc.fishjam.io/api/composition//whip/`. Point any WHIP publisher there and authenticate with that token. A `whip_server` input takes WebRTC from anything that speaks WHIP: a phone's camera, a laptop webcam, a browser tab, a screen share, or a hardware encoder. OBS has WHIP output built in. + +Unregister any input with `POST …/input/{input_id}/unregister`. + +## Outputs + +Register an output with `POST /api/composition/{composition_id}/output/{output_id}/register` and a body whose `type` is one of: + +| `type` | Use it when | Key fields | +| ------------- | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| `whip_client` | Publish the composed result over WebRTC (for example to a Fishjam livestream). | `endpoint_url` (required), `bearer_token` (optional), `video`, `audio`. | +| `rtmp_client` | Publish to an RTMP destination (for example a social platform). | `url` (required), `video`, `audio`. | + +There is no URL you can point a player at to watch a composition. An output **pushes** the composed video to a destination you name when you register it, and your viewers connect to that destination's own playback URL: + +```text +whip_client ──▶ Fishjam livestream ──WHEP──▶ [viewers] +rtmp_client ──▶ YouTube, Twitch, any RTMP service ──▶ [viewers] +``` + +An output's `video` carries the resolution and initial [scene](./../../explanation/compositions#scenes), and `audio` carries the initial audio scene: + +```json +{ + "type": "whip_client", + "endpoint_url": "https://example.com/whip", + "video": { + "resolution": { "width": 1280, "height": 720 }, + "initial": { "root": { "type": "tiles", "children": [] } } + }, + "audio": { "initial": { "inputs": [] } } +} +``` + +`resolution` has to be even on both sides, and at most 3840x2160. + +Other output operations: + +- **Register a templated output** with `POST …/output/{output_id}/template` instead of `…/register`: a multipart request carrying the same configuration plus a template bundle (see [Write and deploy a template](./write-and-deploy-a-template)). +- **Update the scene** live with `POST …/output/{output_id}/update` (see [Update a scene](#update-a-scene) below). +- **Force a keyframe** with `POST …/output/{output_id}/request_keyframe`, useful when a new subscriber joins. +- **Unregister** with `POST …/output/{output_id}/unregister`. + +## Update a scene + +Replace an output's [scene](./../../explanation/compositions#scenes) while the composition is running with `POST …/output/{output_id}/update`: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/update" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "video": { + "root": { + "type": "rescaler", + "child": { "type": "input_stream", "input_id": "camera_1" } + } + }, + "audio": { "inputs": [{ "input_id": "camera_1" }] } + }' +``` + +An update has to mirror the sides the output was registered with. If you registered the output with both `video` and `audio`, every update must carry both, even when only one of them changed. If you registered only one of them, the update may only carry that one. Sending a mismatched update fails. + +Add `schedule_time_ms` to apply the change at a chosen offset on the composition timeline, in milliseconds, instead of immediately: + +```json +{ + "video": { "root": { "type": "tiles", "children": [] } }, + "audio": { "inputs": [] }, + "schedule_time_ms": 5000 +} +``` + +## Renderers + +Renderers are shared assets you register once and then place in any scene. + +### Images + +Register an image with `POST …/image/{image_id}/register`: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/image/logo/register" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ "asset_type": "auto", "url": "https://example.com/logo.png" }' +``` + +`asset_type` selects the format: `png`, `jpeg`, `svg`, `gif`, or `auto` to detect it from the URL. Each takes a `url`; `svg` also accepts a `resolution`. Place the image in a scene with an `image` component: + +```json +{ "type": "image", "image_id": "logo" } +``` + +Unregister with `POST …/image/{image_id}/unregister`. + +### Fonts + +Register a font with a multipart request carrying the font file in a single `font` part: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/font/register" \ + -H "Authorization: Bearer $TOKEN" \ + -F "font=@BrandFont.ttf" +``` + +The font is then available to `text` components. + +See the [Composition API](./../../api/reference#compositions) reference for the complete request and response schemas. diff --git a/docs/how-to/compositions/write-and-deploy-a-template.mdx b/docs/how-to/compositions/write-and-deploy-a-template.mdx new file mode 100644 index 00000000..78858a3c --- /dev/null +++ b/docs/how-to/compositions/write-and-deploy-a-template.mdx @@ -0,0 +1,185 @@ +--- +type: how-to +sidebar_position: 1 +description: Write a React composition template, bundle it with the CLI, and deploy it as an output. +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Write and deploy a template + +A **template** is a React component that describes an output's layout. Instead of sending a static scene, you write a component, bundle it into a single file with the composition CLI, and upload it when registering an output. + +## Scaffold a project + +Create a new template project with the CLI: + +```bash npm2yarn +npx @fishjam-cloud/composition-cli init my-template +cd my-template +npm install +``` + +This generates a ready-to-build project: a `package.json` with `build` and `typecheck` scripts, a TypeScript config, and a starter `src/App.tsx`. + +## Write the template + +A template **default-exports a React component**. You lay out the composition with components from [`@swmansion/smelter`](https://smelter.dev/ts-sdk/overview). + +The example below builds a grid that maintains itself. `useInputStreams()` returns every input currently registered on the composition, so the layout follows them without you sending a single scene update: + +```tsx +// @jsx: react-jsx +// ---cut-before--- +import { + InputStream, + Rescaler, + Text, + Tiles, + View, + useInputStreams, +} from "@swmansion/smelter"; + +export default function App() { + const inputs = Object.values(useInputStreams()); + const playing = inputs.filter((input) => input.videoState === "playing"); + + if (playing.length === 0) { + return ( + + + Waiting for inputs + + + ); + } + + return ( + + + {playing.map((input) => ( + + + + ))} + + + ); +} +``` + +Register an input and a tile appears; unregister it and the grid reflows. Each entry also carries `videoState` and `audioState`, one of `ready`, `playing`, `paused`, or `finished`, which is what the filter above uses to keep inputs off screen until their media actually starts. + +`` renders one of the composition's registered inputs. Layout, styling, and every visual component (`View`, `Tiles`, `InputStream`, `Rescaler`, `Text`, `Image`, …) come from `@swmansion/smelter`. See the [Smelter TypeScript SDK reference](https://smelter.dev/ts-sdk/overview) for every component and its style props. + +## Build the bundle + +```bash npm2yarn +npm run build +``` + +## Deploy the template + +Now create the composition, as in [Step 1 of the tutorial](./../../tutorials/compositions#step-1-create-a-composition), and the livestream with its streamer token, as in [Step 3](./../../tutorials/compositions#step-3-create-a-livestream-to-publish-to): + +```bash +export COMPOSITION_URL="https://rtc.fishjam.io" +export FISHJAM_URL="https://fishjam.io/api/v1/connect/" +export TOKEN="" +export COMPOSITION="" +export STREAM="" +export STREAMER_TOKEN="" +``` + +A templated output is registered in a **single multipart request** to `POST …/output/{output_id}/template`. Do not call the plain `…/register` endpoint first. The request carries two parts: + +- `config`: the same JSON body a regular output registration takes (see [Choose inputs and outputs](./inputs-and-outputs)). Its `video.initial` scene is only a placeholder; the template takes over rendering as soon as it loads. +- `template`: the built bundle. + +`endpoint_url` is wherever the composed stream should go. The quickest destination to watch is a Fishjam livestream, exactly as in [the tutorial](./../../tutorials/compositions#step-3-create-a-livestream-to-publish-to): create one, take its streamer token, and publish to `https://fishjam.io/api/v1/live/api/whip`. + + + + +```bash +CONFIG=$(cat < + + +```js +const config = { + type: "whip_client", + endpoint_url: "https://fishjam.io/api/v1/live/api/whip", + bearer_token: streamerToken, + video: { + resolution: { width: 1280, height: 720 }, + initial: { root: { type: "view" } }, + }, + audio: { initial: { inputs: [] } }, +}; + +const form = new FormData(); +form.append("config", JSON.stringify(config)); +form.append("template", new Blob([bundle]), "App.js"); + +await fetch( + `${COMPOSITION_URL}/api/composition/${composition}/output/main/template`, + { + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + body: form, + }, +); +``` + + + + +## Redeploy after an edit + +Unregister the output first, then deploy again: + +```bash +curl -X POST "$COMPOSITION_URL/api/composition/$COMPOSITION/output/main/unregister" \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -d '{}' +``` + +Rebuild with `npm run build` and repeat the deploy request. The inputs stay registered, so only the layout changes. + +## Clean up + +Deleting the composition removes its inputs and outputs with it, so a finished experiment takes two calls: + +```bash +curl -X DELETE "$COMPOSITION_URL/api/composition/$COMPOSITION" \ + -H "Authorization: Bearer $TOKEN" + +curl -X DELETE "$FISHJAM_URL/livestream/$STREAM" \ + -H "Authorization: Bearer $TOKEN" +``` + +## Next steps + +To feed a whole Fishjam room's peers into the template, continue with [Compose a Fishjam room](./compose-a-fishjam-room). To update the template's on-screen state at runtime, see [Drive a template with events](./drive-a-template-with-events). diff --git a/docs/tutorials/compositions.mdx b/docs/tutorials/compositions.mdx new file mode 100644 index 00000000..489e8861 --- /dev/null +++ b/docs/tutorials/compositions.mdx @@ -0,0 +1,235 @@ +--- +type: tutorial +title: Compositions +sidebar_position: 3.5 +description: Create your first composition end to end, overlaying one video on another and watching the composed result in your browser. +--- + +import useBaseUrl from "@docusaurus/useBaseUrl"; + +# Compositions + +This tutorial walks you through creating your first composition. You will overlay a small player camera on top of full-screen gameplay, publish the result to a livestream, and watch it in your browser. + +Both sources are sample MP4 files, so you need no camera, no encoder, and no publishing tool to get a picture. Nothing else in the tutorial depends on that, so you can swap either one for a live camera afterwards. + +## What you'll build + +```text +[race.mp4] ──┐ + ├─▶ composition ──WHIP──▶ livestream ──WHEP──▶ [your browser] +[player.mp4] ──┘ +``` + +The composed frame puts the race full-screen with the player in a small overlay in the top-left corner, standard gaming stream layout: + +