From cfe12f0fa42450ec3e2cd0a4789ae29f2c1c1f55 Mon Sep 17 00:00:00 2001 From: Lengyue Date: Mon, 3 Aug 2026 21:16:37 +0000 Subject: [PATCH] docs: replace open-source self-hosting pages with enterprise guides The Self-Hosting section documented the open-source fish-speech local model setup, which is documented in its own project and made the enterprise offering ambiguous. Replace it with documentation for self-hosted Fish Audio Enterprise. Removed: local-setup, docker-deployment, running-inference (all three redirect to the new introduction so existing links keep working). Added: introduction, requirements, registry-access, kubernetes, all-in-one, air-gapped, operations. Connection values (registry host, chart base, login username, team ID) and deploy tokens are rendered per team in the dashboard under Developer -> Self Host, so the docs never print them and use placeholders instead. Also adds a /self-host redirect for the dashboard's documentation link. Co-Authored-By: Claude Fable 5 --- developer-guide/self-hosting/air-gapped.mdx | 297 +++++++++++++ developer-guide/self-hosting/all-in-one.mdx | 217 +++++++++ .../self-hosting/docker-deployment.mdx | 343 -------------- developer-guide/self-hosting/introduction.mdx | 163 +++++++ developer-guide/self-hosting/kubernetes.mdx | 321 ++++++++++++++ developer-guide/self-hosting/local-setup.mdx | 155 ------- developer-guide/self-hosting/operations.mdx | 120 +++++ .../self-hosting/registry-access.mdx | 147 ++++++ developer-guide/self-hosting/requirements.mdx | 144 ++++++ .../self-hosting/running-inference.mdx | 419 ------------------ docs.json | 47 +- 11 files changed, 1436 insertions(+), 937 deletions(-) create mode 100644 developer-guide/self-hosting/air-gapped.mdx create mode 100644 developer-guide/self-hosting/all-in-one.mdx delete mode 100644 developer-guide/self-hosting/docker-deployment.mdx create mode 100644 developer-guide/self-hosting/introduction.mdx create mode 100644 developer-guide/self-hosting/kubernetes.mdx delete mode 100644 developer-guide/self-hosting/local-setup.mdx create mode 100644 developer-guide/self-hosting/operations.mdx create mode 100644 developer-guide/self-hosting/registry-access.mdx create mode 100644 developer-guide/self-hosting/requirements.mdx delete mode 100644 developer-guide/self-hosting/running-inference.mdx diff --git a/developer-guide/self-hosting/air-gapped.mdx b/developer-guide/self-hosting/air-gapped.mdx new file mode 100644 index 0000000..c4e048c --- /dev/null +++ b/developer-guide/self-hosting/air-gapped.mdx @@ -0,0 +1,297 @@ +--- +title: "Air-gapped deployments" +description: "Mirror artifacts into a disconnected network, account for usage offline, and prove zero egress" +icon: "network-wired" +--- + +The offline delivery forms run with no outbound network access at all: model +assets are served from inside the deployment and usage is recorded to a local +signed ledger rather than a billing service. Getting there takes one preparation +step — moving the artifacts across the air gap — and one verification step. + +| Delivery form | What has to cross the air gap | +| ------------- | -------------------------------------------------------------------------- | +| All-in-One | One container image. Model weights are already inside it. | +| Offline Helm | The chart archive, every image the chart references, and your values file. | + +Everything below runs on a **connected staging machine** first, then on the +disconnected side. Only the staging machine needs +[registry access](/developer-guide/self-hosting/registry-access). + +## Mirror container images + +On the connected machine, derive the image list from the chart you are about to +install, so the list always matches the release: + +```bash +CHART_REF='' +CHART_VERSION='' + +helm pull "$CHART_REF" --version "$CHART_VERSION" --destination ./transfer + +helm template fish-audio ./transfer/*.tgz \ + --namespace fish-audio \ + --values values.yaml \ + | grep -oE 'image:[[:space:]]*"?[^"[:space:]]+' \ + | awk '{print $2}' | tr -d '"' | sort -u > images.txt +``` + +Review `images.txt`, then pull and pack the images. `zstd` keeps the transfer +archive small; `gzip` works too if `zstd` is not available on both sides. + +```bash +xargs -n1 docker pull < images.txt + +docker save $(tr '\n' ' ' < images.txt) | zstd -T0 -o fish-audio-images.tar.zst +``` + +For the All-in-One image the same pattern applies with a single reference: + +```bash +docker save '' | zstd -T0 -o all-in-one.tar.zst +``` + +Transfer `fish-audio-images.tar.zst`, `images.txt`, the chart archive from +`./transfer/`, and your values file across the air gap by whatever means your +policy allows. + +## Load on the disconnected side + +```bash +zstd -dc fish-audio-images.tar.zst | docker load +``` + +For a Kubernetes install, push the loaded images into the registry your cluster +can reach: + +```bash +INTERNAL_REGISTRY='' + +while read -r image; do + target="$INTERNAL_REGISTRY/${image#*/}" + docker tag "$image" "$target" + docker push "$target" +done < images.txt +``` + +Then point the chart at your mirror by overriding each component's +`image.repository` in your values file, and confirm nothing still refers to an +external host before you install: + +```bash +helm template fish-audio ./fish-audio-chart.tgz \ + --namespace fish-audio \ + --values values.yaml \ + | grep -E 'image:' | sort -u +``` + +Install from the local chart archive exactly as described in +[Kubernetes deployment](/developer-guide/self-hosting/kubernetes#install), +substituting the archive path for the chart reference. The chart archive is +self-contained, so no chart repositories are contacted during the install. + +For the All-in-One container, the loaded image is all you need — continue with +[Run](/developer-guide/self-hosting/all-in-one#run). + +## Model assets offline + +The offline forms never download model weights at runtime. + +| Form | How models are served | +| ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Offline Helm | `global.offlineMode: true` starts an in-cluster model warehouse that serves the bundled weights over an S3-compatible endpoint inside the namespace. Object paths are unchanged; only the endpoint moves. | +| All-in-One | Weights are baked into the image. | + +Two things still come from you rather than from the network: + +- **Reference voices.** Stage the archives yourself, as described in + [Kubernetes deployment](/developer-guide/self-hosting/kubernetes#stage-reference-voices) + or [All-in-One](/developer-guide/self-hosting/all-in-one#reference-voices). + This behavior is identical online and offline. +- **Timestamp alignment.** The forced aligner is not part of the offline bundle. + Contact Fish Audio if your air-gapped deployment needs word or segment timings. + +## Offline usage accounting + +With no billing service to call, the deployment records every charge to a local, +signed, append-only ledger on persistent storage. No request is ever rejected for +billing reasons, and the record is tamper-evident and independently verifiable. + +### How tokens behave + +- Any non-empty `Authorization: Bearer` token is accepted. Empty or missing still + returns 401. +- The token is recorded verbatim as the billing identity, so choose a stable, + distinct value per tenant. Tenants that share a token cannot be told apart in + the ledger. + +### Disk layout + +Files are grouped one directory per UTC day, so a day archives by copying a single +directory: + +```text +offline-billing-ledger/ + 2026-06-11/ + 2026-06-11__.event.jsonl # append-only signed records + 2026-06-11__.manifest.json # signed per-file summary + 2026-06-12/ + ... +``` + +Each file is written by exactly one process, so replicas never interleave into the +same file even though they share the directory. Every file is an independent hash +chain: a new day, a restart, or a new replica starts a new file. + +### Record format + +Each line is one record: a shared envelope plus a payload chosen by `event_type` +(`process_started`, `heartbeat`, `billing_event`, `process_stopping`). + +| Envelope field | Meaning | +| ---------------------------------------- | ------------------------------------------------------------- | +| `version` | Ledger format version. | +| `instance_id`, `run_id` | Which instance and which process run wrote the record. | +| `seq` | 1-based and contiguous within the file. | +| `wall_time_utc_ms`, `wall_time_utc_date` | UTC timestamp and date. | +| `monotonic_ms_since_start` | Monotonic clock since process start, resistant to clock skew. | +| `prev_hash` | Hash of the previous record, `"0"` for the first. | +| `event_type` | Record discriminator. | +| `record_hash` | SHA-256 over all fields above. | +| `signature` | RSA-PSS-SHA256 over `record_hash`, base64. | + +| `billing_event` field | Meaning | +| ----------------------------- | ----------------------------------------------------------- | +| `team_id` | The bearer token the request used. | +| `product` | The product consumed, for example `tts`. | +| `backend` | The model backend that served the request. | +| `quantity` | Units billed in this aggregation bucket. | +| `unit_price_usd_per_1m_chars` | Contract unit price, or `null` for unpriced products. | +| `amount_usd` | Exact decimal string computed from quantity and unit price. | + +Charges accumulate in Redis per token, product, and backend, and a background task +flushes them roughly once a minute as one aggregated `billing_event` per bucket. A +failed write is retried rather than dropped. + +When a file is finished it is sealed into a matching signed manifest carrying the +last record's hash and signature, the record counts, and the file's totals. +Sealing is automatic: yesterday's file is sealed shortly after the UTC day rolls +over, and a graceful shutdown seals the current one. Today's in-progress file has +no manifest yet. + +### Verify the ledger + +Verification needs only the public key that Fish Audio supplies, plus `jq` and +`openssl`: + +```bash +PUB=offline-ledger-signing-key.pub.pem +LINE=$(head -n 1 '.event.jsonl') + +printf '%s' "$LINE" | jq -r .record_hash > /tmp/msg +printf '%s' "$LINE" | jq -r .signature | openssl base64 -d -A > /tmp/sig + +openssl dgst -sha256 -verify "$PUB" \ + -sigopt rsa_padding_mode:pss -sigopt rsa_pss_saltlen:-3 \ + -signature /tmp/sig /tmp/msg +``` + +`Verified OK` confirms the record is authentic. Recompute the hash itself from the +record body and compare it to the `record_hash` field: + +```bash +printf '%s' "$LINE" | jq -c 'del(.record_hash,.signature)' | openssl dgst -sha256 +printf '%s' "$LINE" | jq -r .record_hash +``` + +A full audit additionally checks that `seq` is contiguous, that each `prev_hash` +links the previous record, that the manifest signature verifies, and that the +manifest totals match the event file. Ask Fish Audio for the verification script +that runs all of these over a ledger directory. + + + The ledger is tamper-evident: signature and hash-chain checks reliably detect + corruption and modification of the files. If your audit requirements call for + stronger guarantees, such as an independent write-once anchor or a separately + administered audit sink, raise it with Fish Audio so it can be designed into + the deployment. + + +### Retention and reconciliation + +- The ledger lives on persistent shared storage and survives restarts and + rescheduling. +- Nothing prunes it. Archive completed day directories to your own storage on your + retention schedule, and never delete the live directory out from under a running + instance. +- Usage is reconciled afterwards from the signed day directories, on the cadence + set in your agreement. Copy whole directories, including the manifests, so the + totals can be verified independently. + +## Prove there is no egress + +Configuration review is not proof. Demonstrate it. + +### Kubernetes + +Deny external egress for the namespace, keeping in-cluster traffic and DNS, then +confirm the deployment still generates audio. + +```yaml deny-external-egress.yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: deny-external-egress + namespace: fish-audio +spec: + podSelector: {} + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: {} +``` + +```bash +kubectl apply -f deny-external-egress.yaml +``` + +Re-run the [smoke test](/developer-guide/self-hosting/kubernetes#smoke-test). It +must still return playable audio. This proves nothing unless your CNI actually +enforces NetworkPolicy, so confirm enforcement with a deliberate control: exec +into a pod in the namespace and check that an outbound request fails. + +### All-in-One + +The strongest proof is a container that never had a network interface, on a fresh +volume, so nothing could have been fetched even during the first compile: + +```bash +docker volume create fish-tts-airgap + +docker run -d --name fish-tts-airgap \ + --gpus '"device=0,1"' \ + --shm-size 16g --ulimit memlock=-1 --ulimit stack=67108864 \ + -v fish-tts-airgap:/mnt/shared \ + --network none \ + '' +``` + +With `--network none` no host port can be published, so drive the request from +inside the container once the cold start finishes: + +```bash +docker exec fish-tts-airgap curl -s -m 120 -X POST http://127.0.0.1:8088/v1/tts \ + -H 'Authorization: Bearer airgap' \ + -H 'Content-Type: application/json' \ + -H 'model: ' \ + -d '{"text":"offline","format":"mp3"}' \ + -o /tmp/offline.mp3 -w '%{http_code}\n' +``` + +A `200` with a non-trivial audio file is the proof. A weaker but faster variant +disconnects an already-warmed container from every Docker network and repeats the +request; it demonstrates that the running service survives losing the network, but +not that the cold start never needed it. + +Capture the output of whichever check you run as deployment evidence. diff --git a/developer-guide/self-hosting/all-in-one.mdx b/developer-guide/self-hosting/all-in-one.mdx new file mode 100644 index 0000000..a0d99d9 --- /dev/null +++ b/developer-guide/self-hosting/all-in-one.mdx @@ -0,0 +1,217 @@ +--- +title: "All-in-One container" +description: "Run the entire Fish Audio Enterprise stack from a single docker run" +icon: "box" +--- + +The All-in-One image packages the whole speech stack — edge API, model API layer, +inference router and worker, vocoder, text normalizer, and Redis — into one +container, with every model weight baked in. Once the image is on the host it runs +with no Kubernetes and no internet access, which makes it the turnkey option for +single-node appliances and strict air gaps. + + + The All-in-One image runs a single inference worker across two GPUs. It does + not autoscale or shard across more GPUs or nodes, and it does not ship the + forced aligner, so it serves no word or segment timings. For elastic, + multi-tenant, or higher aggregate throughput deployments, use the [Kubernetes + chart](/developer-guide/self-hosting/kubernetes), which scales replicas across + all GPUs and nodes. This image is also offline only; there is no + hosted-billing variant of it. + + +## Prerequisites + +- A host that meets the [All-in-One host requirements](/developer-guide/self-hosting/requirements#all-in-one-container-host). +- [Registry access](/developer-guide/self-hosting/registry-access), unless you are loading the image from a transfer archive. +- The image reference from **Granted Artifacts** in the dashboard. + +```bash +AIO_IMAGE='' +``` + +## Load the image + +On a host with registry access: + +```bash +docker pull "$AIO_IMAGE" +``` + +On a disconnected host, transfer the image instead — see +[Air-gapped deployments](/developer-guide/self-hosting/air-gapped#mirror-container-images). + +## Run + +Generate a JWT secret once, store it, and reuse the same value on every run. A new +value invalidates tokens and sessions issued under the old one. + +```bash +export FISH_JWT_SECRET="$(openssl rand -hex 32)" +``` + +```bash +docker run -d --name fish-tts \ + --gpus all \ + --shm-size 16g --ulimit memlock=-1 --ulimit stack=67108864 \ + -p 8088:8088 \ + -v fish-tts-shared:/mnt/shared \ + -e JWT_SECRET="$FISH_JWT_SECRET" \ + --restart unless-stopped \ + "$AIO_IMAGE" +``` + +| Flag | Why | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--gpus all` | The container pins the inference worker to the first GPU and the vocoder to the second. On a host with more than two GPUs it uses the first two; pin specific cards with `--gpus '"device=0,1"'`. | +| `--shm-size 16g` and the `--ulimit` flags | Shared memory and locked-memory limits the inference stack needs. | +| `-p 8088:8088` | The API is the only exposed port. | +| `-v fish-tts-shared:/mnt/shared` | One persistent volume for everything that must survive restarts. | +| `-e JWT_SECRET` | Required in production. Without it the container falls back to a fixed development default, which is not secret. | +| `--restart unless-stopped` | Restarts the container if it exits. Inside the container, the supervisor already restarts individual crashed services. | + +Everything in the container runs as a non-root user (UID 1000). A fresh named +volume inherits the right ownership; a reused volume or a host bind mount must be +writable by UID 1000. + +## First start and readiness + +On the first cold start the worker compiles its inference graphs and the vocoder +builds its engine. Expect roughly ten minutes once the image is on the host, and +considerably longer on a fully cold host that also has to transfer the image. +Subsequent starts on the same volume take minutes, because both artifacts are +cached on the volume. + +`GET /health` verifies the speech backend end to end, so it stays unhealthy +through the warmup rather than reporting immediate liveness. Full readiness is a +generation that returns audio: + +```bash +until curl -fsS -m 120 -X POST http://127.0.0.1:8088/v1/tts \ + -H 'Authorization: Bearer my-tenant' \ + -H 'Content-Type: application/json' \ + -H 'model: ' \ + -d '{"text":"ready","format":"mp3"}' -o /tmp/ready.mp3; do + echo "warming up..."; sleep 15 +done; echo "ready" +``` + +Follow the startup with `docker logs -f fish-tts`. + +## Make requests + +```bash +curl -X POST http://127.0.0.1:8088/v1/tts \ + -H 'Authorization: Bearer my-tenant' \ + -H 'Content-Type: application/json' \ + -H 'model: ' \ + -d '{"text":"Hello from Fish Audio Enterprise.","format":"mp3"}' \ + -o out.mp3 +``` + +Supported `format` values are `mp3`, `wav`, `pcm`, and `opus`. Optional fields +include `reference_id`, `mp3_bitrate`, `sample_rate`, `latency`, and +`chunk_length`. + +For the lowest time-to-first-audio, stream over WebSocket at +`ws://:8088/v1/tts/live` with the same `Authorization` and `model` headers, +then send msgpack events: `start`, one or more `text`, then `stop`, and read +`audio` events until `finish`. The payloads match the +[hosted WebSocket API](/api-reference/endpoint/websocket/tts-live). + +### Latency modes + +| Mode | Time-to-first-audio | Use for | +| ---------- | ------------------------------------------ | ---------------------------- | +| `normal` | Highest; emits near the end of generation | File and batch generation | +| `balanced` | Low; the recommended default | Interactive and real-time | +| `low` | Lowest; chunks long text more aggressively | Latency-critical interactive | + +Set it in the request body as `"latency":"balanced"`, or in the WebSocket `start` +event. + +## Reference voices + +Reference-id requests resolve only from a local archive. Place one zip per voice +at `/mnt/shared/reference-archives/.zip`, containing audio files at +the zip root each paired with a same-basename `.txt` transcript. A named Docker +volume has no stable host path, so copy archives in: + +```bash +docker cp my-voice.zip fish-tts:/mnt/shared/reference-archives/my-voice.zip +``` + +Alternatively, bind-mount a host directory at +`/mnt/shared/reference-archives` and drop archives into it directly. Then request +with `"reference_id":"my-voice"`. + +## Authentication and usage + +This build records usage to a local, signed, append-only ledger instead of calling +a billing service: + +- Any non-empty `Authorization: Bearer` token is accepted. A missing or empty + token returns 401. +- The token is recorded verbatim as the billing identity, so use a stable, + distinct token per tenant. Two tenants sharing a token are indistinguishable in + the ledger. +- The ledger is written under `/mnt/shared/offline-billing-ledger/` as signed JSON + Lines, one directory per UTC day. + +See [Offline usage accounting](/developer-guide/self-hosting/air-gapped#offline-usage-accounting) +for the record format, verification, and reconciliation. + +## Persistence + +| Path | Contents | +| ------------------------------------- | ---------------------------------------------------- | +| `/mnt/shared/cache/` | Compile and graph caches for the worker and vocoder. | +| `/mnt/shared/checkpoints/` | The vocoder's built inference engine. | +| `/mnt/shared/reference-archives/` | Reference voice archives. | +| `/mnt/shared/offline-billing-ledger/` | The signed usage ledger. | + +Model weights live in the image, not on the volume. Keep `/mnt/shared` on +persistent storage: without it, every restart pays the full first-start compile +again and the ledger is lost. The vocoder engine is specific to the GPU model, so +moving to different cards rebuilds it once. + +If your platform pins persistent storage somewhere other than `/mnt/shared`, you +can relocate the three compile caches with `COMPILE_CACHE_DIR`, +`TORCHINDUCTOR_CACHE_DIR`, and `VQ_CACHE_DIR`. Point each at a **separate** +subdirectory that is writable by UID 1000, and keep the `/mnt/shared` mount either +way, because reference voices and the ledger always live there. + +## Capacity + +The single worker admits a bounded number of in-flight requests, set by +`LIMIT_MODEL_CONCURRENCY` (default 32); beyond that, requests queue. The ceiling +is bounded by the inference worker's key-value cache VRAM, so cards with more +memory than the 32 GB baseline can run a higher cap. Raise it at launch without +rebuilding: + +```bash +docker run -e LIMIT_MODEL_CONCURRENCY=64 ... +``` + +Validate latency and error rate at the new value before committing to it. + +## Operations + +| Task | Command | +| --------------- | ----------------------------------------------------------------------------- | +| Logs | `docker logs -f fish-tts` — all services interleaved. | +| Health | `curl -fsS http://127.0.0.1:8088/health` | +| GPU check | `nvidia-smi` — expect the first GPU heavily used and the second lightly used. | +| Stop and remove | `docker rm -f fish-tts` | + +## Troubleshooting + +| Symptom | Cause and fix | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Errors or not-ready responses for the first several minutes after start | The worker and vocoder are still compiling on a cold start. Wait, and keep the cache volume so it does not repeat. | +| Vocoder never becomes ready, or every request fails after warmup | Check `docker logs fish-tts` for vocoder startup errors. The container manages its own GPU multi-process service internally; no host-side setup is required. | +| `401` | Missing or empty `Authorization: Bearer` header. | +| `Reference not found` | No matching `.zip` under `/mnt/shared/reference-archives`. | +| Time-to-first-audio climbs under load | Concurrency exceeds the single worker's capacity and requests queue. Reduce concurrency, raise the cap if the GPUs have headroom, or scale out with the Helm chart. | +| `could not select device driver ... gpu` | NVIDIA Container Toolkit is not configured. Run `nvidia-ctk runtime configure --runtime=docker` and restart Docker. | +| `Permission denied` writing the cache or ledger | The container runs as UID 1000. A reused volume or host bind mount must be writable by that UID: `chown -R 1000:1000 /path/to/dir`. | diff --git a/developer-guide/self-hosting/docker-deployment.mdx b/developer-guide/self-hosting/docker-deployment.mdx deleted file mode 100644 index ed8646a..0000000 --- a/developer-guide/self-hosting/docker-deployment.mdx +++ /dev/null @@ -1,343 +0,0 @@ ---- -title: "Docker Deployment" -description: "Deploy Fish Audio models using Docker containers" -icon: "docker" ---- -import { AudioTranscript } from '/snippets/audio-transcript.jsx'; - -{/* speak-mintlify-hash: 7519d2e10765e9a8af2318d43268e2c1f0fe2db4478dadeec9afa3068c0dcb98 */} - - - - - -Fish Audio provides Docker images for both WebUI and API server deployments. You can use pre-built images from Docker Hub or build custom images locally. - -## Prerequisites - -Before deploying with Docker, ensure you have: - -- **Docker** and **Docker Compose** installed -- **NVIDIA Docker runtime** (for GPU support) -- At least **12GB GPU memory** for CUDA inference -- Downloaded model weights (see [Running Inference](/developer-guide/self-hosting/running-inference#download-weights)) - -## Pre-built Images - -Fish Audio provides ready-to-use Docker images on Docker Hub: - -| Image | Description | Best For | -|-------|-------------|----------| -| `fishaudio/fish-speech:latest-webui-cuda` | WebUI with CUDA support | Interactive development with GPU | -| `fishaudio/fish-speech:latest-webui-cpu` | WebUI CPU-only | Testing without GPU | -| `fishaudio/fish-speech:latest-server-cuda` | API server with CUDA | Production deployments with GPU | -| `fishaudio/fish-speech:latest-server-cpu` | API server CPU-only | Low-traffic CPU deployments | - - -For production use, we recommend using specific version tags instead of `latest` to ensure consistency across deployments. - - -## Quick Start with Docker Run - -The fastest way to get started is using `docker run`: - -### WebUI Deployment - -```bash -# Create directories for model weights and reference audio -mkdir -p checkpoints references - -# Start WebUI with CUDA support (recommended) -docker run -d \ - --name fish-speech-webui \ - --gpus all \ - -p 7860:7860 \ - -v ./checkpoints:/app/checkpoints \ - -v ./references:/app/references \ - -e COMPILE=1 \ - fishaudio/fish-speech:latest-webui-cuda - -# For CPU-only deployment -docker run -d \ - --name fish-speech-webui-cpu \ - -p 7860:7860 \ - -v ./checkpoints:/app/checkpoints \ - -v ./references:/app/references \ - fishaudio/fish-speech:latest-webui-cpu -``` - -Access the WebUI at `http://localhost:7860` - -### API Server Deployment - -```bash -# Start API server with CUDA support -docker run -d \ - --name fish-speech-server \ - --gpus all \ - -p 8080:8080 \ - -v ./checkpoints:/app/checkpoints \ - -v ./references:/app/references \ - -e COMPILE=1 \ - fishaudio/fish-speech:latest-server-cuda - -# For CPU-only deployment -docker run -d \ - --name fish-speech-server-cpu \ - -p 8080:8080 \ - -v ./checkpoints:/app/checkpoints \ - -v ./references:/app/references \ - fishaudio/fish-speech:latest-server-cpu -``` - -Access the API documentation at `http://localhost:8080` - - -Enable the `COMPILE=1` environment variable for ~10x faster inference on CUDA deployments. This uses `torch.compile` to optimize the model. - - -## Docker Compose Deployment - -For development or customization, Docker Compose provides easier configuration management: - -### Setup - -```bash -# Clone the repository -git clone https://github.com/fishaudio/fish-speech.git -cd fish-speech -``` - -### Start Services - -```bash -# Start WebUI with CUDA -docker compose --profile webui up - -# Start WebUI with compile optimization -COMPILE=1 docker compose --profile webui up - -# Start API server -docker compose --profile server up - -# Start API server with compile optimization -COMPILE=1 docker compose --profile server up - -# For CPU-only deployment -BACKEND=cpu docker compose --profile webui up -``` - - -Run containers in detached mode by adding the `-d` flag: `docker compose --profile webui up -d` - - -### Environment Variables - -Customize deployment using environment variables or a `.env` file: - -```bash -# .env file example -BACKEND=cuda # or cpu -COMPILE=1 # Enable compile optimization -GRADIO_PORT=7860 # WebUI port -API_PORT=8080 # API server port -UV_VERSION=0.8.15 # UV package manager version -``` - -## Manual Docker Build - -For advanced users who need custom configurations: - -### Build WebUI Image - -```bash -# Build with CUDA support -docker build \ - --platform linux/amd64 \ - -f docker/Dockerfile \ - --build-arg BACKEND=cuda \ - --build-arg CUDA_VER=12.6.0 \ - --build-arg UV_EXTRA=cu126 \ - --target webui \ - -t fish-speech-webui:cuda . - -# Build CPU-only (supports multi-platform) -docker build \ - --platform linux/amd64,linux/arm64 \ - -f docker/Dockerfile \ - --build-arg BACKEND=cpu \ - --target webui \ - -t fish-speech-webui:cpu . -``` - -### Build API Server Image - -```bash -# Build with CUDA support -docker build \ - --platform linux/amd64 \ - -f docker/Dockerfile \ - --build-arg BACKEND=cuda \ - --build-arg CUDA_VER=12.6.0 \ - --build-arg UV_EXTRA=cu126 \ - --target server \ - -t fish-speech-server:cuda . -``` - -### Build Development Image - -```bash -# Build development image with all tools -docker build \ - --platform linux/amd64 \ - -f docker/Dockerfile \ - --build-arg BACKEND=cuda \ - --target dev \ - -t fish-speech-dev:cuda . -``` - -### Build Arguments - -| Argument | Options | Default | Description | -|----------|---------|---------|-------------| -| `BACKEND` | `cuda`, `cpu` | `cuda` | Compute backend | -| `CUDA_VER` | `12.6.0`, etc. | `12.6.0` | CUDA version | -| `UV_EXTRA` | `cu126`, `cu128`, `cu129` | `cu126` | UV extra for CUDA | -| `UBUNTU_VER` | `24.04`, etc. | `24.04` | Ubuntu base version | -| `PY_VER` | `3.12`, etc. | `3.12` | Python version | - -## Volume Mounts - -Both Docker run and Compose methods require these volume mounts: - -| Host Path | Container Path | Purpose | -|-----------|----------------|---------| -| `./checkpoints` | `/app/checkpoints` | Model weights directory | -| `./references` | `/app/references` | Reference audio files for voice cloning | - - -Ensure model weights are downloaded and placed in the `./checkpoints` directory before starting containers. See [Running Inference](/developer-guide/self-hosting/running-inference#download-weights) for download instructions. - - -## Environment Variables Reference - -### WebUI Configuration - -| Variable | Default | Description | -|----------|---------|-------------| -| `GRADIO_SERVER_NAME` | `0.0.0.0` | WebUI server host | -| `GRADIO_SERVER_PORT` | `7860` | WebUI server port | -| `GRADIO_SHARE` | `false` | Enable Gradio public sharing | - -### API Server Configuration - -| Variable | Default | Description | -|----------|---------|-------------| -| `API_SERVER_NAME` | `0.0.0.0` | API server host | -| `API_SERVER_PORT` | `8080` | API server port | - -### Model Configuration - -| Variable | Default | Description | -|----------|---------|-------------| -| `LLAMA_CHECKPOINT_PATH` | `checkpoints/openaudio-s1-mini` | Path to model weights | -| `DECODER_CHECKPOINT_PATH` | `checkpoints/openaudio-s1-mini/codec.pth` | Path to decoder weights | -| `DECODER_CONFIG_NAME` | `modded_dac_vq` | Decoder configuration name | - -### Performance Optimization - -| Variable | Default | Description | -|----------|---------|-------------| -| `COMPILE` | `0` | Enable torch.compile for ~10x speedup (CUDA only) | - -## Container Management - -### View Logs - -```bash -# Docker run -docker logs fish-speech-webui - -# Docker Compose -docker compose logs webui -``` - -### Stop Containers - -```bash -# Docker run -docker stop fish-speech-webui - -# Docker Compose -docker compose down -``` - -### Update Images - -```bash -# Pull latest images -docker pull fishaudio/fish-speech:latest-webui-cuda - -# Restart containers with new image -docker compose --profile webui up -d -``` - -## GPU Support - -### Prerequisites - -Install NVIDIA Container Toolkit: - -```bash -# Ubuntu/Debian -distribution=$(. /etc/os-release;echo $ID$VERSION_ID) -curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | \ - sudo tee /etc/apt/sources.list.d/nvidia-docker.list - -sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit -sudo systemctl restart docker -``` - -### Verify GPU Access - -```bash -docker run --rm --gpus all nvidia/cuda:12.6.0-base-ubuntu24.04 nvidia-smi -``` - - -GPU support requires NVIDIA Docker runtime. For CPU-only deployment, remove the `--gpus all` flag and use CPU images. - - -## Troubleshooting - -### Container Won't Start - -Check logs for errors: -```bash -docker logs fish-speech-webui -``` - -Common issues: -- Missing model weights in `./checkpoints` -- Port already in use (change port mapping) -- Insufficient GPU memory - -### GPU Not Detected - -Verify NVIDIA Docker runtime is installed: -```bash -docker run --rm --gpus all nvidia/cuda:12.6.0-base-ubuntu24.04 nvidia-smi -``` - -### Performance Issues - -1. Enable compile optimization: `COMPILE=1` -2. Ensure GPU is being used (check with `nvidia-smi`) -3. Verify sufficient GPU memory is available - -## Next Steps - -- **[Run inference](/developer-guide/self-hosting/running-inference)** - Learn how to generate speech -- **[Download models](https://huggingface.co/fishaudio)** - Get pre-trained weights -- **[API documentation](/api-reference/introduction)** - Integrate with your applications diff --git a/developer-guide/self-hosting/introduction.mdx b/developer-guide/self-hosting/introduction.mdx new file mode 100644 index 0000000..d03d4c3 --- /dev/null +++ b/developer-guide/self-hosting/introduction.mdx @@ -0,0 +1,163 @@ +--- +title: "Introduction" +description: "Run the Fish Audio Enterprise speech stack inside your own infrastructure" +icon: "server" +--- + +Fish Audio Enterprise can be deployed into infrastructure you control: your own +cloud account, an on-premise data center, or a network with no internet access at +all. Fish Audio delivers container images and a Helm chart through a private +registry; you own the cluster, the network boundary, and the data. + +The self-hosted stack runs the same speech engine as the hosted API, so requests, +audio formats, and voice behavior match what you already build against. + +## Why self-host + +| Reason | What it gives you | +| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Colocation | Run inference in the same region, VPC, or rack as your application and remove public-internet round trips from time-to-first-audio. | +| Single-tenant isolation | Dedicated GPUs and queues. Capacity is not shared with other tenants, and you decide when the deployment is upgraded. | +| Security posture | Voice traffic never leaves your network. The offline delivery forms make no outbound calls at runtime and run on disconnected networks. | +| Data sovereignty | Input text, generated audio, and reference voices stay inside your boundary and under your own retention policy. | + +## Delivery forms + +The same engine ships in three forms. Your enterprise agreement determines which +ones your team is granted. + +| | Online Helm | Offline Helm | Offline All-in-One | +| ---------------- | --------------------------------------------------- | ------------------------------------------------ | --------------------------------------------------- | +| Platform | Kubernetes | Kubernetes | A single container, `docker run` | +| Model assets | Pulled from an S3-compatible endpoint at startup | Served in-cluster from a bundled model warehouse | Baked into the image | +| Usage accounting | Validated and billed against the Fish Audio service | Local signed usage ledger | Local signed usage ledger | +| Runtime egress | Model endpoint and billing endpoint | None | None | +| Air-gap capable | No | Yes, after mirroring images and charts | Yes | +| Scaling | Scales replicas across all GPUs and nodes | Scales replicas across all GPUs and nodes | Single node, single inference worker | +| Best for | Managed clusters with outbound access | Isolated or regulated production clusters | Evaluation, single-node appliances, strict air gaps | + +Both Helm forms share one deployment procedure and differ only in a few values. +See [Kubernetes deployment](/developer-guide/self-hosting/kubernetes) for the +Helm forms and [All-in-One container](/developer-guide/self-hosting/all-in-one) +for the single-container form. + +## What is included + +| Capability | Online Helm | Offline Helm | All-in-One | +| ----------------------------------------- | ---------------------------- | ------------------------ | ------------------------ | +| Text to speech over `POST /v1/tts` | Included | Included | Included | +| WebSocket streaming | Included | Included | Included | +| Reference-voice requests (`reference_id`) | From pre-staged archives | From pre-staged archives | From pre-staged archives | +| Word and segment timestamps | Optional, needs an extra GPU | Not included | Not included | +| Horizontal scaling and autoscaling | Included | Included | Not included | +| Prometheus metrics | Included | Included | Container logs only | + + + Timestamp alignment is served by a separate forced-aligner component. It is + disabled by default, is not part of the offline model bundle, and is not built + into the All-in-One image, so `/v1/tts/stream/with-timestamp` returns audio + without alignment data on those forms. Contact Fish Audio if your deployment + needs timestamps. + + +Reference voices are resolved from archives you stage yourself. A self-hosted +deployment does not read from the hosted voice library, and voice models created +on fish.audio are not automatically available to it. Products other than text to +speech are hosted-only unless your agreement says otherwise. + +## Architecture at a glance + +| Component | Role | +| ---------------------------- | ---------------------------------------------------------------------------------------------- | +| `edge-api-production` | Entry point. Validates requests, applies product logic, and routes work to the model services. | +| `oxidengine` | Model API layer that coordinates normalization, inference, and audio decoding. | +| `sglang-router` and `sglang` | Routing and GPU-backed inference workers. | +| `vocoder` | GPU-backed audio decoding. | +| `text-normalizer` | Text normalization ahead of inference. | +| Redis | Runtime state and cache. Installed by the chart into the release namespace. | +| Shared storage | Model cache, reference archives, and the usage ledger. Mounted on every node. | + +```text +client -> your ingress or load balancer -> edge-api -> oxidengine -> sglang-router -> sglang (GPU) + -> vocoder (GPU) + -> text-normalizer +``` + +Fish Audio delivers the deployment at the Kubernetes service level. Ingress, DNS, +TLS, external load balancing, and network allowlists are yours to choose and +configure. + +## Getting access + + + + Self-hosting is enabled per team under an enterprise agreement. Reach out + through [fish.audio/enterprise](https://fish.audio/enterprise) with your GPU + target, expected concurrency, and whether you need air-gapped operation. + + + Once your agreement is in place, Fish Audio grants your team the artifacts + it is entitled to: the Helm chart, the component images, and the All-in-One + image where applicable. + + + Sign in to fish.audio and open **Developer → Self Host**. That page shows + the registry host, your login username, the Helm chart base, your team ID, + and the exact artifacts your team can pull. Create a deploy token there to + authenticate. See [Registry + access](/developer-guide/self-hosting/registry-access). + + + + + Connection values are per-team and are only shown in the dashboard. This + documentation uses placeholders such as `` and `` + wherever a value from that page belongs. + + +## Next steps + + + + GPU, CPU, memory, storage, and platform baselines. + + + Create a deploy token and authenticate Docker and Helm. + + + Install, upgrade, roll back, and validate the Helm release. + + + Run the whole stack from a single `docker run`. + + + Mirror artifacts, account for usage offline, and prove zero egress. + + + Monitoring signals, scaling, backups, and troubleshooting. + + diff --git a/developer-guide/self-hosting/kubernetes.mdx b/developer-guide/self-hosting/kubernetes.mdx new file mode 100644 index 0000000..668c505 --- /dev/null +++ b/developer-guide/self-hosting/kubernetes.mdx @@ -0,0 +1,321 @@ +--- +title: "Kubernetes deployment" +description: "Install, validate, upgrade, and roll back the Fish Audio Enterprise Helm release" +icon: "dharmachakra" +--- + +Both Helm delivery forms use the same chart and the same procedure. They differ +only in where model assets come from and how usage is accounted, which is +expressed in your values file. + +## Prerequisites + +- A cluster that meets the [requirements](/developer-guide/self-hosting/requirements), preferably a clean cluster dedicated to this deployment. +- Shared storage mounted at the same path on every node. +- [Registry access](/developer-guide/self-hosting/registry-access) configured, with the chart reference and version from **Granted Artifacts**. +- Prometheus installed, or approved for installation. +- Reference voice archives prepared, if your traffic uses `reference_id`. + +Set the values you will reuse: + +```bash +CHART_REF='' +CHART_VERSION='' +NAMESPACE=fish-audio +RELEASE=fish-audio +``` + + + `fish-audio` is the supported default namespace. If you must use a different + one, change **both** the Helm `--namespace` argument **and** + `MODEL_DISCOVERY_NAMESPACE` in your values file. Otherwise the edge API looks + for the model services in the wrong namespace. + + +## Choose a delivery form + +| | Offline | Online | +| -------------------------- | ------------------------------------------------ | ----------------------------------------------------------------- | +| `global.offlineMode` | `true` | `false` | +| Model assets | Served in-cluster by the bundled model warehouse | Downloaded from your configured S3-compatible endpoint at startup | +| Usage accounting | Local signed ledger on shared storage | Validated and billed against the Fish Audio service | +| Object storage credentials | Not validated, placeholder values are tolerated | Must hold real credentials or model download fails | +| Runtime egress | None | Model endpoint and billing endpoint on 443 | + +"Offline" here means runtime-offline. Installation still pulls images and the +chart from a registry. For an install with no network at all, see +[Air-gapped deployments](/developer-guide/self-hosting/air-gapped). + +## Preflight + +```bash +kubectl version +helm version +kubectl get nodes -o wide +kubectl get storageclass +kubectl describe nodes | grep -E "nvidia.com/gpu|Taints|Labels" || true +``` + +Verify on **every** node that the shared storage path is a real network mount and +not a local directory. With node access, `findmnt -T /mnt/share` should report an +`nfs` or `nfs4` filesystem. Without node access: + +```bash +for node in $(kubectl get nodes -o name); do + echo "== ${node}" + kubectl debug "${node}" -it --image=busybox:1.36 -- \ + sh -c 'grep " /mnt/share " /proc/1/mounts || echo "MISSING: /mnt/share is not a mount"' +done +``` + +## Create the namespace and secrets + +```bash +kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - +``` + +The chart expects three existing secrets. The registry pull secret is created in +[Registry access](/developer-guide/self-hosting/registry-access#create-the-kubernetes-pull-secret). +Create the other two: + +```bash +# Runtime secret for the edge API. +kubectl -n "$NAMESPACE" create secret generic fish-audio-edge-api \ + --from-literal=jwt-secret="$(openssl rand -hex 32)" \ + --dry-run=client -o yaml | kubectl apply -f - + +# Object storage credentials for model assets. +kubectl -n "$NAMESPACE" create secret generic fish-audio-model-assets \ + --from-literal=aws-access-key-id='' \ + --from-literal=aws-secret-access-key='' \ + --dry-run=client -o yaml | kubectl apply -f - +``` + +| Secret | Keys | Used by | +| ------------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | +| `fish-audio-registry` | Docker registry credentials | Every workload that pulls a private image. | +| `fish-audio-edge-api` | `jwt-secret` | The edge API. Generate once and keep it stable; changing it invalidates issued tokens. | +| `fish-audio-model-assets` | `aws-access-key-id`, `aws-secret-access-key` | Inference worker and vocoder model asset loading. Required in both forms, validated only in the online form. | + +For production, prefer External Secrets, Sealed Secrets, or your cloud secret +manager over plain Kubernetes Secrets. + +## Prepare a values file + +Start from the profile Fish Audio delivered with your chart and change only what +your cluster needs. The chart ships pinned image references, so do not override +tags unless Fish Audio asks you to. + +```yaml values.yaml +global: + imagePullSecrets: + - name: fish-audio-registry + # Host directory where the shared filesystem is mounted on every node. + sharedStorageHostPath: /mnt/share + # true = model assets served in-cluster, no runtime egress + # false = model assets downloaded from your S3-compatible endpoint + offlineMode: true + +edge-api-production: + replicaCount: 2 + env: + - name: MODEL_DISCOVERY_NAMESPACE + value: fish-audio + ingress: + enabled: false + +sglang: + worker: + replicas: 1 + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +vocoder: + replicaCount: 1 + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule +``` + +Ingress and TLS are disabled by default. Enable them only if you want Kubernetes +to manage your access layer; otherwise put your own ingress controller or load +balancer in front of the `edge-api-production` service. + +## Install Prometheus + +The chart exposes metrics through `prometheus.io/*` pod annotations and creates +no ServiceMonitors. An annotation-scraping Prometheus works out of the box: + +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm upgrade --install prometheus prometheus-community/prometheus \ + --namespace monitoring --create-namespace \ + --set alertmanager.enabled=false +``` + +If you already run kube-prometheus-stack, add an annotation-based scrape config +instead. + +## Stage reference voices + +A self-hosted deployment never downloads reference audio from the network. +Requests that pass a `reference_id` resolve from Redis, then from a local archive +on shared storage. Stage one zip per voice before enabling that traffic: + +```text +///reference-archives/.zip +``` + +Each archive holds one or more audio files at the zip root, each paired with a +UTF-8 `.txt` transcript of the same basename: + +```text +voice-alice-001.zip +|-- sample-01.wav +|-- sample-01.txt +|-- sample-02.mp3 +`-- sample-02.txt +``` + +Supported audio extensions are `.wav`, `.mp3`, and `.opus`. Do not nest +directories inside the archive, and do not add prefixes to the filename — the +file name must match the reference id exactly. + +If you have no node shell, stage archives through a helper pod that mounts the +share, then confirm the edge API can see the file: + +```bash +kubectl -n "$NAMESPACE" exec deploy/edge-api-production -- \ + ls -lh /mnt/share/reference-archives/voice-alice-001.zip +``` + +These archives are durable data, not cache. Back them up and preserve the path +across upgrades. + +## Validate the release + +Render the manifests before touching the cluster: + +```bash +helm show values "$CHART_REF" --version "$CHART_VERSION" > defaults.yaml + +helm template "$RELEASE" "$CHART_REF" \ + --version "$CHART_VERSION" \ + --namespace "$NAMESPACE" \ + --values values.yaml +``` + +## Install + +```bash +helm upgrade --install "$RELEASE" "$CHART_REF" \ + --version "$CHART_VERSION" \ + --namespace "$NAMESPACE" \ + --create-namespace \ + --values values.yaml \ + --timeout 30m +``` + +## Wait for readiness + +On the first install the inference worker compiles its GPU graphs and the vocoder +builds its inference engine, which takes far longer than later rollouts. Wait on +every workload in the release, Deployments and StatefulSets alike: + +```bash +( + set -e + resources="$(kubectl -n "$NAMESPACE" get deployment,statefulset \ + -l app.kubernetes.io/instance="$RELEASE" -o name)" + test -n "$resources" || { echo "No Fish Audio workloads found" >&2; exit 1; } + for r in $resources; do + kubectl -n "$NAMESPACE" rollout status "$r" --timeout=90m + done +) +``` + +The timeout applies per workload, not to the loop. Watch progress with: + +```bash +kubectl -n "$NAMESPACE" get pods,svc +kubectl -n "$NAMESPACE" get events --sort-by=.lastTimestamp +``` + +## Smoke test + +The acceptance signal is a successful generation, not a health probe. With no +ingress configured, port-forward the edge API: + +```bash +kubectl -n "$NAMESPACE" port-forward svc/edge-api-production 8080:80 +``` + +```bash +curl -fsS --noproxy '*' -X POST http://127.0.0.1:8080/v1/tts \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -H 'model: ' \ + -d '{"text": "Hello from Fish Audio Enterprise.", "format": "mp3"}' \ + -o smoke-test.mp3 + +file smoke-test.mp3 +``` + +The test passes when the request returns HTTP 200 and the file contains playable +audio. + +| Detail | Notes | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model` header | Must match the model name registered by your values file. | +| `Authorization`, online form | Your Fish Audio API key. Every request is authorized and billed against the Fish Audio service. | +| `Authorization`, offline form | Any non-empty bearer token is accepted and recorded verbatim as the billing identity. An empty or missing token still returns 401. Use a stable, distinct token per tenant. | +| Proxies | A local HTTP or SOCKS proxy can break a port-forwarded request with an empty reply. `--noproxy '*'` and `127.0.0.1` avoid it. | +| Latency | Latency measured through `kubectl port-forward` includes the round trip to the API server and your own network. Measure from inside the cluster for representative numbers. | + +Also confirm that the GPU workloads see the shared storage mount, that Redis is +ready, that reference archives are visible if you use them, and that no pod is in +a crash or image-pull loop. + +## Upgrade + +```bash +helm diff upgrade "$RELEASE" "$CHART_REF" \ + --version "$CHART_VERSION" \ + --namespace "$NAMESPACE" \ + --values values.yaml + +helm upgrade "$RELEASE" "$CHART_REF" \ + --version "$CHART_VERSION" \ + --namespace "$NAMESPACE" \ + --values values.yaml \ + --timeout 30m +``` + +If the `helm-diff` plugin is not installed, review `helm template` output before +applying. Re-run the smoke test after every upgrade and keep the previous +revision available until the deployment is accepted. + +## Roll back + +```bash +helm -n "$NAMESPACE" history "$RELEASE" +helm -n "$NAMESPACE" rollback "$RELEASE" '' --timeout 30m +kubectl -n "$NAMESPACE" get pods +``` + +## Uninstall + +```bash +helm -n "$NAMESPACE" uninstall "$RELEASE" +``` + +Uninstall does not remove data on shared storage. Reference archives, the usage +ledger, and compile caches survive and should be deleted deliberately. + +## Next steps + +- [Air-gapped deployments](/developer-guide/self-hosting/air-gapped) to install with no network access. +- [Operations](/developer-guide/self-hosting/operations) for monitoring signals, scaling, and troubleshooting. diff --git a/developer-guide/self-hosting/local-setup.mdx b/developer-guide/self-hosting/local-setup.mdx deleted file mode 100644 index c9ffa44..0000000 --- a/developer-guide/self-hosting/local-setup.mdx +++ /dev/null @@ -1,155 +0,0 @@ ---- -title: "Local Model Setup" -description: "Install and configure Fish Audio models for local inference" -icon: "server" ---- -import { AudioTranscript } from '/snippets/audio-transcript.jsx'; - -{/* speak-mintlify-hash: f1038df54a82e77cfe449b4182e5a43b9739fce33ecc1347496c5db4efd98e09 */} - - - - - - -This guide is for advanced users who want to self-host Fish Audio models. For most users, we recommend using the [Fish Audio API](https://fish.audio) for easier integration and automatic updates. - - -## Prerequisites - -Before you begin, ensure you have: - -- **GPU**: 12GB VRAM minimum (for inference) -- **OS**: Linux or WSL (Windows Subsystem for Linux) -- **System dependencies**: Audio processing libraries - -Install required system packages: - -```bash -apt install portaudio19-dev libsox-dev ffmpeg -``` - -## Installation Methods - -Fish Audio supports multiple installation methods. Choose the one that best fits your development environment. - -### Conda Installation - -Conda provides a stable, isolated Python environment: - -```bash -# Create a new environment with Python 3.12 -conda create -n fish-speech python=3.12 -conda activate fish-speech - -# GPU installation (choose your CUDA version: cu126, cu128, cu129) -pip install -e .[cu129] - -# CPU-only installation (slower, not recommended for production) -pip install -e .[cpu] - -# Default installation (uses PyTorch default index) -pip install -e . -``` - - -For best performance, match your CUDA version with your GPU driver. Use `nvidia-smi` to check your CUDA version. - - -### UV Installation - -[UV](https://github.com/astral-sh/uv) provides faster dependency resolution and installation: - -```bash -# GPU installation (choose your CUDA version: cu126, cu128, cu129) -uv sync --python 3.12 --extra cu129 - -# CPU-only installation -uv sync --python 3.12 --extra cpu -``` - - -UV is recommended for faster setup times, especially when working with large dependency trees. - - -### Intel Arc XPU Support - -For Intel Arc GPU users, install with XPU support: - -```bash -# Create environment -conda create -n fish-speech python=3.12 -conda activate fish-speech - -# Install required C++ standard library -conda install libstdcxx -c conda-forge - -# Install PyTorch with Intel XPU support -pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/xpu - -# Install Fish Speech -pip install -e . -``` - - -The `--compile` optimization flag is not supported on Windows and macOS. To use compile acceleration, you need to install Triton manually. - - -## Repository Setup - -Clone the Fish Speech repository to get started: - -```bash -git clone https://github.com/fishaudio/fish-speech.git -cd fish-speech -``` - -Then follow one of the installation methods above. - -## Next Steps - -Once installation is complete, you can: - -- **[Set up Docker deployment](/developer-guide/self-hosting/docker-deployment)** - Use containerized deployment for easier management -- **[Run inference](/developer-guide/self-hosting/running-inference)** - Start generating speech with your local models -- **Download models** - Get pre-trained weights from [Hugging Face](https://huggingface.co/fishaudio) - -## Hardware Recommendations - -For optimal performance: - -| Use Case | Recommended GPU | VRAM | Expected Speed | -|----------|----------------|------|----------------| -| Development | RTX 3060 | 12GB | ~1:15 real-time factor | -| Production | RTX 4090 | 24GB | ~1:7 real-time factor | -| Enterprise | A100 | 40GB+ | ~1:5 real-time factor | - - -Real-time factor indicates how much faster than real-time the model can generate audio. For example, 1:7 means generating 1 minute of audio takes ~8.5 seconds. - - -## Troubleshooting - -### CUDA Out of Memory - -If you encounter CUDA out of memory errors: - -1. Reduce batch size in inference settings -2. Use `--half` flag for FP16 inference -3. Close other GPU-intensive applications - -### Package Installation Errors - -If you encounter dependency conflicts: - -1. Try using UV instead of pip for better dependency resolution -2. Create a fresh conda environment -3. Ensure you're using Python 3.12 (other versions may have compatibility issues) - -## Community Support - -Need help with local setup? - -- Join our [Discord community](https://discord.gg/dF9Db2Tt3Y) for community support -- Check [GitHub Issues](https://github.com/fishaudio/fish-speech/issues) for known problems -- Contact [enterprise support](mailto:support@fish.audio) for commercial deployments \ No newline at end of file diff --git a/developer-guide/self-hosting/operations.mdx b/developer-guide/self-hosting/operations.mdx new file mode 100644 index 0000000..e49b8c0 --- /dev/null +++ b/developer-guide/self-hosting/operations.mdx @@ -0,0 +1,120 @@ +--- +title: "Operations" +description: "Monitoring, scaling, backups, and troubleshooting for a self-hosted deployment" +icon: "chart-line" +--- + +Day-2 guidance for the Kubernetes delivery forms. For the single-container form, +see [All-in-One operations](/developer-guide/self-hosting/all-in-one#operations). + +## Ownership + +| Area | Owner | +| ---------------------------------------------------- | ------------------------------------------------------------- | +| Kubernetes cluster, node lifecycle, platform add-ons | You | +| Fish Audio chart and application configuration | Fish Audio | +| Secrets and credentials | You, with Fish Audio inputs where required | +| Monitoring and alerting | You. Fish Audio can advise on expected signals and thresholds | +| Incident response | Joint during the deployment window, yours after handoff | + +Agree an escalation path before production traffic starts. + +## Monitoring signals + +Prometheus is the metrics baseline. Watch: + +- Pod readiness and restart counts for every service in the release, including Redis. +- Request success rate, error rate, latency, and time-to-first-audio. +- Queue depth, where the application exposes it. +- GPU utilization and GPU memory. +- CPU and memory usage against the configured requests and limits. +- Shared storage usage and latency. +- Redis availability and latency. +- Image pull failures and Kubernetes events in the release namespace. + +Centralize application logs and retain Kubernetes events long enough to +investigate a bad rollout. Redact secrets before sharing any logs outside your +environment. + +## Scaling + +Scaling is controlled through Helm values: API replicas, GPU worker replicas, +CPU and memory requests, GPU resource requests, and per-worker concurrency. Add a +GPU replica only when a GPU is actually free — a pod that requests a GPU on a +full cluster stays `Pending` indefinitely. + +Validate every scale change with the smoke test and a benchmark run at your +expected concurrency, and remember that the first start of a new GPU worker pays +the compile cost before it becomes ready. + +## Backup and retention + +Back up: + +- Reference voice archives, if your traffic uses reference ids. In a self-hosted + deployment these are durable data, not cache — nothing can re-download them. +- The offline usage ledger directories, before they are pruned by your own + archival process. +- The values file used for the production release. +- Secret manager entries. +- Dashboards and alert rules. + +Compile and model caches on shared storage do not need backing up. Losing them +costs a slow first start, nothing more. + +## Upgrades + +1. Review the chart and image changes with Fish Audio. +2. Render the manifests with your production values file. +3. Apply to a staging or dry-run cluster if you have one. +4. Capture the current release history. +5. Upgrade during an approved window. +6. Watch rollout status, then run the smoke test. +7. Keep the previous revision available until the deployment is accepted. + +## Troubleshooting + +Start here: + +```bash +kubectl -n fish-audio get pods,svc +kubectl -n fish-audio get events --sort-by=.lastTimestamp +helm -n fish-audio status fish-audio +helm -n fish-audio history fish-audio +``` + +| Symptom | Likely causes | +| ------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ImagePullBackOff` or `ErrImagePull` | Missing pull secret, a deploy token that was rotated or deleted, a tag that is not in your mirror, or blocked egress to the registry. | +| Pods stay `Pending` | Not enough CPU, memory, or GPU capacity; GPU node taints without matching tolerations; a node selector that matches nothing. | +| GPU not available, or `nvidia-smi` fails in a pod | Driver missing or unhealthy, device plugin or GPU Operator not ready, container runtime not configured for NVIDIA, or an unsupported GPU for the driver stack. | +| hostPath volume errors, or missing model and cache paths | Shared storage is not mounted on that node, a newly added node never got the mount, the mount target is unreachable, or permissions prevent writes. | +| Requests fail after reaching the edge API, with Redis errors in its logs | The Redis master or replica is not ready, the service is missing, or a network policy blocks it. | +| `Reference not found` | The archive was never staged, the filename does not exactly match the reference id, the zip is malformed, or the reference root was changed without mounting the new path into every edge API pod. | +| Metrics missing from Prometheus | Scrape annotations disabled in values, Prometheus not configured for annotation-based discovery, a port mismatch, or a network policy blocking scrapes. | +| `CrashLoopBackOff` | A missing secret or environment variable, model assets unreachable, or a path the container cannot write. | +| Time-to-first-audio high, GPU utilization low | Concurrency above what the current replicas can serve, cold caches after a restart, storage latency, or CPU saturation on the API nodes. | + +Collect detail for a single failing pod: + +```bash +kubectl -n fish-audio describe pod '' +kubectl -n fish-audio logs '' --all-containers --previous --tail=300 +``` + +For GPU incidents, add: + +```bash +kubectl describe node '' +kubectl -n fish-audio exec '' -- nvidia-smi +``` + +## Escalating to Fish Audio + +Include: + +- Kubernetes version, cloud provider, region, node types, and GPU type. +- Chart version and the values file with secrets removed. +- Pod status, relevant events, and logs with secrets redacted. +- The exact command that failed, with its output. +- Timestamp and time zone. diff --git a/developer-guide/self-hosting/registry-access.mdx b/developer-guide/self-hosting/registry-access.mdx new file mode 100644 index 0000000..8182f0c --- /dev/null +++ b/developer-guide/self-hosting/registry-access.mdx @@ -0,0 +1,147 @@ +--- +title: "Registry access" +description: "Create a deploy token and authenticate Docker and Helm against the Fish Audio registry" +icon: "key" +--- + +Self-hosted images and charts are distributed from a private Fish Audio registry. +Your team authenticates to it with a **deploy token** that you create in the +fish.audio dashboard. + +## Prerequisites + +- Self-hosting enabled for your team under an enterprise agreement. +- A fish.audio account that is a member of that team. +- Docker, and Helm 3.8 or newer for the OCI chart commands. + +If **Developer → Self Host** reports that self-host deployment is not enabled for +your team, contact your account manager. + +## Collect your connection values + +Sign in to fish.audio and open **Developer → Self Host**. The **Connection +Values** card holds everything the commands below need, and every value has a +copy button. + +| Dashboard field | Placeholder used in this documentation | +| ---------------- | -------------------------------------- | +| Registry host | `` | +| Login username | `` | +| Helm charts base | `` | +| Team ID | `` | + +The **Granted Artifacts** card lists exactly what your team may pull — the chart, +the component images, and the All-in-One image where your agreement includes it — +with the full reference for each one. Copy references and tags from that card +rather than typing them by hand; it is the authoritative list for your team and it +changes as your entitlement changes. If the card is empty, no artifacts have been +granted yet. + + + Connection values are specific to your team and are only published in the + dashboard, never in this documentation. Substitute the values you copied + wherever a placeholder appears. + + +## Create a deploy token + + + + On **Developer → Self Host**, select **Create Deploy Token**. + + + Use a name that identifies the consumer, such as `prod-cluster` or + `ci-mirror`. The name appears in the token list alongside the creation date + and last-used time. + + + The token value is shown once, at creation. Store it in your secret manager + before closing the dialog. If you lose it, rotate the token to issue a new + one. + + + +A team can hold up to five deploy tokens at a time. Tokens carry the grants of +the team that owns them, not of the person who created them. + +## Authenticate Docker + +Set your values once, then reuse them in the commands below. + +```bash +REGISTRY_HOST='' +LOGIN_USER='' +DEPLOY_TOKEN='' +``` + +Sign in with your account email as the username and the deploy token as the +password: + +```bash +printf '%s' "$DEPLOY_TOKEN" | docker login "$REGISTRY_HOST" -u "$LOGIN_USER" --password-stdin +``` + +Verify access by pulling one of your granted images: + +```bash +docker pull '' +``` + +## Authenticate Helm + +Charts are served as OCI artifacts, so Helm authenticates against the same +registry host: + +```bash +helm registry login "$REGISTRY_HOST" -u "$LOGIN_USER" --password-stdin <<< "$DEPLOY_TOKEN" +``` + +Confirm the chart is reachable, using the chart reference and version from +**Granted Artifacts**: + +```bash +helm show chart '' --version '' +``` + +## Create the Kubernetes pull secret + +The cluster pulls images with the same credentials. Create the pull secret the +chart expects in the release namespace: + +```bash +kubectl -n fish-audio create secret docker-registry fish-audio-registry \ + --docker-server="$REGISTRY_HOST" \ + --docker-username="$LOGIN_USER" \ + --docker-password="$DEPLOY_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - +``` + +Reference it from the release so every workload uses it: + +```yaml +global: + imagePullSecrets: + - name: fish-audio-registry +``` + +## Managing tokens + +| Action | Effect | +| --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| Rotate | Issues a new token value and invalidates the old one immediately. Update every consumer before rotating, or new pods will fail to pull. | +| Delete | Revokes the token immediately. Any deployment still using it stops pulling images. Running pods keep running until they are rescheduled. | +| Last used | Shows when the token last authenticated, which identifies tokens that are safe to retire. | + +Recommended practice: + +- Issue one token per consumer — production cluster, staging cluster, CI mirror — + so a single revocation never takes down more than one of them. +- Store tokens in your secret manager, not in values files or version control. +- Rotate on your normal credential schedule and whenever someone with access to a + token leaves the team. + +## Next step + +With the registry reachable, continue to +[Kubernetes deployment](/developer-guide/self-hosting/kubernetes) or the +[All-in-One container](/developer-guide/self-hosting/all-in-one). diff --git a/developer-guide/self-hosting/requirements.mdx b/developer-guide/self-hosting/requirements.mdx new file mode 100644 index 0000000..7069a05 --- /dev/null +++ b/developer-guide/self-hosting/requirements.mdx @@ -0,0 +1,144 @@ +--- +title: "Requirements" +description: "Hardware, platform, and network baselines for a self-hosted deployment" +icon: "microchip" +--- + +These are the baselines Fish Audio deploys against. Confirm final sizing with your +account team once your traffic profile and target GPU are known: time-to-first-audio +and throughput depend on the model, GPU, text length, and concurrency, and should be +measured on your own hardware before you commit to a capacity plan. + +## Kubernetes deployments + +### Cluster topology + +| Node type | Minimum | Purpose | +| -------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU-only control-plane or system nodes | 3 nodes, roughly 4 vCPU and 16 GB RAM each | Control-plane high availability on self-managed Kubernetes, or a system node group for platform add-ons on managed Kubernetes. | +| CPU workload capacity | Roughly 28 vCPU and 125 GiB of memory requests | Non-GPU workloads at the default chart values: Redis, the edge API, the text normalizer, the model API layer, and the inference router. Provide a CPU node group, or leave GPU nodes schedulable so these can land there. | +| GPU worker nodes | 2 nodes | At least 2 nodes so workloads survive a node drain or failure. | +| Shared storage | 1 TB usable, mounted on every node | Model cache, reference archives, compile caches, and the offline usage ledger. | +| Network | Low-latency east-west networking | GPU workload stability, service-to-service calls, and shared storage access. | + +Verify the CPU and memory figures against your own chart version by rendering the +release and summing the requests; they move with the default values. + +### GPUs + +| Requirement | Detail | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Count | 2 GPUs minimum: one inference worker, one vocoder. Enabling timestamp alignment adds a third. | +| Type | NVIDIA H100 or H200. H200 is the preferred target for its larger memory and bandwidth. | +| Memory | The inference worker uses roughly 26 to 27 GB of VRAM at shipped settings, so 32 GB cards are workable. Larger cards leave headroom for a higher concurrency ceiling. | +| Newer architectures | Contact Fish Audio before standardizing on a GPU generation that is not H100 or H200, so the image, CUDA stack, and driver combination can be confirmed. | +| MIG | Disable MIG unless the configuration has been validated with Fish Audio. | + +On AWS, `p5.48xlarge` (H100) is an acceptable baseline and `p5en.48xlarge` (H200) +is the preferred target. Validate GPU instance quota in the target region before +scheduling a deployment window. + +### Shared storage + +Provide at least 1 TB of usable shared storage backed by EFS, NFS, or an +equivalent service, mounted at the same path on every node that runs Fish Audio +workloads (`/mnt/share` by default, configurable through +`global.sharedStorageHostPath`). The chart mounts it into containers with +hostPath volumes and creates no PersistentVolumeClaims. + + + The hostPath mount is created with `DirectoryOrCreate`. A node that is missing + the shared mount silently gets a local directory instead, and the deployment + looks healthy while data splits across nodes. Verify the mount on **every** + node before installing — see the preflight checks in [Kubernetes + deployment](/developer-guide/self-hosting/kubernetes#preflight). + + +Confirm the following with your storage team: how the mount is applied to newly +added nodes, the throughput mode, the backup policy, mount target reachability, +and expected growth of the model and cache data. + +Redis is installed by the chart and does not use shared storage. Its append-only +file is backed by node-local ephemeral storage, so provision ephemeral capacity +on the nodes that host it, plus headroom for rewrites. + +### Platform baseline + +| Layer | Recommendation | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Kubernetes | A currently supported minor version. Stay one minor behind the newest release if your GPU add-ons have not been validated on it yet. | +| Helm | A current maintained release supported by your platform. | +| Container runtime | The provider-managed containerd runtime where available. | +| GPU runtime | NVIDIA GPU Operator, or the provider-managed driver and device plugin stack. GPU nodes must expose `nvidia.com/gpu`. | +| GPU telemetry | DCGM exporter or the provider equivalent. | +| Storage | EFS, NFS, or an equivalent shared filesystem mounted on every node. | +| Object storage | S3 or an S3-compatible endpoint. Online delivery form only. | +| Redis | Installed by the chart into the release namespace. Do not point the release at a shared Redis without discussing it first. | +| Metrics | Prometheus, scraping `prometheus.io/*` pod annotations. The chart creates no ServiceMonitors, so kube-prometheus-stack users must add an annotation-based scrape config. | +| Access layer | Yours to choose. Ingress controller, DNS, TLS, and load balancing are not part of the delivery. | + +The cluster also needs working in-cluster DNS, a dedicated namespace, node labels +and taints for GPU scheduling, and a Pod Security level in that namespace that +permits `hostPID` and `hostIPC`, which the inference workers require. + +### Network + +| Direction | Requirement | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Install-time egress | Access to the Fish Audio registry to pull images and the chart. Mirror both into your own registry for air-gapped installs. | +| Runtime egress, offline form | None. No model download and no billing call. | +| Runtime egress, online form | HTTPS to the configured model asset endpoint and to the Fish Audio authorization and billing endpoint. Both hostnames are shown with your deployment credentials — allowlist them. | +| Ingress | Customer-approved ingress or a private endpoint, with DNS and TLS in place before production traffic. | + +## All-in-One container host + +| Requirement | Detail | +| ----------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| OS | Linux x86-64. | +| GPUs | 2 GPUs. The first runs the inference worker, the second runs the vocoder. No NVLink required. | +| NVIDIA driver | Must support the CUDA version shipped in the image and your card's compute capability. Confirm with `nvidia-smi` before pulling. | +| Docker | Docker Engine 24 or newer. | +| Container toolkit | NVIDIA Container Toolkit installed and the `nvidia` runtime registered, so `--gpus all` exposes GPUs. | +| RAM | 128 GiB minimum, 192 GiB recommended. The container runs the whole stack in one process tree. | +| CPU | 32 vCPU minimum, 48 to 64 recommended. | +| Disk | Around 60 GB for the image, plus room for the compile and engine caches. | + +## Preflight commands + +Run these before you install anything. + +```bash +kubectl version +helm version +kubectl get nodes -o wide +kubectl get storageclass +kubectl describe nodes | grep -E "nvidia.com/gpu|Taints|Labels" || true +``` + +Confirm that GPU nodes advertise allocatable GPUs: + +```bash +kubectl get nodes -o custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu' +``` + +If your policy allows a validation pod, confirm the runtime end to end: + +```bash +kubectl run nvidia-smi \ + --rm -it \ + --restart=Never \ + --image=nvidia/cuda:12.4.1-base-ubuntu22.04 \ + --limits=nvidia.com/gpu=1 \ + -- nvidia-smi +``` + +On the All-in-One host, the equivalent check is: + +```bash +docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi +``` + +## Next step + +Once the platform checks pass, authenticate to the Fish Audio registry in +[Registry access](/developer-guide/self-hosting/registry-access). diff --git a/developer-guide/self-hosting/running-inference.mdx b/developer-guide/self-hosting/running-inference.mdx deleted file mode 100644 index df0a412..0000000 --- a/developer-guide/self-hosting/running-inference.mdx +++ /dev/null @@ -1,419 +0,0 @@ ---- -title: "Running Inference" -description: "Generate speech using self-hosted Fish Audio models" -icon: "play" ---- -import { AudioTranscript } from '/snippets/audio-transcript.jsx'; - -{/* speak-mintlify-hash: 7698d1a07e9798942356f5040025620002f27a00c185f904d68ca5902892dca4 */} - - - - - -Fish Audio supports multiple inference methods: command line, HTTP API, WebUI, and GUI. Choose the method that best fits your workflow. - - -This guide assumes you have already [installed Fish Audio locally](/developer-guide/self-hosting/local-setup) or [set up Docker deployment](/developer-guide/self-hosting/docker-deployment). - - -## Download Weights - -Before running inference, download the required model weights from Hugging Face: - -```bash -# Install Hugging Face CLI (if not already installed) -pip install huggingface_hub[cli] -# or -uv tool install huggingface_hub[cli] - -# Download Fish Audio S1-mini weights -hf download fishaudio/openaudio-s1-mini --local-dir checkpoints/openaudio-s1-mini -``` - - -**Fish Audio S1-mini** is the open-source distilled version (0.5B parameters) optimized for local deployment. The full **S1** model (4B parameters) is available exclusively on [Fish Audio cloud](https://fish.audio). - - -## Command Line Inference - -Command line inference provides maximum control and is ideal for scripting and batch processing. - -### Step 1: Extract VQ Tokens from Reference Audio - -First, encode your reference audio to get voice characteristics: - -```bash -python fish_speech/models/dac/inference.py \ - -i "reference_audio.wav" \ - --checkpoint-path "checkpoints/openaudio-s1-mini/codec.pth" -``` - -This generates two files: -- `fake.npy` - VQ tokens representing voice characteristics -- `fake.wav` - Reconstructed audio for verification - - -**Skip this step if you want random voice generation** - the model can generate speech without reference audio. - - -### Step 2: Generate Semantic Tokens from Text - -Convert your text to semantic tokens using the language model: - -```bash -python fish_speech/models/text2semantic/inference.py \ - --text "The text you want to convert to speech" \ - --prompt-text "Transcription of your reference audio" \ - --prompt-tokens "fake.npy" \ - --compile -``` - -**Parameters:** -- `--text`: The text to synthesize -- `--prompt-text`: Transcription of the reference audio (for voice cloning) -- `--prompt-tokens`: Path to VQ tokens from Step 1 (for voice cloning) -- `--compile`: Enable kernel fusion for faster inference (~10x speedup on RTX 4090) - - -For random voice generation, omit `--prompt-text` and `--prompt-tokens` parameters. - - -This creates a file named `codes_N.npy` (where N starts from 0) containing semantic tokens. - - -For GPUs that don't support bf16 (bfloat16), add the `--half` flag to use fp16 instead. - - -### Step 3: Generate Audio from Semantic Tokens - -Finally, convert semantic tokens to audio: - -```bash -python fish_speech/models/dac/inference.py \ - -i "codes_0.npy" -``` - -This generates the final audio file. - -### Full Example - -Here's a complete workflow for voice cloning: - -```bash -# 1. Encode reference audio -python fish_speech/models/dac/inference.py \ - -i "my_voice.wav" \ - --checkpoint-path "checkpoints/openaudio-s1-mini/codec.pth" - -# 2. Generate semantic tokens -python fish_speech/models/text2semantic/inference.py \ - --text "Hello, this is a test of voice cloning." \ - --prompt-text "This is my reference voice recording." \ - --prompt-tokens "fake.npy" \ - --compile - -# 3. Generate final audio -python fish_speech/models/dac/inference.py \ - -i "codes_0.npy" -``` - -## HTTP API Inference - -The HTTP API provides a programmatic interface for integrations and production deployments. - -### Start API Server - -```bash -# With local installation -python -m tools.api_server \ - --listen 0.0.0.0:8080 \ - --llama-checkpoint-path "checkpoints/openaudio-s1-mini" \ - --decoder-checkpoint-path "checkpoints/openaudio-s1-mini/codec.pth" \ - --decoder-config-name modded_dac_vq - -# With UV -uv run tools/api_server.py \ - --listen 0.0.0.0:8080 \ - --llama-checkpoint-path "checkpoints/openaudio-s1-mini" \ - --decoder-checkpoint-path "checkpoints/openaudio-s1-mini/codec.pth" \ - --decoder-config-name modded_dac_vq -``` - - -Add the `--compile` flag to enable torch.compile optimization for faster inference. - - -### Access API Documentation - -Once the server is running, access the interactive API documentation at: - -``` -http://localhost:8080/docs -``` - -The API provides endpoints for: -- Text-to-speech synthesis -- Voice cloning with reference audio -- Batch processing -- Model information - -### Example API Request - -```bash -curl -X POST "http://localhost:8080/v1/tts" \ - -H "Content-Type: application/json" \ - -d '{ - "text": "Hello, this is a test", - "reference_audio": "base64_encoded_audio", - "reference_text": "Reference transcription" - }' -``` - -## WebUI Inference - -The WebUI provides an intuitive interface for interactive testing and development. - -### Start WebUI - -```bash -# With all parameters -python -m tools.run_webui \ - --llama-checkpoint-path "checkpoints/openaudio-s1-mini" \ - --decoder-checkpoint-path "checkpoints/openaudio-s1-mini/codec.pth" \ - --decoder-config-name modded_dac_vq - -# Or use defaults (auto-detects models in checkpoints/) -python -m tools.run_webui -``` - - -Add the `--compile` flag for faster inference during interactive sessions. - - -### Access WebUI - -The WebUI starts on port 7860 by default. Access it at: - -``` -http://localhost:7860 -``` - -### Configure with Environment Variables - -Customize the WebUI using Gradio environment variables: - -```bash -# Enable public sharing -GRADIO_SHARE=1 python -m tools.run_webui - -# Change server port -GRADIO_SERVER_PORT=8080 python -m tools.run_webui - -# Change server name -GRADIO_SERVER_NAME=0.0.0.0 python -m tools.run_webui -``` - -### Using Reference Audio Library - -For faster workflow, pre-save reference audio: - -1. Create a `references/` directory in the project root -2. Create subdirectories named by voice ID: `references//` -3. Place files in each subdirectory: - - `sample.wav` - Reference audio file - - `sample.lab` - Text transcription of the audio - -Example structure: -``` -references/ -├── alice/ -│ ├── sample.wav -│ └── sample.lab -└── bob/ - ├── sample.wav - └── sample.lab -``` - -These references will appear as selectable options in the WebUI. - -## GUI Inference - -For users who prefer a native desktop application, a PyQt6-based GUI is available. - -### Download GUI Client - -Download the latest release from the [Fish Speech GUI repository](https://github.com/AnyaCoder/fish-speech-gui/releases). - -**Supported platforms:** -- Linux -- Windows -- macOS - -### Connect to API Server - -The GUI client connects to a running API server (see [HTTP API Inference](#http-api-inference) above). - -1. Start the API server -2. Launch the GUI client -3. Configure the API endpoint (default: `http://localhost:8080`) - -## Docker Inference - -If you're using Docker deployment, refer to the [Docker Deployment guide](/developer-guide/self-hosting/docker-deployment) for detailed instructions on: - -- Running pre-built WebUI containers -- Running pre-built API server containers -- Customizing container configuration -- Volume mounts for models and references - -Quick example: - -```bash -# Start WebUI with Docker -docker run -d \ - --name fish-speech-webui \ - --gpus all \ - -p 7860:7860 \ - -v ./checkpoints:/app/checkpoints \ - -v ./references:/app/references \ - -e COMPILE=1 \ - fishaudio/fish-speech:latest-webui-cuda -``` - -## Performance Optimization - -### Enable Compilation - -Torch compilation provides ~10x speedup on compatible GPUs: - -```bash -# Add --compile flag to any inference command -python -m tools.api_server --compile ... -``` - - -Compilation requires: -- CUDA-compatible GPU -- Triton library (not supported on Windows/macOS) -- First run will be slow due to compilation overhead - - -### Use Mixed Precision - -For GPUs without bf16 support, use fp16: - -```bash -python fish_speech/models/text2semantic/inference.py --half ... -``` - -### Batch Processing - -For multiple audio generations, use batch processing to amortize model loading overhead: - -```python -# Example batch processing script -import fish_speech - -model = fish_speech.load_model("checkpoints/openaudio-s1-mini") - -texts = ["First sentence", "Second sentence", "Third sentence"] -for text in texts: - audio = model.synthesize(text) - audio.save(f"output_{texts.index(text)}.wav") -``` - -## Emotion Control - -Fish Audio S1 supports emotional markers for expressive speech synthesis: - -### Basic Emotions - -``` -(angry) (sad) (excited) (surprised) (satisfied) (delighted) -(scared) (worried) (upset) (nervous) (frustrated) (depressed) -(empathetic) (embarrassed) (disgusted) (moved) (proud) (relaxed) -(grateful) (confident) (interested) (curious) (confused) (joyful) -``` - -### Advanced Emotions - -``` -(disdainful) (unhappy) (anxious) (hysterical) (indifferent) -(impatient) (guilty) (scornful) (panicked) (furious) (reluctant) -(keen) (disapproving) (negative) (denying) (astonished) (serious) -(sarcastic) (conciliative) (comforting) (sincere) (sneering) -(hesitating) (yielding) (painful) (awkward) (amused) -``` - -### Tone Markers - -``` -(in a hurry tone) (shouting) (screaming) (whispering) (soft tone) -``` - -### Special Effects - -``` -(laughing) (chuckling) (sobbing) (crying loudly) (sighing) (panting) -(groaning) (crowd laughing) (background laughter) (audience laughing) -``` - -### Example Usage - -```bash -python fish_speech/models/text2semantic/inference.py \ - --text "(excited)This is amazing! (laughing)Ha ha ha!" \ - --compile -``` - - -Emotion control is currently supported for English, Chinese, and Japanese. More languages coming soon! - - -For more details, see the [Emotion Control guide](/developer-guide/core-features/emotions). - -## Troubleshooting - -### Out of Memory Errors - -If you encounter CUDA out of memory errors: - -1. Reduce input text length -2. Use `--half` flag for fp16 inference -3. Close other GPU applications -4. Use a smaller batch size - -### Slow Inference - -To improve speed: - -1. Enable `--compile` flag -2. Verify GPU is being used (check with `nvidia-smi`) -3. Ensure CUDA version matches PyTorch installation -4. Use fp16 instead of bf16 on older GPUs - -### Poor Audio Quality - -For better quality: - -1. Use high-quality reference audio (clear, no background noise) -2. Ensure reference text accurately matches reference audio -3. Use 10-30 seconds of reference audio -4. See [Voice Cloning Best Practices](/developer-guide/best-practices/voice-cloning) - -### Model Loading Errors - -If models fail to load: - -1. Verify model weights are downloaded completely -2. Check checkpoint paths are correct -3. Ensure sufficient disk space -4. Re-download weights if corrupted - -## Next Steps - -- **[Emotion Control Best Practices](/developer-guide/best-practices/emotion-control)** - Master expressive speech -- **[Voice Cloning Best Practices](/developer-guide/best-practices/voice-cloning)** - Optimize voice cloning quality -- **[API Reference](/api-reference/introduction)** - Integrate with your applications -- **[Cloud API](https://fish.audio)** - Compare with managed service performance diff --git a/docs.json b/docs.json index 3634568..841489c 100644 --- a/docs.json +++ b/docs.json @@ -54,10 +54,7 @@ }, { "group": "Platform (Web App)", - "pages": [ - "overview/platform", - "overview/mcp" - ] + "pages": ["overview/platform", "overview/mcp"] }, { "group": "Models & Pricing", @@ -142,9 +139,13 @@ { "group": "Self-Hosting", "pages": [ - "developer-guide/self-hosting/local-setup", - "developer-guide/self-hosting/docker-deployment", - "developer-guide/self-hosting/running-inference" + "developer-guide/self-hosting/introduction", + "developer-guide/self-hosting/requirements", + "developer-guide/self-hosting/registry-access", + "developer-guide/self-hosting/kubernetes", + "developer-guide/self-hosting/all-in-one", + "developer-guide/self-hosting/air-gapped", + "developer-guide/self-hosting/operations" ] }, { @@ -203,9 +204,7 @@ { "group": "Voice Design", "icon": "wand-magic-sparkles", - "pages": [ - "api-reference/endpoint/openapi-v1/voice-design" - ] + "pages": ["api-reference/endpoint/openapi-v1/voice-design"] } ] }, @@ -235,9 +234,7 @@ { "group": "JavaScript SDK", "icon": "js", - "pages": [ - "api-reference/sdk/javascript/api-reference" - ] + "pages": ["api-reference/sdk/javascript/api-reference"] } ] } @@ -270,13 +267,7 @@ "drilldown": true }, "contextual": { - "options": [ - "copy", - "view", - "chatgpt", - "claude", - "perplexity" - ] + "options": ["copy", "view", "chatgpt", "claude", "perplexity"] }, "fonts": { "family": "Onest" @@ -493,6 +484,22 @@ { "source": "/developer-guide/getting-started/introduction", "destination": "/overview/capabilities" + }, + { + "source": "/self-host", + "destination": "/developer-guide/self-hosting/introduction" + }, + { + "source": "/developer-guide/self-hosting/local-setup", + "destination": "/developer-guide/self-hosting/introduction" + }, + { + "source": "/developer-guide/self-hosting/docker-deployment", + "destination": "/developer-guide/self-hosting/introduction" + }, + { + "source": "/developer-guide/self-hosting/running-inference", + "destination": "/developer-guide/self-hosting/introduction" } ], "footer": {