diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index f1206fe384..14d29ce6f3 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -18,6 +18,19 @@ const STUB_PORT = 8899; const STUB_URL = `http://127.0.0.1:${STUB_PORT}`; const APP_URL = "http://localhost:8001"; +// `slowMo` adds an idle delay between every Playwright action (click, fill, +// goto). The recorded videos play at real time, so without slowMo the test +// runs fast enough that a human can't follow what's happening. 250ms feels +// natural in the recording without bloating wall-clock test time too much. +// Coerce + validate the env override so a malformed value (non-numeric → NaN, +// or negative) falls back to the default instead of reaching Playwright. +const DEFAULT_SLOW_MO_MS = 250; +const parsedSlowMo = Number(process.env.E2E_SLOW_MO_MS); +const SLOW_MO_MS = + Number.isFinite(parsedSlowMo) && parsedSlowMo >= 0 + ? parsedSlowMo + : DEFAULT_SLOW_MO_MS; + export default defineConfig({ testDir: "./playwright/tests", outputDir: "./playwright/test-results", @@ -37,6 +50,9 @@ export default defineConfig({ screenshot: "only-on-failure", trace: "retain-on-failure", video: "retain-on-failure", + launchOptions: { + slowMo: SLOW_MO_MS, + }, }, projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], webServer: [ @@ -52,7 +68,15 @@ export default defineConfig({ env: { STUB_PORT: String(STUB_PORT) }, }, { - command: "npm run dev", + // Force the webpack dev server (the default `npm run dev` uses Turbopack). + // Turbopack's dev server corrupts the RSC client manifest when a route is + // recompiled after a full-page navigation cycle (`evalManifest` throws + // "Invalid or unexpected token"), which surfaces as a Runtime Error overlay + // on the *second* cold load of "/" in a session and breaks every test that + // re-navigates there (onboarding variants, the app-shell error state). The + // bug is dev-only and Turbopack-specific, so we opt this run out of it while + // leaving local `npm run dev` on Turbopack for day-to-day speed. + command: "npm run dev -- --webpack", url: APP_URL, // Never reuse an existing dev server: the BACKEND_INTERNAL_URL below is // only applied to a server Playwright starts. A reused server (e.g. a diff --git a/ui/playwright/README.md b/ui/playwright/README.md index 1d4dde1abb..ddd4eff639 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -11,7 +11,7 @@ flows across components**. | Atoms (`src/components/ui/*`) | shadcn primitives | skip | | Visual / render states | Storybook + Vitest-browser + Chromatic | skip | | Unit / logic | Jest (`*.test.ts(x)`) | skip | -| Page-load smoke (`h1` renders, page reachable) | Playwright (`tests/smoke.spec.ts`, Stage 0) | — | +| Page-load smoke (`h1` renders, page reachable) | Playwright (folded into the app-shell journey, `tests/app-shell.spec.ts`) | — | | **Multi-step flows: form submission, payload correctness, streaming, wizard completion, error/edge states** | **Playwright (this suite)** | — | Rule: if Chromatic / Jest already assert it, Playwright does not. @@ -36,12 +36,13 @@ browser-originated and `page.route`-able — used in the chat spec (Stage 2). ``` playwright/ tests/ # *.spec.ts, one per feature area - helpers/ # (Stage 1) reusable drivers: page, forms, select, dialog, nav + helpers/ # reusable drivers: page, nav (forms/select/dialog land with Stage 2) mocks/ - server.mjs # stub backend (runtime source of happy-path data) - data.ts # typed spec-side builders (mirror server.mjs shapes) + server.mjs # stub backend (happy-path data + /__mock/scenario overrides) + data.ts # typed spec-side builders + ok() envelope (mirror server.mjs shapes) + control.ts # semantic mock seam: mock.noAgents(), mock.agentsError(), … fixtures/ - test.ts # import { test, expect } from here in every spec + test.ts # import { test, expect } from here; provides the `mock` fixture tsconfig.json ``` @@ -68,8 +69,20 @@ BACKEND_INTERNAL_URL=http://127.0.0.1:8899/api npm run dev ## Conventions - Import `{ test, expect }` from `../fixtures/test`, never `@playwright/test`. -- One spec file per feature area (`tests/agents/`, `tests/chat/`, …); use - `test.step()` for sub-phases of a multi-step flow. +- One spec file per feature area (`tests/agents/`, `tests/chat/`, …). Each area is + **two journeys → two videos**: + - a **success journey** that opens on the empty state (where one exists) then + walks the whole happy-path lifecycle (create → configure → use → delete → + confirm), one `test.step()` per phase; + - a **failure journey** that consolidates every negative/edge path (validation + blocks, error toasts, not-found states) into `test.step()`s. + + Splitting success from failure keeps a broken edge case from taking the + happy-path video down with it (and vice versa) while still collapsing to ~2 + report entries per area. Because captured requests and scenario overrides + accumulate across steps in one `test()`, each failure step calls `mock.reset()` + first so it starts from a clean slate. The app-shell smoke journey is the one + exception — a single test covering list states + header nav. - **`data-testid` policy:** prefer `getByRole` / `getByLabel`. Add `data-testid` only where role/text is ambiguous or unstable (list rows, per-item action buttons, wizard steps, combobox options). Add incrementally — no upfront sweep. @@ -79,9 +92,19 @@ BACKEND_INTERNAL_URL=http://127.0.0.1:8899/api npm run dev ## Roadmap - **Stage 0 (done):** foundation — config, stub backend, CI, one smoke test. -- **Stage 1:** helper/driver library (`helpers/*`) + per-test scenario overrides - via the stub's `/__mock/scenario` endpoint. Prefer stateless scenario selection - keyed by request content (e.g. `?namespace=empty-ns` → `[]`); fall back to the - control endpoint with `workers: 1` for endpoints lacking a discriminator. -- **Stage 2:** feature flows (gap-scoped), ordered by importance — - Create Agent → Chat/session (A2A SSE mock) → Models → MCP → Onboarding completion. +- **Stage 1 (done):** page/nav helpers (`helpers/*`) + per-test scenario overrides — + the `mock` fixture drives the stub's `/__mock/scenario` endpoint via `mocks/control.ts` + (e.g. `mock.noAgents()`, `mock.agentsError()`), verified by the app-shell journey. + Runs serially (`workers: 1`) against the shared stub; raising the worker count later + needs per-worker servers or stateless request-keyed scenarios. +- **Stage 2:** feature flows (gap-scoped), then consolidated so each feature area + is **one success journey + one failure journey** (see Conventions), landing at + ~14 videos total. Shared infra (POST-capture, A2A SSE mock, `select` helper) is + demand-driven. The feature areas: + - [x] App shell — list states + header nav — `tests/app-shell.spec.ts` + - [x] Agents — create (declarative + harness) & delete — `tests/agents/agents.spec.ts` + - [x] Chat / session (A2A SSE mock) — `tests/chat/chat-session.spec.ts` + - [x] Models / providers — `tests/models/models.spec.ts` + - [x] MCP servers & tools — `tests/mcp/mcp-server.spec.ts` + - [x] Prompt libraries — `tests/prompts/prompt-libraries.spec.ts` + - [x] Onboarding completion — `tests/onboarding/onboarding.spec.ts` diff --git a/ui/playwright/fixtures/test.ts b/ui/playwright/fixtures/test.ts index c9f1659ef7..10969e7de1 100644 --- a/ui/playwright/fixtures/test.ts +++ b/ui/playwright/fixtures/test.ts @@ -1,28 +1,37 @@ // Shared test fixture. Import { test, expect } from here in every spec (not -// directly from @playwright/test) so all specs get the same setup. -// -// For now it resets the stub backend before each test. Reset is a no-op until -// Stage 1 adds per-test scenarios, but establishing the pattern now means specs -// don't change when scenarios land. +// directly from @playwright/test) so all specs get the same setup: +// - `mock`: the stub-backend control seam (mock.noAgents(), mock.agentsError(), …) +// - the stub is reset before every test (page depends on mock), so scenarios +// set by one test never leak into the next. +// - onboarding wizard is bypassed on every navigation. import { test as base, expect } from "@playwright/test"; +import { makeMock, type MockBackend } from "../mocks/control"; -const STUB_URL = "http://127.0.0.1:8899"; - -export const test = base.extend({ - page: async ({ page, request }, provide) => { +export const test = base.extend<{ mock: MockBackend }>({ + // Param is named `run` (not the Playwright-idiomatic `use`) to avoid the + // react-hooks lint rule mistaking it for React 19's use() hook. + mock: async ({ request }, run) => { + const mock = makeMock(request); + // Reset scenario state before each test. The stub is managed by Playwright's + // webServer, so in CI an unreachable stub is a real failure — fail fast. try { - await request.post(`${STUB_URL}/__mock/reset`); + await mock.reset(); } catch (err) { - // Stub should always be reachable in CI (managed via webServer). Fail fast. if (process.env.CI) throw err; } + await run(mock); + }, + // Depend on `mock` so the reset above runs before every test that uses a page, + // even specs that don't touch scenarios (e.g. smoke). + page: async ({ page, mock }, run) => { + void mock; // Bypass the first-run onboarding wizard (AppInitializer redirects to it when // this key is unset). Runs before any page script, on every navigation. await page.addInitScript(() => { window.localStorage.setItem("kagent-onboarding", "true"); }); - await provide(page); + await run(page); }, }); diff --git a/ui/playwright/helpers/a2a.ts b/ui/playwright/helpers/a2a.ts new file mode 100644 index 0000000000..754f11b6c3 --- /dev/null +++ b/ui/playwright/helpers/a2a.ts @@ -0,0 +1,95 @@ +// A2A chat streaming mock. Unlike the /api data calls (server-side → stub), the +// chat SSE call `POST /a2a//` is a real browser fetch, so page.route +// CAN intercept it. We fulfill it with a text/event-stream body of JSON-RPC +// frames (see a2aClient.ts processSSEStream): frames are "\n\n"-delimited, each +// line prefixed "data: ", each payload unwrapped as `.result`. + +import { type Page } from "@playwright/test"; + +// contextId must match the session id the stub's POST /api/sessions returns. +const CONTEXT_ID = "e2e-session"; +const TASK_ID = "e2e-task"; + +interface ToolCall { + name: string; + args?: Record; + result?: string; +} + +function frame(event: unknown): string { + return `data: ${JSON.stringify({ jsonrpc: "2.0", id: "1", result: event })}\n\n`; +} + +function statusUpdate(status: unknown, final = false): unknown { + return { kind: "status-update", taskId: TASK_ID, contextId: CONTEXT_ID, final, status }; +} + +function agentMessage(parts: unknown[], messageId?: number): unknown { + // A numeric messageId on the terminal text message makes the feedback (thumbs) + // buttons render on the agent reply. + return { kind: "message", role: "agent", parts, contextId: CONTEXT_ID, taskId: TASK_ID, messageId, metadata: {} }; +} + +/** Build the SSE body: optional tool call (request + response), then a final agent text. */ +function buildStream({ text, tool }: { text: string; tool?: ToolCall }): string { + const frames: string[] = []; + if (tool) { + frames.push( + frame( + statusUpdate({ + state: "working", + message: agentMessage([ + { + kind: "data", + data: { id: "call-1", name: tool.name, args: tool.args ?? {} }, + metadata: { adk_type: "function_call" }, + }, + ]), + }), + ), + ); + frames.push( + frame( + statusUpdate({ + state: "working", + message: agentMessage([ + { + kind: "data", + data: { id: "call-1", name: tool.name, response: { result: tool.result ?? "ok", isError: false } }, + metadata: { adk_type: "function_response" }, + }, + ]), + }), + ), + ); + } + // Terminal frame settles the turn and carries the agent's text reply. + frames.push( + frame( + statusUpdate( + { state: "completed", message: agentMessage([{ kind: "text", text }], 1) }, + true, + ), + ), + ); + return frames.join("") + "data: [DONE]\n\n"; +} + +/** Intercept the chat SSE call and reply with an agent message (and optional tool call). */ +export async function mockAgentReply( + page: Page, + opts: { text: string; tool?: ToolCall }, +): Promise { + const body = buildStream(opts); + const handler = (route: import("@playwright/test").Route) => + route.fulfill({ status: 200, contentType: "text/event-stream", body }); + await page.route("**/a2a/**", handler); + await page.route("**/a2a-sandboxes/**", handler); +} + +/** Intercept the chat SSE call and fail it (network error), for the failure path. */ +export async function mockAgentStreamError(page: Page): Promise { + const handler = (route: import("@playwright/test").Route) => route.abort("failed"); + await page.route("**/a2a/**", handler); + await page.route("**/a2a-sandboxes/**", handler); +} diff --git a/ui/playwright/helpers/nav.ts b/ui/playwright/helpers/nav.ts new file mode 100644 index 0000000000..d10a5a1bc0 --- /dev/null +++ b/ui/playwright/helpers/nav.ts @@ -0,0 +1,52 @@ +// Navigation helpers for the persistent header (src/components/Header.tsx). +// +// Routes live inside two Radix dropdown menus ("Create" and "View"), not flat +// links. Menu items only enter the DOM (as role="menuitem") once the menu is +// open. Header markup is duplicated for desktop/mobile, but the hidden block is +// out of the accessibility tree, so role-based locators resolve to the visible +// one on a desktop viewport. + +import { type Page } from "@playwright/test"; + +// The full-screen LoadingState overlay (data-testid="loading-overlay") sits on +// top of the header during a route transition. Waiting only for the URL to +// change leaves it covering the menu triggers, so a follow-up menu click can hit +// the overlay and flake. Wait for it to detach before handing control back. +// "hidden" also resolves immediately when the overlay never mounted. +async function waitForOverlayGone(page: Page): Promise { + await page.getByTestId("loading-overlay").waitFor({ state: "hidden" }); +} + +async function openMenu(page: Page, trigger: "Create" | "View"): Promise { + await page.getByRole("button", { name: trigger, exact: true }).click(); +} + +async function chooseFrom( + page: Page, + trigger: "Create" | "View", + item: string, + urlGlob?: string | RegExp, +): Promise { + await openMenu(page, trigger); + // Exact match: "New Agent" is a substring of "New Agent Harness". + await page.getByRole("menuitem", { name: item, exact: true }).click(); + if (urlGlob) await page.waitForURL(urlGlob); + await waitForOverlayGone(page); +} + +/** Open the "View" menu and go to a listing page, e.g. gotoView(page, "Models", "**\/models"). */ +export function gotoView(page: Page, item: string, urlGlob?: string | RegExp): Promise { + return chooseFrom(page, "View", item, urlGlob); +} + +/** Open the "Create" menu and go to a creation page, e.g. gotoCreate(page, "New Agent", "**\/agents/new"). */ +export function gotoCreate(page: Page, item: string, urlGlob?: string | RegExp): Promise { + return chooseFrom(page, "Create", item, urlGlob); +} + +/** Click the direct "Home" link in the header. */ +export async function gotoHome(page: Page): Promise { + await page.getByRole("link", { name: "Home" }).first().click(); + await page.waitForURL(/\/(agents)?$/); + await waitForOverlayGone(page); +} diff --git a/ui/playwright/helpers/page.ts b/ui/playwright/helpers/page.ts new file mode 100644 index 0000000000..0bffb83cd3 --- /dev/null +++ b/ui/playwright/helpers/page.ts @@ -0,0 +1,48 @@ +// Page-level driver helpers. Backend-agnostic — they assert on rendered DOM +// (roles/text), so they work regardless of how data is mocked. + +import { expect, type Page } from "@playwright/test"; + +/** + * Wait for the full-screen LoadingState overlay to clear. It sits on top of the + * page (z-10, backdrop-blur) while data loads, so content behind it counts as + * "visible" to Playwright even though a user can't see/use it yet. Call this + * before asserting on or interacting with page content. + */ +export async function waitForAppReady(page: Page): Promise { + await expect(page.getByTestId("loading-overlay")).toHaveCount(0); +} + +/** Navigate to a path, wait for loading to finish, and optionally assert the page's

. */ +export async function loadPage( + page: Page, + path: string, + opts: { heading?: string } = {}, +): Promise { + await page.goto(path); + await waitForAppReady(page); + if (opts.heading) { + await expect(page.getByRole("heading", { level: 1, name: opts.heading })).toBeVisible(); + } +} + +/** Assert the ErrorState component ("Error Encountered") is not on the page. */ +export async function expectNoErrors(page: Page): Promise { + await expect(page.getByText("Error Encountered")).toHaveCount(0); +} + +type ToastType = "success" | "error" | "warning" | "info"; + +/** + * Assert a sonner toast with the given text is visible. Toasts auto-dismiss, so + * call this promptly after the triggering action. Pass `type` to also assert the + * severity (data-type on the toast
  • ). + */ +export async function expectToast( + page: Page, + text: string | RegExp, + opts: { type?: ToastType } = {}, +): Promise { + const selector = opts.type ? `[data-sonner-toast][data-type="${opts.type}"]` : "[data-sonner-toast]"; + await expect(page.locator(selector).filter({ hasText: text })).toBeVisible(); +} diff --git a/ui/playwright/helpers/select.ts b/ui/playwright/helpers/select.ts new file mode 100644 index 0000000000..f900a3c46d --- /dev/null +++ b/ui/playwright/helpers/select.ts @@ -0,0 +1,20 @@ +// Driver for Radix