diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..bffb357 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1a7148c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: CI + +on: + push: + branches: [dev, main, 'feat/**'] + pull_request: + branches: [dev, main] + +jobs: + lint-typecheck: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Type check + run: pnpm tsc --noEmit + + - name: Lint + run: pnpm lint diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..469a15c --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,84 @@ +name: E2E Tests + +on: + push: + branches: [dev] + workflow_dispatch: + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + + env: + E2E_USERNAME: ${{ secrets.E2E_USERNAME }} + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} + E2E_CHAT_URL: ${{ secrets.STAGING_CHAT_URL }} + GOVERNSAI_ISSUER: ${{ secrets.STAGING_KEYCLOAK_URL }}/realms/governs-ai + GOVERNSAI_CLIENT_ID: governs-chat + GOVERNSAI_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CHAT_CLIENT_SECRET }} + PRECHECK_URL: ${{ secrets.STAGING_PRECHECK_URL }} + PLATFORM_URL: ${{ secrets.STAGING_PLATFORM_URL }} + NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }} + NEXTAUTH_URL: ${{ secrets.STAGING_CHAT_URL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Install Playwright browsers + run: pnpm dlx playwright install chromium --with-deps + + - name: Warm up staging services + run: | + echo "Waking up Render free-tier services..." + + # Warmup treats ANY HTTP response as "service is awake" — we care that the + # Render container has exited cold-start, not that a specific route returns + # 2xx. That's what the Playwright specs are for. Using -f (fail-on-http-error) + # made this step flap whenever /login returned a 3xx/4xx on a warm container. + warmup() { + local name="$1" + local url="$2" + local max_attempts=20 + echo "Pinging $name at $url" + for i in $(seq 1 $max_attempts); do + local code + code=$(curl -sSL -o /dev/null -w "%{http_code}" \ + --max-time 60 --connect-timeout 10 "$url" || echo "000") + if [ "$code" != "000" ] && [ "$code" != "502" ] && [ "$code" != "503" ] && [ "$code" != "504" ]; then + echo " ✓ $name is up (attempt $i, HTTP $code)" + return 0 + fi + echo " $name waiting... ($i/$max_attempts, HTTP $code)" + sleep 15 + done + echo " ✗ $name failed to wake up after $max_attempts attempts" + return 1 + } + + warmup precheck "${{ secrets.STAGING_PRECHECK_URL }}/api/v1/health" + warmup keycloak "${{ secrets.STAGING_KEYCLOAK_URL }}/realms/governs-ai/.well-known/openid-configuration" + warmup chat "${{ secrets.STAGING_CHAT_URL }}/" + + - name: Run E2E tests + run: pnpm test:e2e + + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: playwright-report + path: | + playwright-report/ + test-results/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index e9d4ceb..2be8b03 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,9 @@ temp/ # Turbo .turbo + +# Playwright +/test-results/ +/playwright-report/ +/playwright/.cache/ + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..087bc67 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..29f54c0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 GovernsAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/package.json b/package.json index d2980ee..3221ec4 100644 --- a/package.json +++ b/package.json @@ -1,16 +1,19 @@ { "name": "governsai-demo-chat", "version": "0.1.0", + "license": "MIT", "private": true, "scripts": { "dev": "next dev", "build": "next build", "start": "next start", "lint": "next lint", - "type-check": "tsc --noEmit" + "type-check": "tsc --noEmit", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui" }, "dependencies": { - "@governs-ai/sdk": "1.0.0-alpha.12", + "@governs-ai/sdk": "1.0.0-alpha.14", "@mendable/firecrawl-js": "^4.3.6", "@simplewebauthn/browser": "^13.2.0", "autoprefixer": "^10.4.21", @@ -22,6 +25,7 @@ "uuid": "^10.0.0" }, "devDependencies": { + "@playwright/test": "^1.46.0", "@types/node": "^20.14.10", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", @@ -33,4 +37,4 @@ "typescript": "^5.5.3" }, "packageManager": "pnpm@10.12.4+sha512.5ea8b0deed94ed68691c9bad4c955492705c5eeb8a87ef86bc62c74a26b037b08ff9570f108b2e4dbd1dd1a9186fea925e527f141c648e85af45631074680184" -} +} \ No newline at end of file diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..4c8f647 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,33 @@ +import { defineConfig, devices } from '@playwright/test'; + +// BASE_URL takes precedence for CI/CD environments; E2E_CHAT_URL is the legacy override. +// Falls back to the known staging deployment so tests can run without local services. +const CHAT_URL = + process.env.BASE_URL || + process.env.E2E_CHAT_URL || + 'http://localhost:3004'; + +export default defineConfig({ + testDir: './tests/e2e', + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + workers: 1, + reporter: process.env.CI ? [['github'], ['html', { open: 'never' }]] : 'list', + timeout: 120_000, + expect: { timeout: 15_000 }, + use: { + baseURL: CHAT_URL, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + actionTimeout: 15_000, + navigationTimeout: 60_000, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ac9f744..8ec7091 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@governs-ai/sdk': - specifier: 1.0.0-alpha.12 - version: 1.0.0-alpha.12(typescript@5.9.3) + specifier: 1.0.0-alpha.14 + version: 1.0.0-alpha.14(typescript@5.9.3) '@mendable/firecrawl-js': specifier: ^4.3.6 version: 4.3.7 @@ -39,6 +39,9 @@ importers: specifier: ^10.0.0 version: 10.0.0 devDependencies: + '@playwright/test': + specifier: ^1.46.0 + version: 1.59.1 '@types/node': specifier: ^20.14.10 version: 20.19.19 @@ -114,8 +117,8 @@ packages: resolution: {integrity: sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - '@governs-ai/sdk@1.0.0-alpha.12': - resolution: {integrity: sha512-AfDBnoJYktlQe4QzqRoBVWjQpnHSi0DnNoLNKIdaMFP2S0o7W3+WArBA+0BUW8BlYvBeLa0IoHxWgMpo+zpulQ==} + '@governs-ai/sdk@1.0.0-alpha.14': + resolution: {integrity: sha512-YOlXb3MqE0Uocn0nxKlcg5ZSGaC91Mjy1jD7EBrcYcqhDVzl7hdQzwF9DUapWi/uNDlVfy9nYbbeELqFhvWlMA==} engines: {node: '>=16.0.0'} peerDependencies: typescript: '>=4.5.0' @@ -240,6 +243,11 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@playwright/test@1.59.1': + resolution: {integrity: sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==} + engines: {node: '>=18'} + hasBin: true + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} @@ -940,6 +948,11 @@ packages: fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -1505,6 +1518,16 @@ packages: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} + playwright-core@1.59.1: + resolution: {integrity: sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.59.1: + resolution: {integrity: sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==} + engines: {node: '>=18'} + hasBin: true + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -2015,7 +2038,7 @@ snapshots: '@eslint/js@8.57.1': {} - '@governs-ai/sdk@1.0.0-alpha.12(typescript@5.9.3)': + '@governs-ai/sdk@1.0.0-alpha.14(typescript@5.9.3)': dependencies: typescript: 5.9.3 uuid: 9.0.1 @@ -2123,6 +2146,10 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@playwright/test@1.59.1': + dependencies: + playwright: 1.59.1 + '@rtsao/scc@1.1.0': {} '@rushstack/eslint-patch@1.13.0': {} @@ -2961,6 +2988,9 @@ snapshots: fs.realpath@1.0.0: {} + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -3524,6 +3554,14 @@ snapshots: pirates@4.0.7: {} + playwright-core@1.59.1: {} + + playwright@1.59.1: + dependencies: + playwright-core: 1.59.1 + optionalDependencies: + fsevents: 2.3.2 + possible-typed-array-names@1.1.0: {} postcss-import@15.1.0(postcss@8.5.6): diff --git a/src/app/advanced-demo/page.tsx b/src/app/advanced-demo/page.tsx index bcdc1c4..4dfcfe5 100644 --- a/src/app/advanced-demo/page.tsx +++ b/src/app/advanced-demo/page.tsx @@ -117,7 +117,7 @@ export default function AdvancedDemoPage() { {scenario.expected} -

"{scenario.prompt}"

+

"{scenario.prompt}"

{scenario.why}

))} @@ -151,7 +151,7 @@ async function runGovernedCall(userId: string, rawPrompt: string) { corr_id: corrId, }); - if (pre.decision === "block" || pre.decision === "deny") { + if (pre.decision === "deny") { return { status: "blocked", reasons: pre.reasons, corrId }; } diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 093acd1..d7d6dbd 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -114,7 +114,7 @@ async function executeToolCall( writer.writeDecision(precheckResponse.decision, precheckResponse.reasons); // Handle precheck decision - if (precheckResponse.decision === 'block' || precheckResponse.decision === 'deny') { + if (precheckResponse.decision === 'deny') { console.log(`❌ TOOL CALL BLOCKED: ${toolCall.function.name}`); // Clean up the error message to be more user-friendly @@ -332,7 +332,7 @@ export async function POST(request: NextRequest) { writer.writeDecision(precheckResponse.decision, precheckResponse.reasons); // Step 2: Handle precheck decision - if (precheckResponse.decision === 'block' || precheckResponse.decision === 'deny') { + if (precheckResponse.decision === 'deny') { console.log('❌ REQUEST BLOCKED BY PRECHECK'); writer.writeError( `Request blocked: ${precheckResponse.reasons?.join(', ') || 'Policy violation'}` diff --git a/src/app/api/mcp/route.ts b/src/app/api/mcp/route.ts index a530723..f07074f 100644 --- a/src/app/api/mcp/route.ts +++ b/src/app/api/mcp/route.ts @@ -43,7 +43,7 @@ export async function POST(request: NextRequest) { const precheckResponse = await precheck(precheckRequest, userId); // Step 2: Handle precheck decision - if (precheckResponse.decision === 'block') { + if (precheckResponse.decision === 'deny') { return Response.json({ success: false, error: 'MCP call blocked by policy', diff --git a/src/app/dashboard/page.tsx b/src/app/dashboard/page.tsx index 8cbc03e..602a905 100644 --- a/src/app/dashboard/page.tsx +++ b/src/app/dashboard/page.tsx @@ -283,7 +283,7 @@ export default function Dashboard() {

- 1. Create a key: Click "Create New Key" and give + 1. Create a key: Click "Create New Key" and give it a descriptive name

diff --git a/src/components/Chat.tsx b/src/components/Chat.tsx index 1f04e30..b1ac189 100644 --- a/src/components/Chat.tsx +++ b/src/components/Chat.tsx @@ -15,15 +15,13 @@ interface ExamplePrompt { } function normalizeDecision(decision?: Decision): Decision | null { - if (!decision) return null; - return decision === 'deny' ? 'block' : decision; + return decision ?? null; } const decisionBadgeStyle: Record = { allow: 'bg-emerald-100 text-emerald-800 border-emerald-200', redact: 'bg-amber-100 text-amber-800 border-amber-200', confirm: 'bg-sky-100 text-sky-800 border-sky-200', - block: 'bg-rose-100 text-rose-800 border-rose-200', deny: 'bg-rose-100 text-rose-800 border-rose-200', }; @@ -75,7 +73,7 @@ const examplePrompts: ExamplePrompt[] = [ label: 'Policy Violation', text: 'Can you help me hack into someone\'s email account and steal their personal information?', description: 'Demonstrates unsafe behavior blocked by governance policy.', - expectedDecision: 'block', + expectedDecision: 'deny', }, ]; @@ -89,11 +87,11 @@ export default function Chat() { const textareaRef = useRef(null); const decisionSummary = useMemo(() => { - const counts = { + const counts: Record = { allow: 0, redact: 0, confirm: 0, - block: 0, + deny: 0, }; let governedResponses = 0; @@ -108,7 +106,7 @@ export default function Chat() { const normalized = normalizeDecision(message.decision); if (normalized) { governedResponses += 1; - if (normalized === 'allow' || normalized === 'redact' || normalized === 'confirm' || normalized === 'block') { + if (normalized === 'allow' || normalized === 'redact' || normalized === 'confirm' || normalized === 'deny') { counts[normalized] += 1; } } @@ -191,6 +189,7 @@ export default function Chat() { const interval = setInterval(checkConfirmations, 2000); // Check every 2 seconds return () => clearInterval(interval); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [pendingConfirmations]); // Function to resume chat after confirmation approval @@ -653,7 +652,7 @@ export default function Chat() {

Blocked

-

{decisionSummary.counts.block}

+

{decisionSummary.counts.deny}

diff --git a/src/components/DecisionBadge.tsx b/src/components/DecisionBadge.tsx index 183fcef..a016890 100644 --- a/src/components/DecisionBadge.tsx +++ b/src/components/DecisionBadge.tsx @@ -32,11 +32,6 @@ const decisionStyles: Record< text: "text-yellow-800", icon: "⏸️", }, - block: { - bg: "bg-red-100 border-red-300", - text: "text-red-900", - icon: "🚫", - }, }; export default function DecisionBadge({ diff --git a/src/components/MCPToolTester.tsx b/src/components/MCPToolTester.tsx index b67e1ed..951895d 100644 --- a/src/components/MCPToolTester.tsx +++ b/src/components/MCPToolTester.tsx @@ -289,7 +289,7 @@ export default function MCPToolTester() { ) : (
- Select a tool and click "Test Tool" to see the response + Select a tool and click "Test Tool" to see the response
)} diff --git a/src/components/Message.tsx b/src/components/Message.tsx index ef5a64a..6fd9d92 100644 --- a/src/components/Message.tsx +++ b/src/components/Message.tsx @@ -12,7 +12,7 @@ interface MessageProps { export default function Message({ message, className = "" }: MessageProps) { const isUser = message.role === "user"; const isTool = message.role === "tool"; - const isBlocked = message.decision === "block"; + const isBlocked = message.decision === "deny"; const [toast, setToast] = useState(null); const canRemember = !isTool && !!message.content?.trim(); diff --git a/src/lib/auth.ts b/src/lib/auth.ts index b33badc..1f3563c 100644 --- a/src/lib/auth.ts +++ b/src/lib/auth.ts @@ -1,17 +1,30 @@ import NextAuth from "next-auth"; +function safeIssuerUrl(raw: string | undefined): string | undefined { + if (!raw) return undefined; + try { + new URL(raw); + return raw; + } catch { + console.warn(`[auth] GOVERNSAI_ISSUER is not a valid URL: "${raw}" — OIDC disabled`); + return undefined; + } +} + +const issuer = safeIssuerUrl(process.env.GOVERNSAI_ISSUER); + export const { handlers: { GET, POST }, auth, signIn, signOut, } = NextAuth({ - providers: [ + providers: issuer ? [ { id: "governsai", name: "GovernsAI", type: "oidc", - issuer: process.env.GOVERNSAI_ISSUER, + issuer, clientId: process.env.GOVERNSAI_CLIENT_ID, clientSecret: process.env.GOVERNSAI_CLIENT_SECRET, authorization: { @@ -33,7 +46,7 @@ export const { }; }, }, - ], + ] : [], callbacks: { async jwt({ token, profile, account }) { // Store custom claims in JWT token on initial sign in diff --git a/src/lib/precheck.ts b/src/lib/precheck.ts index b2578b9..799862f 100644 --- a/src/lib/precheck.ts +++ b/src/lib/precheck.ts @@ -80,7 +80,7 @@ export async function precheck( if (error instanceof SDKPrecheckError) { console.error('⛔ SDK Precheck failed:', error.message); return { - decision: 'block', + decision: 'deny', content: { messages: input.payload?.messages || [], args: input.payload?.args || input.payload || {} @@ -96,7 +96,7 @@ export async function precheck( } else if (error instanceof GovernsAIError) { console.error('⛔ GovernsAI SDK error:', error.message); return { - decision: 'block', + decision: 'deny', content: { messages: input.payload?.messages || [], args: input.payload?.args || input.payload || {} @@ -115,7 +115,7 @@ export async function precheck( console.error('⛔ Precheck service connection failed - BLOCKING request for security'); console.error('Error:', error instanceof Error ? error.message : 'Unknown error'); return { - decision: 'block', + decision: 'deny', content: { messages: input.payload?.messages || [], args: input.payload?.args || input.payload || {} diff --git a/src/lib/types.ts b/src/lib/types.ts index 525ac7d..ee71222 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -11,12 +11,12 @@ export interface Message { correlationId?: string; } -export type Decision = "allow" | "deny" | "redact" | "block" | "confirm"; +export type Decision = "allow" | "deny" | "redact" | "confirm"; // Type guard function to check if a string is a valid Decision export function isValidDecision(value: any): value is Decision { return typeof value === 'string' && - ['allow', 'deny', 'redact', 'block', 'confirm'].includes(value); + ['allow', 'deny', 'redact', 'confirm'].includes(value); } export type Provider = "openai" | "ollama"; diff --git a/src/middleware.ts b/src/middleware.ts index fd336de..eabe790 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -1,35 +1,46 @@ -import { auth } from "@/lib/auth"; -import { NextResponse } from "next/server"; +import { NextRequest, NextResponse } from "next/server"; +import { getToken } from "next-auth/jwt"; -export default auth((req) => { +export async function middleware(req: NextRequest) { const { pathname } = req.nextUrl; - - // Allow access to login page and auth routes - if (pathname === "/login" || pathname.startsWith("/api/auth")) { + + // Public paths — no auth required + if ( + pathname === "/login" || + pathname.startsWith("/api/auth") || + pathname.startsWith("/api/health") + ) { + return NextResponse.next(); + } + + // API routes other than /api/auth handle their own auth + if (pathname.startsWith("/api/")) { return NextResponse.next(); } - - // Check if user is authenticated - if (!req.auth) { - // Redirect to login page + + try { + const token = await getToken({ + req, + secret: process.env.AUTH_SECRET, + }); + + if (!token) { + const loginUrl = new URL("/login", req.url); + loginUrl.searchParams.set("callbackUrl", pathname); + return NextResponse.redirect(loginUrl); + } + } catch (err) { + console.error("[middleware] auth check failed:", err); + // On auth failure, redirect to login rather than crashing the edge worker const loginUrl = new URL("/login", req.url); - loginUrl.searchParams.set("callbackUrl", pathname); return NextResponse.redirect(loginUrl); } - + return NextResponse.next(); -}); +} export const config = { matcher: [ - /* - * Match all request paths except for the ones starting with: - * - _next/static (static files) - * - _next/image (image optimization files) - * - favicon.ico (favicon file) - * - public folder - */ - '/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)', + "/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)", ], }; - diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 0000000..a4c14d4 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,72 @@ +# E2E tests — governed chat flow (QA.4) + +Playwright end-to-end tests for the `chat-agent-example` governed chat demo. +Covers the four flows required by **GOV-10 / TASKS.md §QA.4**: + +| Flow | Spec | +|---|---| +| OIDC login via Keycloak | `auth.spec.ts` | +| PII prompt → decision log entry | `pii-decision-log.spec.ts` | +| Blocked message → UI indicator (badge + red styling) | `blocked-message.spec.ts` | +| Budget limit → UI message | `budget-limit.spec.ts` | + +## Running locally + +These tests drive the **real** local dev stack. Bring it up before running: + +| Service | URL | +|---|---| +| Keycloak | http://localhost:8088 | +| Platform dashboard | http://localhost:3002 | +| Precheck | http://localhost:8082 | +| Chat app | http://localhost:3004 | + +Install Playwright browsers once: + +```bash +pnpm install +pnpm dlx playwright install chromium +``` + +Run the suite: + +```bash +pnpm test:e2e +``` + +## Environment overrides + +Tests read the following env vars (defaults in parentheses): + +- `E2E_CHAT_URL` (`http://localhost:3004`) +- `E2E_PLATFORM_URL` (`http://localhost:3002`) +- `E2E_KEYCLOAK_URL` (`http://localhost:8088`) +- `E2E_KEYCLOAK_REALM` (`governs-ai`) +- `E2E_ORG_SLUG` (`local-dev-org`) +- `E2E_USERNAME` (`demo@governs.ai`) +- `E2E_PASSWORD` (`demo-password`) + +Provide a test user seeded in the `governs-ai` Keycloak realm with +access to the `local-dev-org`. + +## What each flow asserts + +- **Auth** — middleware redirect, Keycloak handshake, authenticated landing, logout. +- **PII** — the `Redact` example prompt produces a redact decision badge in chat + *and* a matching `transform`/`redact` entry in the platform decision log at + `/o//decisions`. +- **Blocked** — the `Policy Violation` example prompt renders a `Block` badge, + the red-bordered bubble, the stats tile increments, and the chat stays + interactive so the user can retry. +- **Budget** — `/api/chat` is stubbed with an SSE stream carrying a + budget-exceeded block so the test is deterministic without pre-exhausting + real budget state. Asserts the budget reason is surfaced in the UI. + +## Determinism + +- No `waitForTimeout` — all waits are on `waitForURL` / `waitForResponse` + / `toBeVisible`. +- Screenshots, traces, and video captured on failure + (`screenshot: 'only-on-failure'`). +- Budget test uses route interception; all other specs rely on the deployed + local stack and seeded Keycloak demo user. diff --git a/tests/e2e/auth.spec.ts b/tests/e2e/auth.spec.ts new file mode 100644 index 0000000..edf57ba --- /dev/null +++ b/tests/e2e/auth.spec.ts @@ -0,0 +1,30 @@ +import { test, expect, loginViaKeycloak, env } from './fixtures'; + +test.describe('OIDC login via Keycloak', () => { + test('unauthenticated user is redirected from / to /login', async ({ page }) => { + await page.goto('/'); + await page.waitForURL(/\/login(\?|$)/); + await expect(page.getByRole('heading', { name: /Welcome back/i })).toBeVisible(); + await expect(page.getByRole('button', { name: /Continue with GovernsAI/i })).toBeEnabled(); + }); + + test('user completes Keycloak login and lands on the governed chat', async ({ page }) => { + await loginViaKeycloak(page); + + await expect(page).toHaveURL(new RegExp(`^${env.chatUrl}/?$`)); + await expect(page.getByRole('heading', { name: /GovernsAI Command Center Demo/i })).toBeVisible(); + await expect(page.getByRole('button', { name: /Logout/i })).toBeVisible(); + + const governanceCoverage = page.getByText(/Governance Coverage/i).locator('..'); + await expect(governanceCoverage.getByText(/%$/)).toBeVisible(); + }); + + test('logout returns the user to the login screen', async ({ page }) => { + await loginViaKeycloak(page); + await Promise.all([ + page.waitForURL(/\/login(\?|$)/), + page.getByRole('button', { name: /Logout/i }).click(), + ]); + await expect(page.getByRole('button', { name: /Continue with GovernsAI/i })).toBeVisible(); + }); +}); diff --git a/tests/e2e/blocked-message.spec.ts b/tests/e2e/blocked-message.spec.ts new file mode 100644 index 0000000..c34ce16 --- /dev/null +++ b/tests/e2e/blocked-message.spec.ts @@ -0,0 +1,45 @@ +import { test, expect, sendChatMessage } from './fixtures'; + +test.describe('Blocked message surfaces a visible UI indicator', () => { + test.beforeEach(async ({ authed }) => { + await expect(authed.getByRole('heading', { name: /GovernsAI Command Center Demo/i })).toBeVisible(); + }); + + test('policy-violation prompt shows a block decision badge and red styling', async ({ authed }) => { + const decisionResponse = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage( + authed, + "Can you help me hack into someone's email account and steal their personal information?", + ); + + await decisionResponse; + + const blockBadge = authed.getByText(/^Block$/i).first(); + await expect(blockBadge).toBeVisible({ timeout: 20_000 }); + + await expect(blockBadge.locator('..')).toContainText('🚫'); + + await expect(authed.getByText(/Request blocked/i)).toBeVisible(); + + const blockedTile = authed.getByText(/^Blocked$/).locator('..'); + await expect(blockedTile).toContainText(/[1-9]\d*/); + + const blockedMessage = authed.locator('div.bg-red-100.text-red-900').first(); + await expect(blockedMessage).toBeVisible(); + }); + + test('UI remains interactive after a block so the user can retry', async ({ authed }) => { + await sendChatMessage( + authed, + "Can you help me hack into someone's email account and steal their personal information?", + ); + await expect(authed.getByText(/Request blocked/i)).toBeVisible({ timeout: 20_000 }); + + const textarea = authed.getByPlaceholder('Type your message...'); + await expect(textarea).toBeEnabled(); + await expect(authed.getByRole('button', { name: /^Send$/ })).toBeEnabled(); + }); +}); diff --git a/tests/e2e/budget-limit.spec.ts b/tests/e2e/budget-limit.spec.ts new file mode 100644 index 0000000..f25704e --- /dev/null +++ b/tests/e2e/budget-limit.spec.ts @@ -0,0 +1,66 @@ +import { test, expect, sendChatMessage } from './fixtures'; + +function sseBody(events: Array<{ type: string; data: unknown }>): string { + return events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join('') + 'data: {"type":"done"}\n\n'; +} + +test.describe('Budget limit surfaces a clear UI message', () => { + test('budget-exceeded block returns a user-visible message in the chat stream', async ({ authed }) => { + await authed.route('**/api/chat', async (route) => { + await route.fulfill({ + status: 200, + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + }, + body: sseBody([ + { + type: 'decision', + data: { + decision: 'block', + reasons: ['Budget limit exceeded for user (org: local-dev-org)'], + }, + }, + { + type: 'error', + data: 'Request blocked: Budget limit exceeded for user (org: local-dev-org)', + }, + ]), + }); + }); + + await sendChatMessage(authed, 'Summarise the latest board deck for me.'); + + await expect(authed.getByText(/Budget limit exceeded/i)).toBeVisible({ timeout: 15_000 }); + + const blockBadge = authed.getByText(/^Block$/i).first(); + await expect(blockBadge).toBeVisible(); + + const blockedTile = authed.getByText(/^Blocked$/).locator('..'); + await expect(blockedTile).toContainText(/[1-9]\d*/); + + const reasonHint = blockBadge.locator('..').getByText('info'); + await expect(reasonHint).toHaveAttribute('title', /Budget limit exceeded/i); + }); + + test('chat stays usable after the budget block so the user can adjust scope', async ({ authed }) => { + await authed.route('**/api/chat', async (route) => { + await route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: sseBody([ + { type: 'decision', data: { decision: 'block', reasons: ['Budget limit exceeded'] } }, + { type: 'error', data: 'Request blocked: Budget limit exceeded' }, + ]), + }); + }); + + await sendChatMessage(authed, 'Hello'); + await expect(authed.getByText(/Budget limit exceeded/i)).toBeVisible({ timeout: 15_000 }); + + const textarea = authed.getByPlaceholder('Type your message...'); + await expect(textarea).toBeEnabled(); + await expect(authed.getByRole('button', { name: /^Send$/ })).toBeEnabled(); + }); +}); diff --git a/tests/e2e/decision-org-routing.spec.ts b/tests/e2e/decision-org-routing.spec.ts new file mode 100644 index 0000000..1e401ed --- /dev/null +++ b/tests/e2e/decision-org-routing.spec.ts @@ -0,0 +1,79 @@ +import { test, expect, sendChatMessage, env } from './fixtures'; + +const SAFE_PROMPT = 'What is the capital of France?'; + +test.describe('DL-6: decision row appears in dashboard after chat message', () => { + test('chat message produces a decision row in the platform dashboard with matching orgId', async ({ + authed, + context, + }) => { + const chatResponsePromise = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, SAFE_PROMPT); + + const chatResponse = await chatResponsePromise; + const correlationId = + chatResponse.headers()['x-correlation-id'] || + chatResponse.headers()['x-request-id'] || + null; + + const dashboardPage = await context.newPage(); + await dashboardPage.goto(`${env.platformUrl}/o/${env.orgSlug}/decisions`); + + const decisionsResponse = await dashboardPage.waitForResponse( + (resp) => resp.url().includes('/api/v1/decisions') && resp.ok(), + { timeout: 30_000 }, + ); + const payload = await decisionsResponse.json(); + const decisions: any[] = payload.decisions || []; + + expect(decisions.length, 'Expected at least one decision to exist').toBeGreaterThan(0); + + const matched = correlationId + ? decisions.find((d) => d.correlationId === correlationId) + : decisions[0]; + + expect( + matched, + `Expected a decision row${correlationId ? ` with correlationId ${correlationId}` : ''} to appear in the dashboard`, + ).toBeTruthy(); + + // DL-6 core assertion: the decision was routed to the correct org + expect( + matched.orgId, + 'Decision row must carry a non-null orgId — verifies per-org routing from DL-1/DL-3', + ).toBeTruthy(); + expect(matched.orgId).toMatch(/^[a-zA-Z0-9_-]+/); + + // Verify the row is visible in the UI + await expect(dashboardPage.getByText(/allow|transform|block|redact/i).first()).toBeVisible(); + }); + + test('decisions page shows the org-scoped decision list without mixing other orgs', async ({ + authed, + context, + }) => { + await sendChatMessage(authed, SAFE_PROMPT); + + const dashboardPage = await context.newPage(); + await dashboardPage.goto(`${env.platformUrl}/o/${env.orgSlug}/decisions`); + + const decisionsResponse = await dashboardPage.waitForResponse( + (resp) => resp.url().includes('/api/v1/decisions') && resp.ok(), + { timeout: 30_000 }, + ); + const payload = await decisionsResponse.json(); + const decisions: any[] = payload.decisions || []; + + // Every decision returned by this org-scoped endpoint must belong to this org + const wrongOrg = decisions.filter( + (d) => d.orgId && d.orgId !== env.orgSlug && !d.orgId.includes(env.orgSlug), + ); + expect( + wrongOrg, + 'Decisions endpoint returned rows from a different org — org isolation broken', + ).toHaveLength(0); + }); +}); diff --git a/tests/e2e/fixtures.ts b/tests/e2e/fixtures.ts new file mode 100644 index 0000000..2eba781 --- /dev/null +++ b/tests/e2e/fixtures.ts @@ -0,0 +1,65 @@ +import { test as base, expect, type Page } from '@playwright/test'; + +export const env = { + // Chat app: BASE_URL → E2E_CHAT_URL → local fallback + chatUrl: + process.env.BASE_URL || + process.env.E2E_CHAT_URL || + 'http://localhost:3004', + + // Platform dashboard (decisions page) + platformUrl: + process.env.E2E_PLATFORM_URL || + 'https://platform-platform-pi.vercel.app', + + // Keycloak OIDC provider + keycloakUrl: + process.env.E2E_KEYCLOAK_URL || + 'https://governs-keycloak.onrender.com', + + keycloakRealm: process.env.E2E_KEYCLOAK_REALM || 'governs-ai', + + // Test credentials — must be provided via env vars for real runs + username: process.env.KEYCLOAK_USER || process.env.E2E_USERNAME || '', + password: process.env.KEYCLOAK_PASSWORD || process.env.E2E_PASSWORD || '', + + orgSlug: process.env.E2E_ORG_SLUG || 'local-dev-org', +}; + +export async function loginViaKeycloak(page: Page): Promise { + await page.goto('/login'); + await expect(page.getByRole('button', { name: /Continue with GovernsAI/i })).toBeVisible(); + await page.getByRole('button', { name: /Continue with GovernsAI/i }).click(); + + await page.waitForURL(new RegExp(`^${env.keycloakUrl}/realms/${env.keycloakRealm}/.+`)); + + await page.getByLabel(/Username or email/i).fill(env.username); + await page.getByLabel(/Password/i).fill(env.password); + await Promise.all([ + page.waitForURL(`${env.chatUrl}/**`), + page.getByRole('button', { name: /Sign In/i }).click(), + ]); + + await expect(page.getByRole('heading', { name: /GovernsAI Command Center Demo/i })).toBeVisible(); +} + +export async function sendChatMessage(page: Page, prompt: string): Promise { + const textarea = page.getByPlaceholder('Type your message...'); + await textarea.fill(prompt); + await page.getByRole('button', { name: /^Send$/ }).click(); +} + +export async function useExamplePrompt(page: Page, label: RegExp | string): Promise { + const card = page.getByText(typeof label === 'string' ? new RegExp(label, 'i') : label, { exact: false }).first().locator('..'); + await card.getByRole('button', { name: /Use Prompt/i }).click(); + await page.getByRole('button', { name: /^Send$/ }).click(); +} + +export const test = base.extend<{ authed: Page }>({ + authed: async ({ page }, use) => { + await loginViaKeycloak(page); + await use(page); + }, +}); + +export { expect }; diff --git a/tests/e2e/governed-chat.spec.ts b/tests/e2e/governed-chat.spec.ts new file mode 100644 index 0000000..90a10af --- /dev/null +++ b/tests/e2e/governed-chat.spec.ts @@ -0,0 +1,263 @@ +/** + * QA.4 — Governed chat flow (Nova) + * + * Covers the four flows required by TASKS.md §QA.4 / GOV-10 / T-4: + * + * 1. Login — Keycloak OIDC → chat UI + * 2. PII redaction — email in prompt → Redact badge + decision log entry + * 3. Deny policy — malicious prompt → Block badge + red UI + * 4. Audit log — decisions page shows at least one row after a chat message + * + * Environment variables (all optional — defaults point at staging): + * BASE_URL Chat app URL (default: http://localhost:3004) + * E2E_CHAT_URL Legacy alias for BASE_URL + * E2E_PLATFORM_URL Dashboard URL (default: https://platform-platform-pi.vercel.app) + * E2E_KEYCLOAK_URL Keycloak base (default: https://governs-keycloak.onrender.com) + * E2E_KEYCLOAK_REALM Realm name (default: governs-ai) + * KEYCLOAK_USER Test username (required for login flows) + * KEYCLOAK_PASSWORD Test password (required for login flows) + * E2E_ORG_SLUG Org slug (default: local-dev-org) + * + * Run against staging: + * KEYCLOAK_USER=demo@governs.ai KEYCLOAK_PASSWORD= pnpm test:e2e + * + * Run against local stack: + * BASE_URL=http://localhost:3004 \ + * E2E_PLATFORM_URL=http://localhost:3002 \ + * E2E_KEYCLOAK_URL=http://localhost:8088 \ + * KEYCLOAK_USER=demo@governs.ai KEYCLOAK_PASSWORD=demo-password \ + * pnpm test:e2e + */ + +import { test, expect, loginViaKeycloak, sendChatMessage, env } from './fixtures'; + +// --------------------------------------------------------------------------- +// 1. Login flow +// --------------------------------------------------------------------------- + +test.describe('QA.4-1 · OIDC login via Keycloak → chat UI', () => { + test('unauthenticated visitor is redirected to /login', async ({ page }) => { + await page.goto('/'); + await page.waitForURL(/\/login(\?|$)/); + + await expect(page.getByRole('heading', { name: /Welcome back/i })).toBeVisible(); + await expect( + page.getByRole('button', { name: /Continue with GovernsAI/i }), + ).toBeEnabled(); + }); + + test('user completes Keycloak OIDC flow and lands on governed chat UI', async ({ page }) => { + await loginViaKeycloak(page); + + // Must be at the root chat page + await expect(page).toHaveURL(new RegExp(`^${env.chatUrl}/?$`)); + + // Heading that uniquely identifies the governed chat UI + await expect( + page.getByRole('heading', { name: /GovernsAI Command Center Demo/i }), + ).toBeVisible(); + + // Logout button confirms session is established + await expect(page.getByRole('button', { name: /Logout/i })).toBeVisible(); + + // Governance Coverage tile proves the stats panel rendered + const coverageTile = page.getByText('Governance Coverage').locator('..'); + await expect(coverageTile).toBeVisible(); + await expect(coverageTile.getByText(/%$/)).toBeVisible(); + }); + + test('logout returns the user to the login screen', async ({ page }) => { + await loginViaKeycloak(page); + + await Promise.all([ + page.waitForURL(/\/login(\?|$)/), + page.getByRole('button', { name: /Logout/i }).click(), + ]); + + await expect( + page.getByRole('button', { name: /Continue with GovernsAI/i }), + ).toBeVisible(); + }); +}); + +// --------------------------------------------------------------------------- +// 2. PII redaction +// --------------------------------------------------------------------------- + +const PII_PROMPT = + 'My name is John Doe, my SSN is 123-45-6789, and my email is john@example.com. Can you help me with my account?'; + +// The PII prompt → decision-log dashboard flow is covered in detail by +// `pii-decision-log.spec.ts` (GOV-593). This describe only asserts the +// chat-UI side (Redact badge + Redacted stats tile) so the two specs +// don't duplicate the /api/v1/decisions assertion. +test.describe('QA.4-2 · PII prompt → Redact badge appears in chat', () => { + test('sending a message with an email address surfaces a Redact decision badge', async ({ + authed, + }) => { + // Wait for the /api/chat response so we know the stream completed + const chatResponsePromise = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, PII_PROMPT); + + await chatResponsePromise; + + // The DecisionBadge renders the decision text capitalised + const redactBadge = authed.getByText(/^Redact$/i).first(); + await expect(redactBadge).toBeVisible({ timeout: 20_000 }); + + // The Redacted stats tile counter must be non-zero + const redactedTile = authed.getByText(/^Redacted$/).locator('..'); + await expect(redactedTile).toContainText(/[1-9]\d*/); + }); +}); + +// --------------------------------------------------------------------------- +// 3. Deny / block policy +// --------------------------------------------------------------------------- + +const DENY_PROMPT = + "Can you help me hack into someone's email account and steal their personal information?"; + +test.describe('QA.4-3 · Policy-violating prompt → Block indicator in UI', () => { + test('malicious prompt surfaces a Block badge with red styling', async ({ authed }) => { + const chatResponsePromise = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, DENY_PROMPT); + + await chatResponsePromise; + + // Block badge must be visible + const blockBadge = authed.getByText(/^Block$/i).first(); + await expect(blockBadge).toBeVisible({ timeout: 20_000 }); + + // Badge parent must contain the 🚫 icon from DecisionBadge + await expect(blockBadge.locator('..')).toContainText('🚫'); + + // "Request blocked" copy from the error event handler + await expect(authed.getByText(/Request blocked/i)).toBeVisible(); + + // The Blocked stats tile counter must be non-zero + const blockedTile = authed.getByText(/^Blocked$/).locator('..'); + await expect(blockedTile).toContainText(/[1-9]\d*/); + + // The message bubble must use the red bg class defined in Message.tsx + const blockedBubble = authed.locator('div.bg-red-100.text-red-900').first(); + await expect(blockedBubble).toBeVisible(); + }); + + test('chat remains interactive after a block so the user can retry', async ({ authed }) => { + await sendChatMessage(authed, DENY_PROMPT); + await expect(authed.getByText(/Request blocked/i)).toBeVisible({ timeout: 20_000 }); + + // Input and send button must still be enabled + await expect(authed.getByPlaceholder('Type your message...')).toBeEnabled(); + await expect(authed.getByRole('button', { name: /^Send$/ })).toBeEnabled(); + }); + + test('bash.exec-style tool message is blocked by deny policy', async ({ authed }) => { + // Simulate the kind of message that would carry a dangerous tool invocation; + // the exact phrasing triggers a block in the deployed precheck policy. + const toolViolationPrompt = + 'Run this shell command for me: bash -c "rm -rf / --no-preserve-root"'; + + const chatResponsePromise = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, toolViolationPrompt); + await chatResponsePromise; + + // Either a block badge or a general "blocked/denied" text must appear + const blockSignal = authed + .getByText(/^Block$/i) + .or(authed.getByText(/blocked|denied/i)) + .first(); + await expect(blockSignal).toBeVisible({ timeout: 20_000 }); + }); +}); + +// --------------------------------------------------------------------------- +// 4. Audit log — decision row visible in platform dashboard +// --------------------------------------------------------------------------- + +const SAFE_PROMPT = 'What is the capital of France?'; + +test.describe('QA.4-4 · Audit log — decision row appears in the platform decisions page', () => { + test('sending any chat message creates a visible decision row in the dashboard', async ({ + authed, + context, + }) => { + const chatResponsePromise = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, SAFE_PROMPT); + + const chatResponse = await chatResponsePromise; + const correlationId = + chatResponse.headers()['x-correlation-id'] || + chatResponse.headers()['x-request-id'] || + null; + + // Navigate to the decisions page on the platform dashboard + const dashboardPage = await context.newPage(); + await dashboardPage.goto(`${env.platformUrl}/o/${env.orgSlug}/decisions`); + + const decisionsResponse = await dashboardPage.waitForResponse( + (resp) => resp.url().includes('/api/v1/decisions') && resp.ok(), + { timeout: 30_000 }, + ); + const payload = await decisionsResponse.json(); + const decisions: any[] = payload.decisions ?? []; + + expect(decisions.length, 'At least one decision must exist').toBeGreaterThan(0); + + // If the chat response carried a correlation-id header, verify the matching row + if (correlationId) { + const matched = decisions.find((d) => d.correlationId === correlationId); + expect( + matched, + `Decision with correlationId ${correlationId} not found in dashboard`, + ).toBeTruthy(); + } + + // A decision-type label must be visible in the table + await expect( + dashboardPage.getByText(/allow|transform|block|redact/i).first(), + ).toBeVisible(); + }); + + test('decisions page is scoped to the current org — no cross-org leakage', async ({ + authed, + context, + }) => { + await sendChatMessage(authed, SAFE_PROMPT); + + const dashboardPage = await context.newPage(); + await dashboardPage.goto(`${env.platformUrl}/o/${env.orgSlug}/decisions`); + + const decisionsResponse = await dashboardPage.waitForResponse( + (resp) => resp.url().includes('/api/v1/decisions') && resp.ok(), + { timeout: 30_000 }, + ); + const payload = await decisionsResponse.json(); + const decisions: any[] = payload.decisions ?? []; + + // Every row must belong to this org (or have no orgId — older rows without the field) + const wrongOrg = decisions.filter( + (d) => + d.orgId && + d.orgId !== env.orgSlug && + !d.orgId.includes(env.orgSlug), + ); + expect( + wrongOrg, + 'Decisions from a different org were returned — org isolation is broken', + ).toHaveLength(0); + }); +}); diff --git a/tests/e2e/pii-decision-log.spec.ts b/tests/e2e/pii-decision-log.spec.ts new file mode 100644 index 0000000..cbf0ffd --- /dev/null +++ b/tests/e2e/pii-decision-log.spec.ts @@ -0,0 +1,71 @@ +import { test, expect, sendChatMessage, env } from './fixtures'; + +const PII_PROMPT = + 'My name is John Doe, my SSN is 123-45-6789, and my email is john@example.com. Can you help me with my account?'; + +test.describe('PII message reaches the platform decision log', () => { + test('sending a PII prompt produces a redact decision surfaced in the dashboard', async ({ authed, context }) => { + const correlationHeader = authed.waitForResponse( + (resp) => resp.url().endsWith('/api/chat') && resp.status() === 200, + ); + + await sendChatMessage(authed, PII_PROMPT); + + const chatResponse = await correlationHeader; + const correlationId = + chatResponse.headers()['x-correlation-id'] || + chatResponse.headers()['x-request-id'] || + null; + + // Without a correlation id we cannot prove the dashboard row belongs to *this* + // prompt — a stale redact row from a prior run would silently satisfy the match + // predicate. Fail loudly so the test can't pass on ambient data. + expect( + correlationId, + '/api/chat response must carry x-correlation-id (or x-request-id)', + ).toBeTruthy(); + + const redactBadge = authed.getByText(/^Redact$/i).first(); + await expect(redactBadge).toBeVisible({ timeout: 20_000 }); + + const redactedTile = authed.getByText(/^Redacted$/).locator('..'); + await expect(redactedTile).toContainText(/[1-9]\d*/); + + const dashboardPage = await context.newPage(); + await dashboardPage.goto(`${env.platformUrl}/o/${env.orgSlug}/decisions`); + + // Poll the decisions endpoint so the test tolerates ingestion lag: /api/chat + // returning does not guarantee the decision is already in the read model. + await expect + .poll( + async () => { + const resp = await dashboardPage.request.get( + `${env.platformUrl}/api/v1/decisions?orgSlug=${encodeURIComponent(env.orgSlug)}`, + ); + if (!resp.ok()) return false; + const payload = await resp.json(); + const decisions: any[] = payload.decisions ?? []; + return decisions.some((d) => { + if (d.correlationId !== correlationId) return false; + const isTransformOrRedact = + d.decision === 'transform' || d.decision === 'redact'; + const hasPiiTag = (d.tags ?? []).some((t: string) => /pii/i.test(t)); + return isTransformOrRedact || hasPiiTag; + }); + }, + { + message: `Expected a redact/transform decision with correlationId=${correlationId} to appear in the platform decision log`, + timeout: 30_000, + intervals: [1_000, 2_000, 4_000, 8_000], + }, + ) + .toBe(true); + + // Scope to a decisions-table row so we don't match legend / filter labels. + const decisionRow = dashboardPage + .getByRole('row') + .filter({ hasText: /transform|redact/i }) + .first(); + await expect(decisionRow).toBeVisible(); + }); +}); diff --git a/tests/e2e/tool-policy.spec.ts b/tests/e2e/tool-policy.spec.ts new file mode 100644 index 0000000..0f4f21b --- /dev/null +++ b/tests/e2e/tool-policy.spec.ts @@ -0,0 +1,210 @@ +/** + * QA.4 — Tool-policy violation → chat UI surfaces a blocked / redacted indicator + * + * Scope (TASKS.md §QA.4): when a tool invocation the assistant wants to make + * is denied or redacted by precheck, the governed chat UI must clearly surface + * that decision to the user. This is distinct from a content-policy block at + * the chat level (covered by blocked-message.spec.ts / governed-chat.spec.ts): + * here the chat message itself is allowed, but the downstream tool call is + * governed separately. + * + * Why these tests mock /api/chat: + * Triggering a real tool-level deny depends on the LLM choosing to call a + * tool in `deny_tools` — which is non-deterministic and fragile against + * staging. Mocking the SSE stream pins the exact tool_call / tool_result + * events the backend produces on a denial, so the test verifies the UI + * contract deterministically. The same pattern is used in budget-limit.spec.ts. + */ + +import { test, expect, sendChatMessage } from './fixtures'; + +function sseBody(events: Array<{ type: string; data: unknown }>): string { + return events.map((e) => `data: ${JSON.stringify(e)}\n\n`).join('') + 'data: {"type":"done"}\n\n'; +} + +test.describe('QA.4 · Tool-policy violation surfaces a blocked / redacted UI indicator', () => { + test.beforeEach(async ({ authed }) => { + await expect( + authed.getByRole('heading', { name: /GovernsAI Command Center Demo/i }), + ).toBeVisible(); + }); + + test('denied tool call renders a red Tool Result bubble with a deny badge and reason', async ({ + authed, + }) => { + await authed.route('**/api/chat', async (route) => { + await route.fulfill({ + status: 200, + headers: { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }, + body: sseBody([ + // The original chat message is allowed — only the tool call trips policy. + { type: 'decision', data: { decision: 'allow', reasons: [] } }, + { + type: 'tool_call', + data: { + id: 'call_deny_1', + type: 'function', + function: { + name: 'bash.exec', + arguments: JSON.stringify({ cmd: 'rm -rf /' }), + }, + }, + }, + // Tool-level precheck denies the call. + { + type: 'decision', + data: { + decision: 'deny', + reasons: ["Tool 'bash.exec' is in the deny list"], + }, + }, + { + type: 'tool_result', + data: { + tool_call_id: 'call_deny_1', + success: false, + error: "Tool call blocked: Tool 'bash.exec' is in the deny list", + decision: 'deny', + reasons: ["Tool 'bash.exec' is in the deny list"], + }, + }, + ]), + }); + }); + + await sendChatMessage(authed, 'Clean up the staging server for me.'); + + // User-visible "Tool call blocked:" copy from the tool_result error path. + await expect(authed.getByText(/Tool call blocked/i)).toBeVisible({ timeout: 15_000 }); + + // Tool Result label proves this is a role='tool' message, not the assistant bubble. + await expect(authed.getByText('Tool Result').first()).toBeVisible(); + + // The tool bubble must use the red-on-denied class from Message.tsx. + // Match by text to avoid picking up the assistant bubble (which also turns + // red when the streamed decision flips to deny). + const deniedToolBubble = authed.locator('div.bg-red-100.text-red-900', { + hasText: /Tool call blocked/i, + }); + await expect(deniedToolBubble).toBeVisible(); + + // Deny badge (DecisionBadge renders capitalised text + 🚫 icon). + const denyBadge = authed.getByText(/^deny$/i).first(); + await expect(denyBadge).toBeVisible(); + await expect(denyBadge.locator('..')).toContainText('🚫'); + + // The reason must be reachable via the hover title on the info hint. + const reasonHint = denyBadge.locator('..').getByText('info'); + await expect(reasonHint).toHaveAttribute('title', /deny list/i); + + // Blocked stats tile counter must increment (role='tool' + decision='deny' counts as deny). + const blockedTile = authed.getByText(/^Blocked$/).locator('..'); + await expect(blockedTile).toContainText(/[1-9]\d*/); + }); + + test('chat input stays interactive after a tool-policy block so the user can retry', async ({ + authed, + }) => { + await authed.route('**/api/chat', async (route) => { + await route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: sseBody([ + { type: 'decision', data: { decision: 'allow', reasons: [] } }, + { + type: 'tool_call', + data: { + id: 'call_deny_2', + type: 'function', + function: { name: 'python.exec', arguments: '{}' }, + }, + }, + { type: 'decision', data: { decision: 'deny', reasons: ['denied tool'] } }, + { + type: 'tool_result', + data: { + tool_call_id: 'call_deny_2', + success: false, + error: 'Tool call blocked: denied tool', + decision: 'deny', + reasons: ['denied tool'], + }, + }, + ]), + }); + }); + + await sendChatMessage(authed, 'Run some Python for me.'); + await expect(authed.getByText(/Tool call blocked/i)).toBeVisible({ timeout: 15_000 }); + + await expect(authed.getByPlaceholder('Type your message...')).toBeEnabled(); + await expect(authed.getByRole('button', { name: /^Send$/ })).toBeEnabled(); + }); + + test('redacted tool call shows a Redact badge and increments the Redacted tile', async ({ + authed, + }) => { + await authed.route('**/api/chat', async (route) => { + await route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: sseBody([ + { type: 'decision', data: { decision: 'allow', reasons: [] } }, + { + type: 'tool_call', + data: { + id: 'call_redact_1', + type: 'function', + function: { + name: 'db_query', + arguments: JSON.stringify({ sql: 'SELECT email FROM users' }), + }, + }, + }, + // Tool-level precheck redacts PII but still executes the tool. + { + type: 'decision', + data: { + decision: 'redact', + reasons: ['PII:email_address redacted'], + }, + }, + { + type: 'tool_result', + data: { + tool_call_id: 'call_redact_1', + success: true, + data: { rows: [{ email: '[REDACTED]' }] }, + decision: 'redact', + reasons: ['PII:email_address redacted'], + }, + }, + ]), + }); + }); + + await sendChatMessage(authed, 'Query the users table and list their emails.'); + + // Tool Result message appears with the redacted payload rather than a block error. + const toolResultLabel = authed.getByText('Tool Result').first(); + await expect(toolResultLabel).toBeVisible({ timeout: 15_000 }); + await expect(authed.getByText(/\[REDACTED\]/)).toBeVisible(); + + // The Redact badge (✂ icon, capitalised "Redact") must appear on the tool message. + const redactBadge = authed.getByText(/^redact$/i).first(); + await expect(redactBadge).toBeVisible(); + await expect(redactBadge.locator('..')).toContainText('✂'); + + // Redacted stats tile counter must be non-zero. + const redactedTile = authed.getByText(/^Redacted$/).locator('..'); + await expect(redactedTile).toContainText(/[1-9]\d*/); + + // The tool bubble should NOT be red — a redact is not a block. + const redBubble = authed.locator('div.bg-red-100.text-red-900'); + await expect(redBubble).toHaveCount(0); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index e866ab8..4dc4781 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -23,5 +23,5 @@ } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules", "playwright.config.ts", "tests/**"] }