diff --git a/app/(main)/evaluations/[id]/page.tsx b/app/(main)/evaluations/[id]/page.tsx index f4f4f27..535a88a 100644 --- a/app/(main)/evaluations/[id]/page.tsx +++ b/app/(main)/evaluations/[id]/page.tsx @@ -8,9 +8,9 @@ import { useState, useEffect, useCallback, useRef } from "react"; import { useRouter, useParams } from "next/navigation"; import { apiFetch } from "@/app/lib/apiClient"; -import { invalidateConfigCache } from "@/app/lib/utils"; import { useAuth } from "@/app/lib/context/AuthContext"; import { useApp } from "@/app/lib/context/AppContext"; +import { usePromptImprovement } from "@/app/hooks/usePromptImprovement"; import type { EvalJob, EvalJobApiResponse, @@ -71,8 +71,12 @@ export default function EvaluationReport() { const [isConfigModalOpen, setIsConfigModalOpen] = useState(false); const [exportFormat, setExportFormat] = useState<"row" | "grouped">("row"); const [isResyncing, setIsResyncing] = useState(false); - const [isImprovingPrompt, setIsImprovingPrompt] = useState(false); const [showNoTracesModal, setShowNoTracesModal] = useState(false); + const { isImprovingPrompt, handleIteratePrompt } = usePromptImprovement({ + jobId, + apiKey, + isAuthenticated, + }); const fetchJobDetails = useCallback(async () => { if (!isAuthenticated || !jobId) return; @@ -182,51 +186,6 @@ export default function EvaluationReport() { } }; - const handleIteratePrompt = async () => { - if (!isAuthenticated || !jobId) return; - setIsImprovingPrompt(true); - try { - const data = await apiFetch<{ - success: boolean; - error?: string; - data?: { - config_id?: string; - id?: string; - version?: number; - } | null; - }>(`/api/evaluations/${jobId}/improve-prompt`, apiKey, { - method: "POST", - body: JSON.stringify({}), - }); - - if (!data.success) { - toast.error(data.error || "Failed to iterate on prompt"); - return; - } - - invalidateConfigCache(); - toast.success("Prompt iteration created"); - - const newConfigId = data.data?.config_id ?? data.data?.id; - const newVersion = data.data?.version; - if (newConfigId) { - const versionParam = - typeof newVersion === "number" ? `&version=${newVersion}` : ""; - router.push( - `/configurations/prompt-editor?config=${encodeURIComponent( - newConfigId, - )}${versionParam}&from=evaluations`, - ); - } - } catch (err: unknown) { - toast.error( - `Failed to iterate on prompt: ${err instanceof Error ? err.message : "Unknown error"}`, - ); - } finally { - setIsImprovingPrompt(false); - } - }; - const handleResync = async () => { if (!isAuthenticated || !jobId) return; @@ -429,7 +388,7 @@ export default function EvaluationReport() {
-
+
{hasScore && isNewFormat ? ( <> = {}; try { - body = await request.json(); + const parsed = await request.json(); + if (parsed && typeof parsed === "object") { + userBody = parsed as Record; + } } catch { - body = {}; + userBody = {}; } + const payload = { + ...userBody, + callback_url: userBody.callback_url ?? resolveCallbackUrl(request), + }; + const { status, data } = await apiClient( request, `/api/v1/evaluations/${id}/improve-prompt`, { method: "POST", - body: JSON.stringify(body ?? {}), + body: JSON.stringify(payload), }, ); + if (status === 202 && data && typeof data === "object") { + const envelope = data as { + success?: boolean; + data?: LLMJobImmediatePublic; + }; + const jobId = envelope.data?.job_id; + if (jobId) markJobPending(jobId); + } + return NextResponse.json(data ?? { error: "Backend error" }, { status }); } catch (error: unknown) { console.error("improve-prompt proxy error:", error); diff --git a/app/api/webhooks/prompt-improvement/route.ts b/app/api/webhooks/prompt-improvement/route.ts new file mode 100644 index 0000000..8fe0ea9 --- /dev/null +++ b/app/api/webhooks/prompt-improvement/route.ts @@ -0,0 +1,66 @@ +import { NextRequest, NextResponse } from "next/server"; +import { + getJobSnapshot, + saveJobSnapshot, +} from "@/app/lib/store/promptImprovementStore"; +import { readWebhookSecret } from "@/app/lib/webhookSecret"; +import type { PromptImprovementJobPublic } from "@/app/lib/types/promptImprovement"; + +function isAuthorized(request: NextRequest): boolean { + const secretResult = readWebhookSecret(); + if (!secretResult.ok || !secretResult.secret) return false; + const provided = request.nextUrl.searchParams.get("secret_value"); + return provided === secretResult.secret; +} + +export async function POST(request: NextRequest) { + try { + if (!isAuthorized(request)) { + return NextResponse.json( + { success: false, error: "unauthorized" }, + { status: 401 }, + ); + } + const body = (await request.json()) as { + success?: boolean; + data?: PromptImprovementJobPublic; + error?: string | null; + }; + if (!body.data || !body.data.job_id) { + return NextResponse.json( + { success: false, error: "invalid_payload" }, + { status: 400 }, + ); + } + saveJobSnapshot(body.data); + return NextResponse.json({ success: true, data: { received: true } }); + } catch (error) { + console.error("prompt-improvement webhook error:", error); + return NextResponse.json( + { + success: false, + error: + error instanceof Error ? error.message : "webhook_processing_failed", + }, + { status: 500 }, + ); + } +} + +export async function GET(request: NextRequest) { + const jobId = request.nextUrl.searchParams.get("job_id"); + if (!jobId) { + return NextResponse.json( + { success: false, error: "job_id_required" }, + { status: 400 }, + ); + } + const snapshot = getJobSnapshot(jobId); + if (!snapshot) { + return NextResponse.json( + { success: false, error: "job_not_found", data: null }, + { status: 404 }, + ); + } + return NextResponse.json({ success: true, data: snapshot }); +} diff --git a/app/api/webhooks/prompt-improvement/stream/route.ts b/app/api/webhooks/prompt-improvement/stream/route.ts new file mode 100644 index 0000000..0fc4e68 --- /dev/null +++ b/app/api/webhooks/prompt-improvement/stream/route.ts @@ -0,0 +1,89 @@ +import { NextRequest } from "next/server"; +import { + getJobSnapshot, + subscribeJobSnapshot, +} from "@/app/lib/store/promptImprovementStore"; +import type { PromptImprovementJobSnapshot } from "@/app/lib/types/promptImprovement"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +export async function GET(request: NextRequest) { + const jobId = request.nextUrl.searchParams.get("job_id"); + if (!jobId) { + return new Response("job_id_required", { status: 400 }); + } + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + const send = (event: string, data: unknown) => { + try { + controller.enqueue( + encoder.encode( + `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`, + ), + ); + } catch { + // stream closed + } + }; + + const finish = () => { + clearInterval(heartbeat); + unsubscribe(); + try { + controller.close(); + } catch { + // already closed + } + }; + + const push = (snap: PromptImprovementJobSnapshot) => { + send("snapshot", snap); + if (snap.status === "SUCCESS" || snap.status === "FAILED") { + finish(); + } + }; + + // Prime with current state (or a synthetic PENDING when store hasn't + // seen it yet — keeps the client from thinking the stream is stalled). + const current = getJobSnapshot(jobId); + if (current) { + push(current); + } else { + send("snapshot", { + job_id: jobId, + status: "PENDING", + config_version: null, + error_message: null, + updated_at: new Date().toISOString(), + }); + } + + const unsubscribe = subscribeJobSnapshot(jobId, push); + + // every 20s so intermediaries (load balancers, browsers) don't + // consider the connection idle. + const heartbeat = setInterval(() => { + try { + controller.enqueue(encoder.encode(`: ping\n\n`)); + } catch { + finish(); + } + }, 20000); + + const abort = () => finish(); + request.signal.addEventListener("abort", abort); + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream; charset=utf-8", + "Cache-Control": "no-cache, no-transform", + Connection: "keep-alive", + "X-Accel-Buffering": "no", + }, + }); +} diff --git a/app/hooks/usePromptImprovement.ts b/app/hooks/usePromptImprovement.ts new file mode 100644 index 0000000..6b77bb4 --- /dev/null +++ b/app/hooks/usePromptImprovement.ts @@ -0,0 +1,213 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { useRouter } from "next/navigation"; +import { apiFetch } from "@/app/lib/apiClient"; +import { invalidateConfigCache } from "@/app/lib/utils"; +import { useToast } from "@/app/hooks/useToast"; +import type { + IterateSettle, + JobSnapshotLike, + LLMJobImmediatePublic, + UsePromptImprovementArgs, + UsePromptImprovementResult, +} from "@/app/lib/types/promptImprovement"; + +/** Map a sync validation code (from the initial 202 request) to a user-facing message. */ +function iteratePromptSyncError(code: string): string { + switch (code) { + case "evaluation_not_found": + return "Evaluation not found"; + case "evaluation_not_completed": + return "Prompt iteration is only available for completed evaluations"; + case "source_config_unavailable": + return "Source config is no longer available for this evaluation"; + case "traces_not_available": + return "Evaluation traces aren't available yet — try Resync first"; + case "invalid_callback_url": + return "Frontend callback URL is invalid. Set NEXT_PUBLIC_APP_URL to a public HTTPS host."; + case "prompt_improvement_enqueue_failed": + return "Backend couldn't queue the improvement job. Retry in a moment."; + default: + return code || "Failed to queue prompt iteration"; + } +} + +function readSnapshot(snap: JobSnapshotLike, settle: IterateSettle): boolean { + if (snap.status === "SUCCESS") { + if (!snap.config_version) { + settle({ + status: "FAILED", + message: "Backend reported SUCCESS without a config version.", + }); + return true; + } + settle({ status: "SUCCESS", version: snap.config_version }); + return true; + } + if (snap.status === "FAILED") { + settle({ status: "FAILED", message: snap.error_message ?? null }); + return true; + } + return false; +} + +/** + * Owns the async prompt-improvement flow for an evaluation: fires the initial + * POST, keeps a live channel open (SSE first, poll fallback), settles on + * SUCCESS/FAILED, and navigates to the new config version. + */ +export function usePromptImprovement({ + jobId, + apiKey, + isAuthenticated, +}: UsePromptImprovementArgs): UsePromptImprovementResult { + const router = useRouter(); + const toast = useToast(); + const [isImprovingPrompt, setIsImprovingPrompt] = useState(false); + const pollRef = useRef | null>(null); + const sourceRef = useRef(null); + + const stopWatch = useCallback(() => { + if (pollRef.current) { + clearInterval(pollRef.current); + pollRef.current = null; + } + if (sourceRef.current) { + sourceRef.current.close(); + sourceRef.current = null; + } + }, []); + + useEffect(() => () => stopWatch(), [stopWatch]); + + const pollJob = useCallback( + (pollJobId: string, settle: IterateSettle) => { + const tick = async () => { + try { + const res = await fetch( + `/api/webhooks/prompt-improvement?job_id=${encodeURIComponent(pollJobId)}`, + ); + if (res.status === 404 || !res.ok) return; + const body = (await res.json()) as { + success: boolean; + data?: JobSnapshotLike; + }; + if ( + body.data && + readSnapshot(body.data, (result) => { + stopWatch(); + settle(result); + }) + ) { + // terminal — stopWatch already ran + } + } catch (e) { + console.error("iterate poll error:", e); + } + }; + tick(); + pollRef.current = setInterval(tick, 2000); + }, + [stopWatch], + ); + + const watchJob = useCallback( + (watchJobId: string, settle: IterateSettle) => { + if (typeof window === "undefined" || typeof EventSource === "undefined") { + pollJob(watchJobId, settle); + return; + } + const source = new EventSource( + `/api/webhooks/prompt-improvement/stream?job_id=${encodeURIComponent(watchJobId)}`, + ); + sourceRef.current = source; + let fellBack = false; + const doFallback = () => { + if (fellBack) return; + fellBack = true; + source.close(); + sourceRef.current = null; + pollJob(watchJobId, settle); + }; + source.addEventListener("snapshot", (e) => { + try { + const snap = JSON.parse((e as MessageEvent).data) as JobSnapshotLike; + readSnapshot(snap, (result) => { + stopWatch(); + settle(result); + }); + } catch (err) { + console.error("iterate SSE parse error:", err); + } + }); + source.addEventListener("error", () => { + // CLOSED = server hung up. CONNECTING = browser is retrying, leave it. + if (source.readyState === EventSource.CLOSED) doFallback(); + }); + }, + [pollJob, stopWatch], + ); + + const settleIteration: IterateSettle = useCallback( + (result) => { + setIsImprovingPrompt(false); + if (result.status === "SUCCESS") { + invalidateConfigCache(); + toast.success("Prompt iteration ready"); + const cfg = result.version; + if (cfg.config_id) { + router.push( + `/configurations/prompt-editor?config=${encodeURIComponent( + cfg.config_id, + )}&version=${cfg.version}&from=evaluations`, + ); + } + return; + } + toast.error( + result.message + ? `Prompt iteration failed: ${result.message}` + : "Prompt iteration failed", + ); + }, + [router, toast], + ); + + const handleIteratePrompt = useCallback(async () => { + if (!isAuthenticated || !jobId) return; + stopWatch(); + setIsImprovingPrompt(true); + try { + const data = await apiFetch<{ + success: boolean; + error?: string; + data?: LLMJobImmediatePublic | null; + }>(`/api/evaluations/${jobId}/improve-prompt`, apiKey, { + method: "POST", + body: JSON.stringify({}), + }); + + if (!data.success || !data.data?.job_id) { + toast.error(iteratePromptSyncError(data.error || "")); + setIsImprovingPrompt(false); + return; + } + watchJob(data.data.job_id, settleIteration); + } catch (err: unknown) { + const code = err instanceof Error ? err.message : String(err); + toast.error(iteratePromptSyncError(code)); + setIsImprovingPrompt(false); + } + }, [ + isAuthenticated, + jobId, + apiKey, + stopWatch, + toast, + watchJob, + settleIteration, + ]); + + return { isImprovingPrompt, handleIteratePrompt }; +} diff --git a/app/lib/store/promptImprovementStore.ts b/app/lib/store/promptImprovementStore.ts new file mode 100644 index 0000000..9909fbd --- /dev/null +++ b/app/lib/store/promptImprovementStore.ts @@ -0,0 +1,105 @@ +/** + * Server-side in-memory store for prompt-improvement job status. The backend + * pushes results to `/api/webhooks/prompt-improvement`; this store keeps the + * latest snapshot per job_id so the browser can poll a Kaapi-frontend BFF + * endpoint instead of receiving webhooks directly. + */ + +import type { + PromptImprovementJobPublic, + PromptImprovementJobSnapshot, + PromptImprovementStoreShape, + SnapshotListener, +} from "@/app/lib/types/promptImprovement"; + +const globalKey = Symbol.for("kaapi.promptImprovementStore"); + +type WithStore = { [globalKey]?: PromptImprovementStoreShape }; +const g = globalThis as unknown as WithStore; + +if (!g[globalKey]) { + g[globalKey] = { jobs: new Map(), listeners: new Map() }; +} + +const store = g[globalKey]!; + +function emit(snapshot: PromptImprovementJobSnapshot): void { + const set = store.listeners.get(snapshot.job_id); + if (!set) return; + for (const cb of set) { + try { + cb(snapshot); + } catch (e) { + console.error("prompt-improvement listener error:", e); + } + } +} + +export function subscribeJobSnapshot( + jobId: string, + cb: SnapshotListener, +): () => void { + let set = store.listeners.get(jobId); + if (!set) { + set = new Set(); + store.listeners.set(jobId, set); + } + set.add(cb); + return () => { + const s = store.listeners.get(jobId); + if (!s) return; + s.delete(cb); + if (s.size === 0) store.listeners.delete(jobId); + }; +} + +const MAX_JOBS = 500; +const TTL_MS = 30 * 60 * 1000; // drop entries older than 30 min + +function prune(): void { + const now = Date.now(); + for (const [jobId, snap] of store.jobs.entries()) { + if (now - new Date(snap.updated_at).getTime() > TTL_MS) { + store.jobs.delete(jobId); + } + } + if (store.jobs.size > MAX_JOBS) { + const excess = store.jobs.size - MAX_JOBS; + let dropped = 0; + for (const jobId of store.jobs.keys()) { + store.jobs.delete(jobId); + dropped += 1; + if (dropped >= excess) break; + } + } +} + +export function saveJobSnapshot(job: PromptImprovementJobPublic): void { + const snapshot: PromptImprovementJobSnapshot = { + ...job, + updated_at: new Date().toISOString(), + }; + store.jobs.set(job.job_id, snapshot); + prune(); + emit(snapshot); +} + +export function markJobPending(jobId: string): void { + if (store.jobs.has(jobId)) return; + const snapshot: PromptImprovementJobSnapshot = { + job_id: jobId, + status: "PENDING", + config_version: null, + error_message: null, + updated_at: new Date().toISOString(), + }; + store.jobs.set(jobId, snapshot); + prune(); + emit(snapshot); +} + +export function getJobSnapshot( + jobId: string, +): PromptImprovementJobSnapshot | null { + return store.jobs.get(jobId) ?? null; +} diff --git a/app/lib/types/promptImprovement.ts b/app/lib/types/promptImprovement.ts new file mode 100644 index 0000000..2b9361c --- /dev/null +++ b/app/lib/types/promptImprovement.ts @@ -0,0 +1,62 @@ +export type PromptImprovementStatus = + | "PENDING" + | "PROCESSING" + | "SUCCESS" + | "FAILED"; + +export interface LLMJobImmediatePublic { + job_id: string; + status: PromptImprovementStatus; + message?: string | null; + job_inserted_at?: string; + job_updated_at?: string; +} + +import type { ConfigVersionPublic } from "@/app/lib/types/configs"; + +export type PromptImprovementConfigVersion = ConfigVersionPublic; + +export interface PromptImprovementJobPublic { + job_id: string; + status: PromptImprovementStatus; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; +} + +export interface PromptImprovementJobSnapshot { + job_id: string; + status: PromptImprovementStatus; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; + updated_at: string; +} + +export interface UsePromptImprovementArgs { + jobId: string; + apiKey: string; + isAuthenticated: boolean; +} + +export interface UsePromptImprovementResult { + isImprovingPrompt: boolean; + handleIteratePrompt: () => Promise; +} + +export type IterateSettle = ( + result: + | { status: "SUCCESS"; version: PromptImprovementConfigVersion } + | { status: "FAILED"; message: string | null }, +) => void; + +export interface JobSnapshotLike { + status: string; + config_version: PromptImprovementConfigVersion | null; + error_message: string | null; +} + +export type SnapshotListener = (snapshot: PromptImprovementJobSnapshot) => void; + +export interface PromptImprovementStoreShape { + jobs: Map; + listeners: Map>; +} diff --git a/app/lib/types/webhookSecret.ts b/app/lib/types/webhookSecret.ts new file mode 100644 index 0000000..957ef57 --- /dev/null +++ b/app/lib/types/webhookSecret.ts @@ -0,0 +1,10 @@ +export type WebhookSecretFailureReason = + | "prompt_improvement_webhook_secret_missing" + | "prompt_improvement_webhook_secret_placeholder" + | "prompt_improvement_webhook_secret_invalid_format"; + +export interface WebhookSecretResult { + ok: boolean; + secret?: string; + reason?: WebhookSecretFailureReason; +} diff --git a/app/lib/webhookSecret.ts b/app/lib/webhookSecret.ts new file mode 100644 index 0000000..d75add1 --- /dev/null +++ b/app/lib/webhookSecret.ts @@ -0,0 +1,42 @@ +const HEX_SECRET_REGEX = /^[a-f0-9]{32,}$/i; + +const PLACEHOLDER_VALUES = new Set([ + "changeme", + "change_me", + "secret", + "test", + "password", + "todo", +]); + +import type { WebhookSecretResult } from "@/app/lib/types/webhookSecret"; + +export type { + WebhookSecretFailureReason, + WebhookSecretResult, +} from "@/app/lib/types/webhookSecret"; + +/** + * Read + validate the secret. Returns a discriminated result so callers can + * surface a specific error code instead of a generic 500. + */ +export function readWebhookSecret(): WebhookSecretResult { + const raw = process.env.PROMPT_IMPROVEMENT_WEBHOOK_SECRET; + if (!raw) { + return { ok: false, reason: "prompt_improvement_webhook_secret_missing" }; + } + const trimmed = raw.trim(); + if (PLACEHOLDER_VALUES.has(trimmed.toLowerCase())) { + return { + ok: false, + reason: "prompt_improvement_webhook_secret_placeholder", + }; + } + if (!HEX_SECRET_REGEX.test(trimmed)) { + return { + ok: false, + reason: "prompt_improvement_webhook_secret_invalid_format", + }; + } + return { ok: true, secret: trimmed }; +}