-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathbatch-eval.html
More file actions
17 lines (17 loc) · 74.5 KB
/
Copy pathbatch-eval.html
File metadata and controls
17 lines (17 loc) · 74.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
<!doctype html>
<html lang="en" dir="ltr" class="plugin-pages plugin-id-default" data-has-hydrated="false">
<head>
<meta charset="UTF-8">
<meta name="generator" content="Docusaurus v3.10.2">
<title data-rh="true">Batch eval | neokapi</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" property="og:image" content="https://neokapi.github.io/img/og-card.png"><meta data-rh="true" name="twitter:image" content="https://neokapi.github.io/img/og-card.png"><meta data-rh="true" property="og:url" content="https://neokapi.github.io/batch-eval"><meta data-rh="true" property="og:locale" content="en"><meta data-rh="true" property="og:locale:alternate" content="nb"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docusaurus_tag" content="default"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docsearch:docusaurus_tag" content="default"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" name="twitter:site" content="@neokapi"><meta data-rh="true" property="og:title" content="Batch eval | neokapi"><meta data-rh="true" name="description" content="What batching costs: cost per thousand words, throughput, and structural integrity against the number of blocks packed into one LLM call, measured per model and tracked over time."><meta data-rh="true" property="og:description" content="What batching costs: cost per thousand words, throughput, and structural integrity against the number of blocks packed into one LLM call, measured per model and tracked over time."><link data-rh="true" rel="icon" href="/img/favicon.png"><link data-rh="true" rel="canonical" href="https://neokapi.github.io/batch-eval"><link data-rh="true" rel="alternate" href="https://neokapi.github.io/batch-eval" hreflang="en"><link data-rh="true" rel="alternate" href="https://neokapi.github.io/nb/batch-eval" hreflang="nb"><link data-rh="true" rel="alternate" href="https://neokapi.github.io/batch-eval" hreflang="x-default"><link rel="stylesheet" href="/assets/css/styles.a74d5c23.css">
<script src="/assets/js/runtime~main.b9b4a951.js" defer="defer"></script>
<script src="/assets/js/main.666d8dbf.js" defer="defer"></script>
</head>
<body>
<svg style="display: none;"><defs>
<symbol id="theme-svg-external-link" viewBox="0 0 24 24"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"/></symbol>
</defs></svg>
<script>!function(){var t=function(){try{return new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}}()||function(){try{return window.localStorage.getItem("theme")}catch(t){}}();document.documentElement.setAttribute("data-theme",t||"light"),document.documentElement.setAttribute("data-theme-choice",t||"light")}(),function(){try{const c=new URLSearchParams(window.location.search).entries();for(var[t,e]of c)if(t.startsWith("docusaurus-data-")){var a=t.replace("docusaurus-data-","data-");document.documentElement.setAttribute(a,e)}}catch(t){}}()</script><div id="__docusaurus"><link rel="preload" as="image" href="/img/logo.png"><div role="region" aria-label="Skip to main content"><a class="skipToContent_B9G6" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><nav aria-label="Main" class="theme-layout-navbar navbar navbar--fixed-top"><div class="navbar__inner"><div class="theme-layout-navbar-left navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/"><div class="navbar__logo"><img src="/img/logo.png" alt="neokapi" class="themedComponent_XdmY themedComponent--light_Mg5p"><img src="/img/logo.png" alt="neokapi" class="themedComponent_XdmY themedComponent--dark_dkVE"></div><b class="navbar__title text--truncate">neokapi</b></a><a class="navbar__item navbar__link" href="/kapi/overview">Kapi</a><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Labs</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/labs">Labs overview</a></li><li><a class="dropdown__link" href="/lab">Content Model Workspace</a></li><li><a class="dropdown__link" href="/lab/segmentation">Segmentation</a></li><li><a class="dropdown__link" href="/lab/convert">File Conversion</a></li><li><a class="dropdown__link" href="/lab/structure">Structure & Layout</a></li><li><a class="dropdown__link" href="/lab/vision">Vision</a></li><li><a class="dropdown__link" href="/lab/media">Audio & Video</a></li><li><a class="dropdown__link" href="/playground-cli">CLI Playground</a></li><li><a class="dropdown__link" href="/kbf-lab">KBF Anatomy</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Reference</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/reference">Reference Overview</a></li><li><a class="dropdown__link" href="/commands">Kapi CLI Commands</a></li><li><a class="dropdown__link" href="/formats">Formats</a></li><li><a class="dropdown__link" href="/tools">Tools</a></li><li><a class="dropdown__link" href="/models">AI Models</a></li><li><a class="dropdown__link" href="/reference/project-file">Project file</a></li><li><a class="dropdown__link" href="/reference/serialization/overview">kapi serialization formats</a></li><li><a class="dropdown__link" href="/reference/mcp">MCP Server</a></li><li><a class="dropdown__link" href="/reference/cli-contract">Scripting & JSON contract</a></li><li><a class="dropdown__link" href="/reference/engine-service">Engine service (gRPC)</a></li><li><a class="dropdown__link" href="/reference/telemetry">Telemetry</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Toolbox</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/toolbox/overview">CLI tools</a></li><li><a class="dropdown__link" href="/react/introduction">Kapi React</a></li></ul></div><a class="navbar__item navbar__link" href="/framework/architecture">Framework</a></div><div class="theme-layout-navbar-right navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link navbar-locale-icon"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_PEKS"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/batch-eval" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li><li><a href="/nb/batch-eval" target="_self" rel="noopener noreferrer" class="dropdown__link" lang="nb">Norsk (bokmål)</a></li></ul></div><a href="https://github.com/neokapi/neokapi" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_F2l_ colorModeToggle_rTAI"><button class="clean-btn toggleButton_G7K1 toggleButtonDisabled_Lph7" type="button" disabled="" title="system mode" aria-label="Switch between dark and light mode (currently system mode)"><svg viewBox="0 0 24 24" width="24" height="24" aria-hidden="true" class="toggleIcon_dDEW lightToggleIcon_lf2M"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" aria-hidden="true" class="toggleIcon_dDEW darkToggleIcon_m9fn"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" aria-hidden="true" class="toggleIcon_dDEW systemToggleIcon_dqiZ"><path fill="currentColor" d="m12 21c4.971 0 9-4.029 9-9s-4.029-9-9-9-9 4.029-9 9 4.029 9 9 9zm4.95-13.95c1.313 1.313 2.05 3.093 2.05 4.95s-0.738 3.637-2.05 4.95c-1.313 1.313-3.093 2.05-4.95 2.05v-14c1.857 0 3.637 0.737 4.95 2.05z"></path></svg></button></div><div class="navbarSearchContainer_XvcB"></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="theme-layout-main main-wrapper mainWrapper_Tjsm"><main style="max-width:940px;margin:0 auto;padding:2.5rem 1.25rem 4rem"><h1>Batch eval</h1><p style="font-size:1.05rem;color:var(--ifm-color-emphasis-700)">kapi packs several blocks into a single LLM call, because one call per string is slow and expensive. How many is safe? kapi’s ceiling began as an informed guess; this page is the measurement that replaced it, re-run as models change so it does not quietly go stale. The sibling <a href="/context-eval">context eval</a> measures the other axis: whether a model follows the context kapi injects.</p><h2>How it is measured</h2><p>A fixed corpus is translated at each batch size, and the result is scored for<!-- --> <strong>structural integrity</strong> — not wording. The failure that matters when batching is not clumsier phrasing; it is a segment that comes back<!-- --> <strong>dropped, merged, renumbered, or stripped of a placeholder or tag</strong>. In a translation pipeline that is a correctness failure, because a translation missing its<!-- --> <code>{0}</code> cannot be written back into the source file at all. Scoring structure needs no reference translation, so the same measurement works for any model in any language pair. Cost and throughput are recorded from the same runs.</p><p>The corpus is 600 blocks of the things batching is supposed to break — ambiguous UI strings, placeholders, inline markup, and long prose — with every string distinct, so a model cannot pass by copying one segment’s answer into the next. Each size is swept more than once, because a single run of a stochastic model is an anecdote.</p><h2>What varies with batch size</h2><p style="color:var(--ifm-color-emphasis-700);font-size:0.92rem">Measured <!-- -->2026-07-15<!-- --> · target <!-- -->de<!-- --> · <!-- -->600 blocks (181 ui, 118 ui-ambiguous, 120 placeholder, 61 inline-tag, 120 prose)<!-- --> · corpus<!-- --> <code>d46f4b2feac7</code></p><p>Quality does not move with batch size — every model stays structurally intact at every N — so the chart that would show it is a flat line at the ceiling. What moves is what batching <em>costs</em>, so that is the chart.</p><svg viewBox="0 0 720 300" style="width:100%;height:auto;overflow:visible" role="img" aria-label="Cost per thousand source words against blocks per call, one line per model"><g><line x1="62" x2="704" y1="256" y2="256" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="260" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">$0.0054</text></g><g><line x1="62" x2="704" y1="136" y2="136" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="140" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">$0.030</text></g><g><line x1="62" x2="704" y1="16" y2="16" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="20" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">$0.171</text></g><g><line x1="371.2079085459252" x2="371.2079085459252" y1="16" y2="256" stroke="var(--ifm-color-emphasis-500)" stroke-width="1.5" stroke-dasharray="5 4"></line><text x="377.2079085459252" y="28" font-size="11" fill="var(--ifm-color-emphasis-700)">kapi ships N≤<!-- -->64</text></g><text x="62" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">8</text><text x="165.0693028486417" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">16</text><text x="268.1386056972834" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">32</text><text x="371.2079085459252" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">64</text><text x="474.2772113945669" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">128</text><text x="577.3465142432086" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">256</text><text x="704" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">600</text><text x="383" y="294" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">blocks per call (N)</text><text x="-136" y="13" transform="rotate(-90)" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">USD / 1,000 source words</text><g><path d="M165.0693028486417,67.46816454515772 L268.1386056972834,75.01773838990155 L371.2079085459252,78.18218253844927 L474.2772113945669,80.0919651491038 L577.3465142432086,80.58844574552779 L704,16" fill="none" stroke="#4c78a8" stroke-width="2"></path><circle cx="165.0693028486417" cy="67.46816454515772" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="268.1386056972834" cy="75.01773838990155" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="371.2079085459252" cy="78.18218253844927" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="474.2772113945669" cy="80.0919651491038" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="577.3465142432086" cy="80.58844574552779" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="704" cy="16" r="3.5" fill="#4c78a8"><title></title></circle></g><g><path d="M62,250.77179672283137 L165.0693028486417,252.91295926218478 L268.1386056972834,252.86987682300236 L371.2079085459252,253.57951328364686 L474.2772113945669,255.2806288471552 L577.3465142432086,256 L704,202.9642828094871" fill="none" stroke="#f58518" stroke-width="2"></path><circle cx="62" cy="250.77179672283137" r="3.5" fill="#f58518"><title></title></circle><circle cx="165.0693028486417" cy="252.91295926218478" r="3.5" fill="#f58518"><title></title></circle><circle cx="268.1386056972834" cy="252.86987682300236" r="3.5" fill="#f58518"><title></title></circle><circle cx="371.2079085459252" cy="253.57951328364686" r="3.5" fill="#f58518"><title></title></circle><circle cx="474.2772113945669" cy="255.2806288471552" r="3.5" fill="#f58518"><title></title></circle><circle cx="577.3465142432086" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="704" cy="202.9642828094871" r="3.5" fill="#f58518"><title></title></circle></g><g><path d="M62,131.9091819861855 L165.0693028486417,133.05712443056493 L268.1386056972834,128.22770212845234 L371.2079085459252,128.2544212458577 L474.2772113945669,128.2049836054997 L577.3465142432086,128.28145793484003 L704,76.97997625853998" fill="none" stroke="#54a24b" stroke-width="2"></path><circle cx="62" cy="131.9091819861855" r="3.5" fill="#54a24b"><title></title></circle><circle cx="165.0693028486417" cy="133.05712443056493" r="3.5" fill="#54a24b"><title></title></circle><circle cx="268.1386056972834" cy="128.22770212845234" r="3.5" fill="#54a24b"><title></title></circle><circle cx="371.2079085459252" cy="128.2544212458577" r="3.5" fill="#54a24b"><title></title></circle><circle cx="474.2772113945669" cy="128.2049836054997" r="3.5" fill="#54a24b"><title></title></circle><circle cx="577.3465142432086" cy="128.28145793484003" r="3.5" fill="#54a24b"><title></title></circle><circle cx="704" cy="76.97997625853998" r="3.5" fill="#54a24b"><title></title></circle></g></svg><div style="display:flex;gap:16px;flex-wrap:wrap;margin:8px 0 24px"><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#4c78a8;display:inline-block"></span>eu.anthropic.claude-sonnet-4-6<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->bedrock<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#f58518;display:inline-block"></span>gemini-3.1-flash-lite<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#54a24b;display:inline-block"></span>gemini-3.5-flash<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span></div><p style="font-size:0.9rem;color:var(--ifm-color-emphasis-700)">Log scale, because the models are an order of magnitude apart in price and a linear axis would flatten the cheap ones into the baseline. Two shapes to read. The<!-- --> <strong>fall on the left</strong> is the per-call overhead — system prompt plus JSON schema, paid once per call — being amortised over more blocks; how steep it is depends on how big that overhead is, which is a fact about the provider, not about batching. On Bedrock that fall is steep — 985 tokens of overhead per call is a lot to repeat. On Gemini it is barely a fall at all: ~106 tokens of overhead leaves almost nothing to amortise, and the curve is flat until it turns up. The<!-- --> <strong>rise on the right</strong> is the same on every provider, because it is not the provider’s doing: past the output ceiling the reply cannot be emitted in one go, kapi splits the batch and redoes the work, and you pay for the same words twice. Cheapest is in between — and the minimum is broad and shallow, which is the useful part: the exact N barely matters, so long as you are not at either extreme.</p><h3>Structural integrity, on an axis where a regression would show</h3><p>Breaks per 1,000 blocks — dropped, merged, renumbered segments, and mangled placeholders or tags. Zero is the expected reading, and the reason to keep drawing it is that a model can regress behind a stable alias without anyone being told. This is the guard, not the headline.</p><svg viewBox="0 0 720 300" style="width:100%;height:auto;overflow:visible" role="img" aria-label="Structural breaks per thousand blocks against blocks per call, one line per model"><g><line x1="62" x2="704" y1="256" y2="256" stroke="var(--ifm-color-emphasis-300)" stroke-width="1"></line><text x="54" y="260" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">0.0</text></g><g><line x1="62" x2="704" y1="136" y2="136" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="140" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">4.6</text></g><g><line x1="62" x2="704" y1="16" y2="16" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="20" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">9.2</text></g><g><line x1="371.2079085459252" x2="371.2079085459252" y1="16" y2="256" stroke="var(--ifm-color-emphasis-500)" stroke-width="1.5" stroke-dasharray="5 4"></line><text x="377.2079085459252" y="28" font-size="11" fill="var(--ifm-color-emphasis-700)">kapi ships N≤<!-- -->64</text></g><text x="62" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">8</text><text x="165.0693028486417" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">16</text><text x="268.1386056972834" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">32</text><text x="371.2079085459252" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">64</text><text x="474.2772113945669" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">128</text><text x="577.3465142432086" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">256</text><text x="704" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">600</text><text x="383" y="294" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">blocks per call (N)</text><text x="-136" y="13" transform="rotate(-90)" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">structural breaks / 1,000 blocks</text><g><path d="M165.0693028486417,256 L268.1386056972834,256 L371.2079085459252,256 L474.2772113945669,256 L577.3465142432086,256 L704,256" fill="none" stroke="#4c78a8" stroke-width="2"></path><circle cx="165.0693028486417" cy="256" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="268.1386056972834" cy="256" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="371.2079085459252" cy="256" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="474.2772113945669" cy="256" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="577.3465142432086" cy="256" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="704" cy="256" r="3.5" fill="#4c78a8"><title></title></circle></g><g><path d="M62,256 L165.0693028486417,256 L268.1386056972834,256 L371.2079085459252,256 L474.2772113945669,256 L577.3465142432086,256 L704,256" fill="none" stroke="#f58518" stroke-width="2"></path><circle cx="62" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="165.0693028486417" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="268.1386056972834" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="371.2079085459252" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="474.2772113945669" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="577.3465142432086" cy="256" r="3.5" fill="#f58518"><title></title></circle><circle cx="704" cy="256" r="3.5" fill="#f58518"><title></title></circle></g><g><path d="M62,59.636363636363626 L165.0693028486417,212.36363636363635 L268.1386056972834,256 L371.2079085459252,256 L474.2772113945669,256 L577.3465142432086,16 L704,256" fill="none" stroke="#54a24b" stroke-width="2"></path><circle cx="62" cy="59.636363636363626" r="3.5" fill="#54a24b"><title></title></circle><circle cx="165.0693028486417" cy="212.36363636363635" r="3.5" fill="#54a24b"><title></title></circle><circle cx="268.1386056972834" cy="256" r="3.5" fill="#54a24b"><title></title></circle><circle cx="371.2079085459252" cy="256" r="3.5" fill="#54a24b"><title></title></circle><circle cx="474.2772113945669" cy="256" r="3.5" fill="#54a24b"><title></title></circle><circle cx="577.3465142432086" cy="16" r="3.5" fill="#54a24b"><title></title></circle><circle cx="704" cy="256" r="3.5" fill="#54a24b"><title></title></circle></g></svg><div style="display:flex;gap:16px;flex-wrap:wrap;margin:8px 0 24px"><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#4c78a8;display:inline-block"></span>eu.anthropic.claude-sonnet-4-6<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->bedrock<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#f58518;display:inline-block"></span>gemini-3.1-flash-lite<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#54a24b;display:inline-block"></span>gemini-3.5-flash<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span></div><h3>Throughput</h3><p>Source words per second. Comparable <em>within</em> a model across N, not between vendors: the sweeps ran at different concurrencies, and the drop at the right-hand end is the same truncation-and-retry that lifts the cost curve.</p><svg viewBox="0 0 720 300" style="width:100%;height:auto;overflow:visible" role="img" aria-label="Throughput in source words per second against blocks per call, one line per model"><g><line x1="62" x2="704" y1="256" y2="256" stroke="var(--ifm-color-emphasis-300)" stroke-width="1"></line><text x="54" y="260" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">0</text></g><g><line x1="62" x2="704" y1="136" y2="136" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="140" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">227</text></g><g><line x1="62" x2="704" y1="16" y2="16" stroke="var(--ifm-color-emphasis-300)" stroke-width="1" stroke-dasharray="3 3"></line><text x="54" y="20" text-anchor="end" font-size="11" fill="var(--ifm-color-emphasis-600)">454</text></g><g><line x1="371.2079085459252" x2="371.2079085459252" y1="16" y2="256" stroke="var(--ifm-color-emphasis-500)" stroke-width="1.5" stroke-dasharray="5 4"></line><text x="377.2079085459252" y="28" font-size="11" fill="var(--ifm-color-emphasis-700)">kapi ships N≤<!-- -->64</text></g><text x="62" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">8</text><text x="165.0693028486417" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">16</text><text x="268.1386056972834" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">32</text><text x="371.2079085459252" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">64</text><text x="474.2772113945669" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">128</text><text x="577.3465142432086" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">256</text><text x="704" y="274" text-anchor="middle" font-size="11" fill="var(--ifm-color-emphasis-600)">600</text><text x="383" y="294" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">blocks per call (N)</text><text x="-136" y="13" transform="rotate(-90)" text-anchor="middle" font-size="12" fill="var(--ifm-color-emphasis-700)">source words / second</text><g><path d="M165.0693028486417,232.19971620415615 L268.1386056972834,227.26157467782497 L371.2079085459252,225.9169593654333 L474.2772113945669,224.41066570385448 L577.3465142432086,222.64768837528948 L704,248.08696543679142" fill="none" stroke="#4c78a8" stroke-width="2"></path><circle cx="165.0693028486417" cy="232.19971620415615" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="268.1386056972834" cy="227.26157467782497" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="371.2079085459252" cy="225.9169593654333" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="474.2772113945669" cy="224.41066570385448" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="577.3465142432086" cy="222.64768837528948" r="3.5" fill="#4c78a8"><title></title></circle><circle cx="704" cy="248.08696543679142" r="3.5" fill="#4c78a8"><title></title></circle></g><g><path d="M62,87.5403019964935 L165.0693028486417,42.25713589039482 L268.1386056972834,16 L371.2079085459252,36.97499075319199 L474.2772113945669,33.611492255748296 L577.3465142432086,86.81006855206013 L704,216.83121312680058" fill="none" stroke="#f58518" stroke-width="2"></path><circle cx="62" cy="87.5403019964935" r="3.5" fill="#f58518"><title></title></circle><circle cx="165.0693028486417" cy="42.25713589039482" r="3.5" fill="#f58518"><title></title></circle><circle cx="268.1386056972834" cy="16" r="3.5" fill="#f58518"><title></title></circle><circle cx="371.2079085459252" cy="36.97499075319199" r="3.5" fill="#f58518"><title></title></circle><circle cx="474.2772113945669" cy="33.611492255748296" r="3.5" fill="#f58518"><title></title></circle><circle cx="577.3465142432086" cy="86.81006855206013" r="3.5" fill="#f58518"><title></title></circle><circle cx="704" cy="216.83121312680058" r="3.5" fill="#f58518"><title></title></circle></g><g><path d="M62,198.5079505891954 L165.0693028486417,170.273524147205 L268.1386056972834,147.73696789681426 L371.2079085459252,134.23090974319405 L474.2772113945669,125.44784934671651 L577.3465142432086,168.80933074863339 L704,237.56289548461618" fill="none" stroke="#54a24b" stroke-width="2"></path><circle cx="62" cy="198.5079505891954" r="3.5" fill="#54a24b"><title></title></circle><circle cx="165.0693028486417" cy="170.273524147205" r="3.5" fill="#54a24b"><title></title></circle><circle cx="268.1386056972834" cy="147.73696789681426" r="3.5" fill="#54a24b"><title></title></circle><circle cx="371.2079085459252" cy="134.23090974319405" r="3.5" fill="#54a24b"><title></title></circle><circle cx="474.2772113945669" cy="125.44784934671651" r="3.5" fill="#54a24b"><title></title></circle><circle cx="577.3465142432086" cy="168.80933074863339" r="3.5" fill="#54a24b"><title></title></circle><circle cx="704" cy="237.56289548461618" r="3.5" fill="#54a24b"><title></title></circle></g></svg><div style="display:flex;gap:16px;flex-wrap:wrap;margin:8px 0 24px"><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#4c78a8;display:inline-block"></span>eu.anthropic.claude-sonnet-4-6<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->bedrock<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#f58518;display:inline-block"></span>gemini-3.1-flash-lite<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span><span style="display:flex;align-items:center;gap:6px;font-size:0.88rem"><span style="width:14px;height:3px;background:#54a24b;display:inline-block"></span>gemini-3.5-flash<span style="color:var(--ifm-color-emphasis-600)">(<!-- -->gemini<!-- -->)</span></span></div><h2>What the sweep found</h2><p><strong>There is no quality cliff.</strong> Batches of <!-- -->600<!-- --> segments — most of a document, answered in a single call — came back structurally intact.<!-- --> <!-- -->The worst point anywhere in the sweep is<!-- --> <strong>99.1<!-- -->% intact</strong> (<!-- -->gemini-3.5-flash<!-- --> at N=<!-- -->256<!-- -->).<!-- --> <!-- -->Batching is often expected to degrade as N grows, but that did not happen here, and the reason is structural: translating one segment does not depend on having translated the others correctly. The segments are independent, so the failures that do occur are sporadic rather than progressive.</p><p><strong>If anything, the small end is worse.</strong> <!-- -->These are stochastic models: a sweep this size drops a segment somewhere, so the claim worth making is about the trend, not about perfection. Counted as breaks per 1,000 blocks — a rate, so a batch size is not flattered by having fewer blocks behind it — small batches (N <!-- -->≤<!-- --> <!-- -->16<!-- -->) broke<!-- --> <strong>2.0</strong> and large ones (N <!-- -->≥<!-- --> <!-- -->128<!-- -->) broke <strong>1.2</strong>.<!-- --> <!-- -->More calls means more chances to fumble a placeholder, and less surrounding context in which to recognise one — the opposite of the effect the ceiling was set to guard against.</p><p><strong>What actually binds is output tokens, and it bills rather than breaks.</strong> <!-- -->A blocking call may emit at most <!-- -->16,000<!-- --> tokens (<code>NonStreamingMaxOutputTokens</code>: asking a synchronous request for more means holding an HTTP connection open for many minutes). A batch whose reply would exceed that comes back truncated — under a JSON schema, a fragment is invalid JSON — so kapi halves the batch and translates each half. Nothing is corrupted, which is exactly why the integrity line stays at 100%. You simply pay for the work twice.<!-- --> <!-- -->At the top of this sweep that is not a rounding error:<!-- --> <span><strong>eu.anthropic.claude-sonnet-4-6</strong> at N=<!-- -->600<!-- --> cost<!-- --> <strong>2.5<!-- -->×</strong> the tokens and ran<!-- --> <strong>4.2<!-- -->×</strong> slower than at N=<!-- -->256</span><span>; <strong>gemini-3.1-flash-lite</strong> at N=<!-- -->600<!-- --> cost<!-- --> <strong>2.1<!-- -->×</strong> the tokens and ran<!-- --> <strong>4.3<!-- -->×</strong> slower than at N=<!-- -->256</span><span>; <strong>gemini-3.5-flash</strong> at N=<!-- -->600<!-- --> cost<!-- --> <strong>2.2<!-- -->×</strong> the tokens and ran<!-- --> <strong>4.6<!-- -->×</strong> slower than at N=<!-- -->16</span>.</p><p>So the limit that matters is a <strong>token budget</strong>, not a block count, and that is what kapi packs against. The block cap is a backstop, set at N<!-- -->≤<!-- -->64<!-- --> — inside the measured-clean range, and few enough that the output budget binds first on anything longer than a short UI string.</p><h2>What it costs, and what you give up</h2><p>Every call carries a <strong>fixed overhead</strong> — the system prompt and the JSON schema that constrains the reply — paid once per call however many blocks ride along. Batching amortises it. How much that is worth depends on how big the overhead is, which varies by provider by an order of magnitude, so the honest answer to “does batching save money?” is <em>it depends on who you call</em> — read the Δ column below rather than trusting a rule of thumb.</p><p>What batching buys on <em>every</em> provider is <strong>throughput</strong>: you wait on a handful of round trips instead of hundreds. And on rate-limited routes it buys the run itself — see <code>eu.anthropic.claude-sonnet-4-6</code> below, where the smallest batch sizes issue so many calls that the account’s Bedrock quota refuses them outright. There the scarce resource is <em>requests</em>, not tokens.</p><p>The unit is <strong>USD per 1,000 source words</strong>, because content budgets are denominated in words and a token count is a fact about a vendor’s tokenizer rather than about the work. Nothing here is specific to translation: any AI pass over content — a review, a check, terminology, entity extraction — has the same shape and the same economics. Translation is simply the pass this harness drives, because it is the one that batches.</p><p style="font-size:0.92rem;color:var(--ifm-color-emphasis-700)">The smallest batch size measured, against kapi’s shipped ceiling of<!-- --> <!-- -->64<!-- -->:</p><table style="width:100%;border-collapse:collapse;font-size:0.9rem"><thead><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums" rowspan="2">Model</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums" colspan="3">$ / 1,000 words</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums" colspan="3">Words / second</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums" rowspan="2">Intact at N=<!-- -->64</th></tr><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">smallest N</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">N=<!-- -->64</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Δ</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">smallest N</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">N=<!-- -->64</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Δ</th></tr></thead><tbody><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">eu.anthropic.claude-sonnet-4-6</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.081<span style="margin-left:4px;font-size:0.72rem;color:var(--ifm-color-emphasis-600)">(N=<!-- -->16<!-- -->)</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.070</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">−14%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">45</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">57</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">+26%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">gemini-3.1-flash-lite</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0058<span style="margin-left:4px;font-size:0.72rem;color:var(--ifm-color-emphasis-600)">(N=<!-- -->8<!-- -->)</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0056</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">−4%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">319</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">414</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">+30%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">gemini-3.5-flash</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.032<span style="margin-left:4px;font-size:0.72rem;color:var(--ifm-color-emphasis-600)">(N=<!-- -->8<!-- -->)</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.034</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">+5%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">109</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">230</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:var(--ifm-color-emphasis-700)">+112%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr></tbody></table><p style="font-size:0.85rem;color:var(--ifm-color-emphasis-700)">Read the two Δ columns together against the last one: whatever batching does to your bill, it buys throughput and costs no structural integrity. That is the trade it actually offers.</p><p style="font-size:0.85rem;color:var(--ifm-color-emphasis-700)">Three honest caveats about these numbers.<!-- --> <strong>Bedrock is billed through AWS Marketplace</strong>, and the <code>eu.</code> <!-- -->cross-region inference profile is a <em>regional</em> profile: AWS charges exactly 10% more for it than for the otherwise identical <code>global.</code> profile ($3.30/$16.50 per million tokens against $3.00/$15.00). The rates here are the ones actually paid. <strong>The claude-code models show no cost</strong>, because that sweep reached them through the Claude subscription, which is not billed per token — and whose token counts do not describe an API call either: the CLI wraps every request in its own agent system prompt and reports it as cache creation, so it recorded 240 input tokens across sixty calls. A dollar figure built on that would be fiction in whichever direction happened to flatter the conclusion, so the column is blank. Cost for those models needs a sweep against the metered Anthropic API.<!-- --> <strong>Speed is not comparable across providers</strong>: the sweeps ran at different concurrencies<!-- --> <!-- -->(<!-- -->eu.anthropic.claude-sonnet-4-6: 2, gemini-3.1-flash-lite: 4, gemini-3.5-flash: 4<!-- -->)<!-- -->, and the claude-code route pays a CLI session start-up on every call that a direct API call does not. Treat words/sec as a within-model comparison across N, not a race between vendors. <strong>Subscription routes are not billed per token</strong>: for those, the rate shown is what the same model costs on the metered API — the right number to reason with, but not an invoice.</p><p style="font-size:0.85rem;color:var(--ifm-color-emphasis-600)">Rates as published <!-- -->2026-07-14<!-- -->, pinned into each run so a later price change cannot silently restate what a past run cost. Refresh with<!-- --> <code>make update-model-prices</code>.</p><h2>The numbers</h2><div style="margin-bottom:28px"><h3 style="margin-bottom:4px">eu.anthropic.claude-sonnet-4-6<!-- --> <span style="font-weight:400;font-size:0.85rem;color:var(--ifm-color-emphasis-600)">bedrock<!-- --> · <!-- -->1<!-- -->× per N</span></h3><table style="width:100%;border-collapse:collapse;font-size:0.9rem"><thead><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">N</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Intact</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Missing</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Placeholder</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Tags</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Untranslated</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$ / 1k words</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Words/sec</th></tr></thead><tbody><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">16</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.081</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">45.0</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">32</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.073</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">54.4</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">64<span style="margin-left:6px;font-size:0.75rem;color:var(--ifm-color-emphasis-600)">shipped ceiling</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.070</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">56.9</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">128</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.068</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">59.8</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">256</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.067</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">63.1</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">600</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.171</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">15.0</td></tr></tbody></table></div><div style="margin-bottom:28px"><h3 style="margin-bottom:4px">gemini-3.1-flash-lite<!-- --> <span style="font-weight:400;font-size:0.85rem;color:var(--ifm-color-emphasis-600)">gemini<!-- --> · <!-- -->2<!-- -->× per N</span></h3><table style="width:100%;border-collapse:collapse;font-size:0.9rem"><thead><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">N</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Intact</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Missing</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Placeholder</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Tags</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Untranslated</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$ / 1k words</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Words/sec</th></tr></thead><tbody><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">8</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0058</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">318.8</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">16</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0056</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">404.5</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">32</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0057</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">454.1</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">64<span style="margin-left:6px;font-size:0.75rem;color:var(--ifm-color-emphasis-600)">shipped ceiling</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0056</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">414.5</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">128</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0055</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">420.8</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">256</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.0054</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">320.2</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">600</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.012</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">74.1</td></tr></tbody></table></div><div style="margin-bottom:28px"><h3 style="margin-bottom:4px">gemini-3.5-flash<!-- --> <span style="font-weight:400;font-size:0.85rem;color:var(--ifm-color-emphasis-600)">gemini<!-- --> · <!-- -->2<!-- -->× per N</span></h3><table style="width:100%;border-collapse:collapse;font-size:0.9rem"><thead><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">N</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Intact</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Missing</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Placeholder</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Tags</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Untranslated</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$ / 1k words</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">Words/sec</th></tr></thead><tbody><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">8</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:#d65a5a;font-weight:600">99.3%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">8</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">1</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.032</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">108.8</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">16</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:#d65a5a;font-weight:600">99.8%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">2</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.032</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">162.2</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">32</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.034</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">204.9</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">64<span style="margin-left:6px;font-size:0.75rem;color:var(--ifm-color-emphasis-600)">shipped ceiling</span></td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.034</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">230.4</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">128</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.034</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">247.0</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">256</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums;color:#d65a5a;font-weight:600">99.1%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">11</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.034</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">165.0</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">600</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">0</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">$0.071</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">34.9</td></tr></tbody></table></div><h2>Over time</h2><p>Intact% at the shipped ceiling (N=<!-- -->64<!-- -->), by run date. A model that silently regresses behind a stable alias shows up here — which is the point of keeping the record rather than measuring once.</p><table style="width:100%;border-collapse:collapse;font-size:0.9rem"><thead><tr><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">Model</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">2026-07-14</th><th style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">2026-07-15</th></tr></thead><tbody><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">eu.anthropic.claude-sonnet-4-6</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">gemini-3.1-flash-lite</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr><tr><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:left;font-variant-numeric:tabular-nums">gemini-3.5-flash</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td><td style="padding:6px 10px;border-bottom:1px solid var(--ifm-table-border-color);text-align:right;font-variant-numeric:tabular-nums">100.0%</td></tr></tbody></table><h2>Reproducing this</h2><p>The harness is <code>scripts/batcheval</code>. A run against the built-in demo stub exercises the harness and measures nothing about any model; such runs are marked<!-- --> <code>simulated</code> and are excluded from every chart on this page.</p><pre><code>make batch-eval # demo stub: proves the harness, measures nothing
make batch-eval-publish # the real sweep → this page's data</code></pre></main></div><footer class="theme-layout-footer footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="theme-layout-footer-column col footer__col"><div class="footer__title">Documentation</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/kapi/get-started/quickstart">Get started</a></li><li class="footer__item"><a class="footer__link-item" href="/kapi/overview">Kapi</a></li><li class="footer__item"><a class="footer__link-item" href="/framework/architecture">Framework</a></li><li class="footer__item"><a class="footer__link-item" href="/kapi/cli">Kapi CLI</a></li><li class="footer__item"><a class="footer__link-item" href="/toolbox/overview">CLI tools</a></li><li class="footer__item"><a class="footer__link-item" href="/react/introduction">Kapi React</a></li><li class="footer__item"><a class="footer__link-item" href="/reference">Reference</a></li><li class="footer__item"><a class="footer__link-item" href="/formats">Format Reference</a></li></ul></div><div class="theme-layout-footer-column col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/neokapi/neokapi" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-label="(opens in new tab)" class="iconExternalLink_oqBP"><use href="#theme-svg-external-link"></use></svg></a></li><li class="footer__item"><a href="https://github.com/neokapi/homebrew-tap" target="_blank" rel="noopener noreferrer" class="footer__link-item">Homebrew Tap<svg width="13.5" height="13.5" aria-label="(opens in new tab)" class="iconExternalLink_oqBP"><use href="#theme-svg-external-link"></use></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2026 neokapi contributors. Built with Docusaurus. · built 2026-07-26 02:57 UTC · 26464b2</div></div></div></footer></div>
</body>
</html>