From 295d1ae3b78a18a07ceec2f84e0eb2aabf7d70a6 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 15:51:49 -0400 Subject: [PATCH 1/8] fix: make docs generation alerts say why and what to do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly Slack alert named the affected toolkits and nothing else, so "MicrosoftUsers has no docs" required reading the run log to learn that the cause was a missing design-system entry — and the log's `0 tools` line reads like a different problem entirely. - Carry the failure reason into the report instead of dropping it, and build the message in `src/alerts/docs-alert.ts` where it can be tested. A jq program embedded in workflow YAML could only ever repeat toolkit names. - Lead with omitted toolkits, which have no page at all, over preserved ones, which still serve their last good output. - Map each known reason to its fix. - Link the job log rather than the run summary, and attach the full report as an artifact. - Drop `continue-on-error` from the Slack step. It swallowed a curl failure against an unset webhook secret, which is how the alert went unnoticed. The reviewer step keeps running on failure but now annotates the run. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 58 ++++++--- toolkit-docs-generator/README.md | 15 +++ .../src/alerts/docs-alert.ts | 96 +++++++++++++++ toolkit-docs-generator/src/cli/index.ts | 57 +++++++-- toolkit-docs-generator/src/utils/run-logs.ts | 16 ++- .../tests/alerts/docs-alert.test.ts | 111 ++++++++++++++++++ .../workflows/generate-toolkit-docs.test.ts | 26 +++- 7 files changed, 347 insertions(+), 32 deletions(-) create mode 100644 toolkit-docs-generator/src/alerts/docs-alert.ts create mode 100644 toolkit-docs-generator/tests/alerts/docs-alert.test.ts diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index e34997998..831e9d567 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -16,6 +16,9 @@ on: permissions: contents: write pull-requests: write + # Read this run's jobs so the Slack alert can deep-link the failing job log + # rather than the run summary. + actions: read concurrency: group: generate-toolkit-docs @@ -108,38 +111,59 @@ jobs: branch: automation/toolkit-docs delete-branch: true + # A token without reviewer permission is a real gap, but it must not fail + # the run: the generated PR is already open and useful without a reviewer + # attached. An annotation keeps it visible instead of silent. - name: Request team review if: steps.cpr.outputs.pull-request-number != '' - continue-on-error: true - run: gh pr edit ${{ steps.cpr.outputs.pull-request-number }} --add-reviewer ArcadeAI/engineering-tools-and-dx + run: | + if ! gh pr edit ${{ steps.cpr.outputs.pull-request-number }} \ + --add-reviewer ArcadeAI/engineering-tools-and-dx 2>review-error.log; then + echo "::warning::Could not request review on PR #${{ steps.cpr.outputs.pull-request-number }}: $(cat review-error.log)" + fi env: GH_TOKEN: ${{ secrets.DOCS_PUBLISHABLE_GH_TOKEN }} - - name: Warn #proj-docs about preserved or omitted toolkit docs - continue-on-error: true + - name: Upload generation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: failed-tools + path: toolkit-docs-generator-verification/logs/failed-tools.json + if-no-files-found: ignore + + # No `continue-on-error` here on purpose. A Slack step that dies quietly + # is how a wrong webhook secret went unnoticed: the alert it was meant to + # deliver is exactly what nobody was watching for. + - name: Report preserved or omitted toolkits to Slack run: | - report=toolkit-docs-generator-verification/logs/failed-tools.json - if [ ! -f "$report" ]; then - exit 0 - fi + job_url=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \ + --jq '.jobs[] | select(.name == "generate") | .html_url' | head -n 1) + + payload=$(../node_modules/.bin/tsx src/cli/index.ts alert \ + --report ../toolkit-docs-generator-verification/logs/failed-tools.json \ + --log-url "${job_url:-$RUN_URL}") - preserved=$(jq -r '(.preservedToolkits // []) | join(", ")' "$report") - omitted=$(jq -r '(.omittedToolkits // []) | join(", ")' "$report") - if [ -z "$preserved" ] && [ -z "$omitted" ]; then + if [ -z "$payload" ]; then + echo "No preserved or omitted toolkits to report." exit 0 fi - payload=$(jq -n \ - --arg run_url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ - --arg preserved "$preserved" \ - --arg omitted "$omitted" \ - '{text: (":warning: Toolkit docs generation completed with recoverable failures\n\n*Workflow run:* <" + $run_url + "|Open run>" + (if $preserved != "" then "\n*Continuing to serve previous docs:* " + $preserved else "" end) + (if $omitted != "" then "\n*No docs are being served (no prior output):* " + $omitted else "" end))}') + if [ -z "$SLACK_WEBHOOK_URL" ]; then + echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured" + exit 1 + fi curl --fail-with-body --silent --show-error \ -X POST \ -H "Content-Type: application/json" \ --data "$payload" \ - "${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }}" + "$SLACK_WEBHOOK_URL" + working-directory: toolkit-docs-generator + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }} alert: name: Alert on generation failure diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index 223425986..9ece729b8 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -252,6 +252,21 @@ authored prose on the next generation run. Invalid frontmatter, invalid MDX, unknown tool targets, symlinks, unsafe subpage paths, and leftover JSON curation fail generation instead of silently falling back to stale generated content. +## Run alerts + +The nightly workflow posts to Slack when a toolkit fails and the run recovers +from it. Two outcomes, and the difference matters: + +- **Missing entirely**: the toolkit failed and had no previous artifact to fall + back on, so it has no page on the docs site at all. +- **Still serving the previous docs**: the toolkit failed but still publishes + its last good output, so readers see yesterday's page. + +Each entry names the reason and the fix. Every run attaches the full report as +its `failed-tools` artifact. Build the message locally against any report with +`pnpm dlx tsx src/cli/index.ts alert --report `. It prints nothing when a +run has nothing to report. + ## Troubleshooting - **Nothing regenerated**: `--skip-unchanged` exits early when tool definitions did not change. diff --git a/toolkit-docs-generator/src/alerts/docs-alert.ts b/toolkit-docs-generator/src/alerts/docs-alert.ts new file mode 100644 index 000000000..cb6a7b1fc --- /dev/null +++ b/toolkit-docs-generator/src/alerts/docs-alert.ts @@ -0,0 +1,96 @@ +/** + * Slack message for a generation run that recovered from toolkit failures. + * + * The text is built here rather than in workflow YAML so that it is testable, + * and so it can carry the two things the report already knows but a bare list + * of toolkit names cannot say: why the toolkit failed, and what to do about it. + * + * `omitted` outranks `preserved` throughout. A preserved toolkit still serves + * yesterday's page and nobody notices; an omitted toolkit has no page at all. + */ +import type { + FailedToolsReport, + RecoveredToolkitEntry, +} from "../utils/run-logs"; + +export interface SlackMessage { + readonly text: string; +} + +export interface DocsAlertLinks { + /** Deep link to the job log, so the reader lands on the error, not a summary. */ + readonly logUrl?: string; +} + +const MISSING_METADATA_REASON = "missing design-system metadata"; +const CURATION_REASON_PREFIX = "Curation"; + +const suggestFix = (entry: RecoveredToolkitEntry): string => { + if (entry.reason.includes(MISSING_METADATA_REASON)) { + return `Add a \`${entry.id}\` entry to \`@arcadeai/design-system\` and bump the pin in this repo, or add it to \`skip-toolkits.txt\` if it is not meant to be public yet.`; + } + if (entry.reason.startsWith(CURATION_REASON_PREFIX)) { + return `Fix the curation source under \`toolkit-docs-generator/curation/${entry.id.toLowerCase()}/\`.`; + } + return "Open the failing step for the full error."; +}; + +const formatEntry = (entry: RecoveredToolkitEntry): string => + [`• *${entry.id}* — ${entry.reason}`, ` Fix: ${suggestFix(entry)}`].join( + "\n" + ); + +const formatSection = ( + heading: string, + entries: readonly RecoveredToolkitEntry[] +): string[] => + entries.length === 0 + ? [] + : [`*${heading}*`, entries.map(formatEntry).join("\n")]; + +const formatHeadline = ( + omitted: readonly RecoveredToolkitEntry[], + preserved: readonly RecoveredToolkitEntry[] +): string => { + if (omitted.length === 1) { + return `:no_entry: ${omitted[0]?.id} is missing from the docs site`; + } + if (omitted.length > 1) { + return `:no_entry: ${omitted.length} toolkits are missing from the docs site`; + } + if (preserved.length === 1) { + return `:warning: ${preserved[0]?.id} kept its previous docs`; + } + return `:warning: ${preserved.length} toolkits kept their previous docs`; +}; + +/** + * Build the Slack payload for a finished run, or null when the run had nothing + * worth interrupting anyone about. + */ +export const buildDocsAlert = ( + report: Pick, + links: DocsAlertLinks = {} +): SlackMessage | null => { + const recovered = report.recoveredToolkits ?? []; + if (recovered.length === 0) { + return null; + } + + const omitted = recovered.filter((entry) => entry.recovery === "omitted"); + const preserved = recovered.filter((entry) => entry.recovery === "preserved"); + + const blocks = [ + formatHeadline(omitted, preserved), + ...formatSection("Missing entirely (no previous output)", omitted), + ...formatSection("Still serving the previous docs", preserved), + ]; + + if (links.logUrl) { + blocks.push( + `<${links.logUrl}|Open the failing step> · full detail in the \`failed-tools\` artifact on the run` + ); + } + + return { text: blocks.join("\n\n") }; +}; diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index dedac9e17..449b378d1 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -13,9 +13,11 @@ import chalk from "chalk"; import { Command } from "commander"; +import { existsSync } from "fs"; import { readdir, readFile } from "fs/promises"; import ora from "ora"; import { join, resolve } from "path"; +import { buildDocsAlert } from "../alerts/docs-alert"; import { detectChanges, formatChangeSummary, @@ -1826,12 +1828,21 @@ program 0 ); const failedToolkits = mergeFailures.map((result) => result.toolkit.id); - const preservedToolkits = mergeFailures - .filter((result) => result.recovery === "preserved") - .map((result) => result.toolkit.id); - const omittedToolkits = mergeFailures - .filter((result) => result.recovery === "omitted") - .map((result) => result.toolkit.id); + // The reason travels with the toolkit id: it is what turns "MicrosoftUsers + // has no page" into something a reader can act on without opening the log. + const recoveredToolkits = mergeFailures + .filter((result) => result.recovery !== undefined) + .map((result) => ({ + id: result.toolkit.id, + recovery: result.recovery as "preserved" | "omitted", + reason: result.error ?? "unknown error", + })); + const preservedToolkits = recoveredToolkits + .filter((entry) => entry.recovery === "preserved") + .map((entry) => entry.id); + const omittedToolkits = recoveredToolkits + .filter((entry) => entry.recovery === "omitted") + .map((entry) => entry.id); const failedTools = allResults.flatMap((result) => result.failedTools); const failedToolkitsFromTools = Array.from( new Set(failedTools.map((tool) => tool.toolkitId)) @@ -1901,8 +1912,7 @@ program generatedAt: new Date().toISOString(), toolkits: failedToolkitsFromTools, failedToolkits, - preservedToolkits, - omittedToolkits, + recoveredToolkits, tools: failedTools, }); @@ -3032,5 +3042,36 @@ program } ); +program + .command("alert") + .description( + "Print the Slack payload for a finished run, or nothing when the run had no recoverable failures" + ) + .option( + "--report ", + "Failed tools report to summarize", + buildLogPaths(getDefaultLogDir()).failedToolsPath + ) + .option( + "--log-url ", + "Link the message should point at, ideally the job log rather than the run summary" + ) + .action(async (options: { report: string; logUrl?: string }) => { + // Silence rather than failure when there is no report: a clean run never + // writes one, and CI should not need to know the difference. + if (!existsSync(options.report)) { + return; + } + + const report = await readFailedToolsReport(options.report); + const message = buildDocsAlert( + report, + options.logUrl ? { logUrl: options.logUrl } : {} + ); + if (message) { + console.log(JSON.stringify(message)); + } + }); + // Parse command line arguments program.parse(); diff --git a/toolkit-docs-generator/src/utils/run-logs.ts b/toolkit-docs-generator/src/utils/run-logs.ts index ecf4045d5..156eda297 100644 --- a/toolkit-docs-generator/src/utils/run-logs.ts +++ b/toolkit-docs-generator/src/utils/run-logs.ts @@ -22,12 +22,24 @@ export interface FailedToolEntry { readonly reason: string; } +/** + * A toolkit whose merge failed and was recovered rather than aborting the run. + * + * `preserved` keeps serving the previous artifact; `omitted` means there was no + * previous artifact, so the toolkit has no page at all. The reason travels with + * the entry because it is the only thing that tells a reader what to fix. + */ +export interface RecoveredToolkitEntry { + readonly id: string; + readonly recovery: "preserved" | "omitted"; + readonly reason: string; +} + export interface FailedToolsReport { readonly generatedAt: string; readonly toolkits: readonly string[]; readonly failedToolkits: readonly string[]; - readonly preservedToolkits?: readonly string[]; - readonly omittedToolkits?: readonly string[]; + readonly recoveredToolkits?: readonly RecoveredToolkitEntry[]; readonly tools: readonly FailedToolEntry[]; } diff --git a/toolkit-docs-generator/tests/alerts/docs-alert.test.ts b/toolkit-docs-generator/tests/alerts/docs-alert.test.ts new file mode 100644 index 000000000..55296fa54 --- /dev/null +++ b/toolkit-docs-generator/tests/alerts/docs-alert.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; + +import { buildDocsAlert } from "../../src/alerts/docs-alert"; + +const omittedMicrosoftUsers = { + id: "MicrosoftUsers", + recovery: "omitted" as const, + reason: "missing design-system metadata", +}; + +const preservedGithub = { + id: "Github", + recovery: "preserved" as const, + reason: "Curation for Github targets unknown tool(s): CreateIsue", +}; + +describe("buildDocsAlert", () => { + it("stays silent when nothing was preserved or omitted", () => { + expect(buildDocsAlert({ recoveredToolkits: [] })).toBeNull(); + expect(buildDocsAlert({})).toBeNull(); + }); + + it("names the toolkit in the headline when only one is missing", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain( + ":no_entry: MicrosoftUsers is missing from the docs site" + ); + }); + + it("reports the reason, not just the toolkit name", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain("missing design-system metadata"); + }); + + it("tells the reader how to fix missing design-system metadata", () => { + const message = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(message?.text).toContain("`MicrosoftUsers` entry to"); + expect(message?.text).toContain("@arcadeai/design-system"); + expect(message?.text).toContain("skip-toolkits.txt"); + }); + + it("points curation failures at the curation directory", () => { + const message = buildDocsAlert({ recoveredToolkits: [preservedGithub] }); + + expect(message?.text).toContain("curation/github/"); + }); + + it("falls back to the log for an unrecognized reason", () => { + const message = buildDocsAlert({ + recoveredToolkits: [ + { id: "Slack", recovery: "preserved", reason: "socket hang up" }, + ], + }); + + expect(message?.text).toContain( + "Open the failing step for the full error." + ); + }); + + // An omitted toolkit has no page at all; a preserved one still serves + // yesterday's. Leading with the softer of the two is how the real alert + // read as "handled, ignore me". + it("leads with the omitted toolkits when both kinds are present", () => { + const message = buildDocsAlert({ + recoveredToolkits: [preservedGithub, omittedMicrosoftUsers], + }); + const text = message?.text ?? ""; + + expect(text.startsWith(":no_entry:")).toBe(true); + expect(text.indexOf("Missing entirely")).toBeLessThan( + text.indexOf("Still serving the previous docs") + ); + }); + + it("counts toolkits when more than one is missing", () => { + const message = buildDocsAlert({ + recoveredToolkits: [ + omittedMicrosoftUsers, + { ...omittedMicrosoftUsers, id: "MicrosoftDynamics" }, + ], + }); + + expect(message?.text).toContain( + ":no_entry: 2 toolkits are missing from the docs site" + ); + }); + + it("links the job log when one is given, and omits the line when not", () => { + const withLink = buildDocsAlert( + { recoveredToolkits: [omittedMicrosoftUsers] }, + { logUrl: "https://github.com/ArcadeAI/docs/actions/runs/1/job/2" } + ); + const withoutLink = buildDocsAlert({ + recoveredToolkits: [omittedMicrosoftUsers], + }); + + expect(withLink?.text).toContain( + "" + ); + expect(withoutLink?.text).not.toContain("Open the failing step>"); + }); +}); diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index 2e739f501..cf131e261 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -80,11 +80,27 @@ test("porter workflow alerts Slack when generation fails", () => { expect(workflowContents).not.toContain("\\\\n"); }); -test("porter workflow warns when it preserves or omits a broken toolkit", () => { - expect(workflowContents).toContain("preservedToolkits"); - expect(workflowContents).toContain("omittedToolkits"); - expect(workflowContents).toContain("Continuing to serve previous docs"); - expect(workflowContents).toContain("No docs are being served"); +test("porter workflow builds the preserved/omitted message in tested code", () => { + // The message text lives in src/alerts/docs-alert.ts, not in this YAML. A jq + // program embedded in a workflow is untestable and drifts from what the + // generator actually knows about each failure. + expect(workflowContents).toContain("src/cli/index.ts alert"); + expect(workflowContents).toContain("--log-url"); +}); + +test("porter workflow lets a failed Slack post fail the job", () => { + // A wrong webhook secret went unnoticed because this step swallowed its own + // failure. The alert nobody is watching for is the one that must be loud. + const slackStep = workflowContents.slice( + workflowContents.indexOf("Report preserved or omitted toolkits to Slack") + ); + expect(slackStep).not.toContain("continue-on-error"); +}); + +test("porter workflow keeps the failure report as an artifact", () => { + expect(workflowContents).toContain("actions/upload-artifact"); + expect(workflowContents).toContain("failed-tools"); + expect(workflowContents).toContain("actions: read"); }); test("workflow dispatch keeps default full-run behavior", () => { From d96b106e43990f6c99437604be13789fb0e6c1ec Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 16:06:19 -0400 Subject: [PATCH 2/8] fix: keep a failed Slack post from claiming generation failed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Slack step can now fail the job, and the alert job fires on any job failure — so a broken webhook would announce a generation failure that never happened, sending people to look for one. The generate step records that it finished, and the alert job requires that marker to be absent. A later step failing still turns the run red, which is the visible signal it should be. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 13 ++++++++++++- .../tests/workflows/generate-toolkit-docs.test.ts | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index 831e9d567..e64e8a032 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -31,6 +31,11 @@ jobs: # switch on 2026-06-02. Harmless today; unblocks the cutover. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + outputs: + # Distinguishes "generation itself failed" from "a later step failed". + # Without it, a broken notification would make the alert job announce a + # generation failure that never happened. + generation-succeeded: ${{ steps.generate-docs.outputs.succeeded }} steps: - name: Checkout repository @@ -51,6 +56,7 @@ jobs: run: pnpm install --frozen-lockfile - name: Generate toolkit docs + id: generate-docs # Invoked by path rather than through `pnpm exec`, which would reset the # working directory to the repo root and break the relative paths below. run: | @@ -75,6 +81,8 @@ jobs: --ignore-file ./skip-toolkits.txt \ --custom-sections ./curation \ --output data/toolkits + + echo "succeeded=true" >>"$GITHUB_OUTPUT" working-directory: toolkit-docs-generator env: ENGINE_API_URL: ${{ secrets.ENGINE_API_URL }} @@ -167,7 +175,10 @@ jobs: alert: name: Alert on generation failure - if: ${{ always() && needs.generate.result == 'failure' }} + # Only when generation itself failed. A red job whose generate step + # succeeded means a later step broke — reporting that as "generation + # failed" would send people looking for a failure that isn't there. + if: ${{ always() && needs.generate.result == 'failure' && needs.generate.outputs.generation-succeeded != 'true' }} needs: generate runs-on: ubuntu-latest permissions: {} diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index cf131e261..4c710834f 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -97,6 +97,17 @@ test("porter workflow lets a failed Slack post fail the job", () => { expect(slackStep).not.toContain("continue-on-error"); }); +test("porter workflow only claims generation failed when generation failed", () => { + // The Slack step can now fail the job. Without this guard, a broken webhook + // would make the alert job announce a generation failure that never happened. + expect(workflowContents).toContain("generation-succeeded:"); + expect(workflowContents).toContain("steps.generate-docs.outputs.succeeded"); + expect(workflowContents).toContain('echo "succeeded=true"'); + expect(workflowContents).toContain( + "needs.generate.outputs.generation-succeeded != 'true'" + ); +}); + test("porter workflow keeps the failure report as an artifact", () => { expect(workflowContents).toContain("actions/upload-artifact"); expect(workflowContents).toContain("failed-tools"); From f5f855bdaac868f2974bce0b09efa6c7e6d53c94 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 16:20:47 -0400 Subject: [PATCH 3/8] fix: don't let the job-URL lookup swallow the Slack alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Actions runs `run:` steps with `-e -o pipefail`, so the job-URL lookup could abort the step before curl and lose the alert it exists to send. `${job_url:-$RUN_URL}` covers an empty URL, not a failed lookup, and because generation itself succeeded the failure-alert job is suppressed — so nothing reaches Slack. Two ways it could fail: - `gh api` exits non-zero (the token already hits "Resource not accessible" on the reviewer step, so this is not hypothetical). - `gh api` succeeds and `| head -n 1` SIGPIPEs it, which `pipefail` turns into exit 141. Select the first match in jq instead of piping to head, and fall back to the run URL on any failure. Verified under `-e -o pipefail` that all four outcomes — API failure, no matching job, empty job list, and a match — now reach curl. Reported by Cursor Bugbot on #1124. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index e64e8a032..53f4bce16 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -145,8 +145,13 @@ jobs: # deliver is exactly what nobody was watching for. - name: Report preserved or omitted toolkits to Slack run: | + # The deep link is a nicety; the alert is the point. Actions runs this + # with `-e -o pipefail`, so a failed `gh api` — or a SIGPIPE from + # trimming its output — would abort the step before curl and lose the + # alert entirely. Select the first match in jq instead of piping to + # head, and fall back to the run URL on any failure. job_url=$(gh api "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs" \ - --jq '.jobs[] | select(.name == "generate") | .html_url' | head -n 1) + --jq '[.jobs[] | select(.name == "generate") | .html_url] | .[0] // empty') || job_url="" payload=$(../node_modules/.bin/tsx src/cli/index.ts alert \ --report ../toolkit-docs-generator-verification/logs/failed-tools.json \ From e6da31b1c2584ee406b3b7c25207ba21c2a7decd Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 16:28:15 -0400 Subject: [PATCH 4/8] fix: report preserved or omitted toolkits even when a later step fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recoverable Slack step ran last, under the default "everything so far succeeded" gate. A failed sidebar sync or PR creation therefore skipped it — and because generation itself succeeded, `generation-succeeded` suppressed the failure-alert job too. Missing and stale pages went unannounced in both directions, with the reason sitting in an uploaded artifact nobody had a reason to open. Gate the step on the generate step instead, which is the signal the workflow already tracks to tell "generation broke" from "a later step broke". `!cancelled()` rather than `always()`, since `cancel-in-progress` cancels superseded runs and those have nothing to report. Checked every combination of cancelled, generate outcome, and job result: no case sends both alerts, and no case drops a recoverable report. Reported by Cursor Bugbot on #1124. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index 53f4bce16..dcf8f6ee2 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -143,7 +143,16 @@ jobs: # No `continue-on-error` here on purpose. A Slack step that dies quietly # is how a wrong webhook secret went unnoticed: the alert it was meant to # deliver is exactly what nobody was watching for. + # + # Gated on the generate step rather than on the default "everything so far + # succeeded". Missing or stale pages are worth reporting even when a later + # step broke — under the default gate a failed sidebar sync or PR creation + # would skip this step, while `generation-succeeded` suppressed the alert + # job below, and nobody heard about the toolkits at all. `!cancelled()` + # rather than `always()` because `cancel-in-progress` cancels superseded + # runs, and those have nothing to report. - name: Report preserved or omitted toolkits to Slack + if: ${{ !cancelled() && steps.generate-docs.outputs.succeeded == 'true' }} run: | # The deep link is a nicety; the alert is the point. Actions runs this # with `-e -o pipefail`, so a failed `gh api` — or a SIGPIPE from From a3e987b1a3ac39d7513a6d9a5788d83d3887b610 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 16:31:48 -0400 Subject: [PATCH 5/8] fix: alert Slack when docs generate but never publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A red run whose generate step succeeded sent nothing to Slack: the recoverable report had nothing to say, and the alert job deliberately excluded that case to avoid calling a failed sidebar sync a "generation failure". So an auto-PR that never got created was visible only to someone who happened to open the Actions tab. Give the alert job the third case instead of excluding it. It now fires on every failed generate job and picks wording from `generation-succeeded`: a validation error points at the run, and a publishing break points at sidebar sync and PR creation, and says there may be no auto-PR to merge. A run that recovers from a toolkit failure and then fails to publish sends two messages, on purpose — missing pages and a broken publish are separate things to fix. Checked every combination of cancelled, generate outcome, job result, and recoverable-toolkit presence: no red run is silent. Rendered both alert bodies from the workflow's own run block to confirm valid JSON and the right copy. README documents the three messages. Reported by Cursor Bugbot on #1124. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/generate-toolkit-docs.yml | 24 +++++++++++++++------ toolkit-docs-generator/README.md | 15 +++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index dcf8f6ee2..ff10d2abc 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -188,11 +188,12 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROJ_DOCS_WEBHOOK_URL }} alert: - name: Alert on generation failure - # Only when generation itself failed. A red job whose generate step - # succeeded means a later step broke — reporting that as "generation - # failed" would send people looking for a failure that isn't there. - if: ${{ always() && needs.generate.result == 'failure' && needs.generate.outputs.generation-succeeded != 'true' }} + name: Alert on failure + # Every red run gets a message. Which message depends on whether generation + # itself broke or something after it did: those need different people to do + # different things, and calling a failed sidebar sync a "generation failure" + # sends someone hunting for a validation error that doesn't exist. + if: ${{ !cancelled() && needs.generate.result == 'failure' }} needs: generate runs-on: ubuntu-latest permissions: {} @@ -203,17 +204,28 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} TRIGGER: ${{ github.event_name }} COMMIT: ${{ github.sha }} + GENERATION_SUCCEEDED: ${{ needs.generate.outputs.generation-succeeded }} run: | if [ -z "$SLACK_WEBHOOK_URL" ]; then echo "::error::SLACK_PROJ_DOCS_WEBHOOK_URL is not configured" exit 1 fi + if [ "$GENERATION_SUCCEEDED" = "true" ]; then + headline=":warning: Toolkit docs were generated but never published" + detail="Generation succeeded, so the toolkit JSON is fine — a step after it failed (sidebar sync, PR creation, or the Slack report). Today's docs are not on their way to the site until this run's work lands, and there may be no auto-PR to merge." + else + headline=":rotating_light: Toolkit docs generation failed" + detail="The failed run contains the exact file path and validation error." + fi + payload=$(jq -n \ + --arg headline "$headline" \ + --arg detail "$detail" \ --arg run_url "$RUN_URL" \ --arg trigger "$TRIGGER" \ --arg commit "$COMMIT" \ - '{text: (":rotating_light: Toolkit docs generation failed\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\nThe failed run contains the exact file path and validation error.")}') + '{text: ($headline + "\n\n*Workflow run:* <" + $run_url + "|Open failed run>\n*Trigger:* " + $trigger + "\n*Commit:* " + $commit + "\n\n" + $detail)}') curl --fail-with-body --silent --show-error \ -X POST \ diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index 9ece729b8..62ed6ca64 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -267,6 +267,21 @@ its `failed-tools` artifact. Build the message locally against any report with `pnpm dlx tsx src/cli/index.ts alert --report `. It prints nothing when a run has nothing to report. +No red run stays quiet. Three messages can arrive, and they answer different +questions: + +| Message | Means | Who acts | +| --- | --- | --- | +| Toolkits missing or stale | Generation recovered from a per-toolkit failure | Follow the fix named in the entry | +| Toolkit docs generation failed | The generate step itself broke | Open the run for the path and validation error | +| Generated but never published | Generation was fine, a later step broke | Check sidebar sync and PR creation — there may be no auto-PR to merge | + +The first is reported from inside the generate job, gated on the generate step +rather than on the steps after it, so a broken sidebar sync or PR creation does +not hide missing pages. The other two come from the alert job. A run that both +recovers from a toolkit failure and then fails to publish sends the first +message and the third, because those are two separate things to fix. + ## Troubleshooting - **Nothing regenerated**: `--skip-unchanged` exits early when tool definitions did not change. From 38c3a4b0435e059d0a97f066fc7089b88067e989 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 12 Aug 2026 16:50:10 -0400 Subject: [PATCH 6/8] test: match the workflow assertions to the two-headline alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The alert job now picks its headline from whether generation finished, rather than staying silent when a later step fails. Two assertions still described the old shape: one pinned the literal "generation failed" prefix that jq now builds from a variable, and one pinned the `if:` guard that moved into the step body. Both keep their original intent — that the escaping stays correct, and that a red run after successful generation is not reported as a generation failure. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/generate-toolkit-docs.test.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index 4c710834f..d8850b6af 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -70,13 +70,13 @@ test("porter workflow opts JS actions into Node 24 to unblock the 2026-06-02 dep ); }); -test("porter workflow alerts Slack when generation fails", () => { +test("porter workflow alerts Slack on a red run", () => { expect(workflowContents).toContain("needs.generate.result == 'failure'"); expect(workflowContents).toContain("SLACK_PROJ_DOCS_WEBHOOK_URL"); // The jq program is single-quoted, so the shell passes backslashes through // untouched. `\n` reaches jq as a newline escape; `\\n` would reach it as an // escaped backslash followed by "n" and Slack would print a literal "\n". - expect(workflowContents).toContain("generation failed\\n\\n*Workflow run:*"); + expect(workflowContents).toContain('$headline + "\\n\\n*Workflow run:* <"'); expect(workflowContents).not.toContain("\\\\n"); }); @@ -98,14 +98,21 @@ test("porter workflow lets a failed Slack post fail the job", () => { }); test("porter workflow only claims generation failed when generation failed", () => { - // The Slack step can now fail the job. Without this guard, a broken webhook - // would make the alert job announce a generation failure that never happened. + // Steps after generation can now fail the job. Calling that a generation + // failure sends someone hunting for a validation error that doesn't exist, + // so the alert picks its headline from whether generation itself finished. expect(workflowContents).toContain("generation-succeeded:"); expect(workflowContents).toContain("steps.generate-docs.outputs.succeeded"); expect(workflowContents).toContain('echo "succeeded=true"'); + expect(workflowContents).toContain("GENERATION_SUCCEEDED:"); expect(workflowContents).toContain( - "needs.generate.outputs.generation-succeeded != 'true'" + "needs.generate.outputs.generation-succeeded" ); + expect(workflowContents).toContain('if [ "$GENERATION_SUCCEEDED" = "true" ]'); + expect(workflowContents).toContain( + "Toolkit docs were generated but never published" + ); + expect(workflowContents).toContain("Toolkit docs generation failed"); }); test("porter workflow keeps the failure report as an artifact", () => { From 346f4ff16325aec2898bcbf79739dce09a171ccd Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 13 Aug 2026 13:00:33 -0400 Subject: [PATCH 7/8] docs: clarify tsx invocation requirement --- toolkit-docs-generator/CURATION.md | 5 ++--- toolkit-docs-generator/README.md | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/toolkit-docs-generator/CURATION.md b/toolkit-docs-generator/CURATION.md index 3fa6b0319..3b14d09f6 100644 --- a/toolkit-docs-generator/CURATION.md +++ b/toolkit-docs-generator/CURATION.md @@ -241,9 +241,8 @@ toolkit with its message, and exits non-zero on failure. It runs the same compiler generation uses, so a pass means kinds, frontmatter, and MDX are all valid. It cannot check `tool:` targets. -Invoke `tsx` by path, not through `pnpm exec`, which resets the working -directory to the repo root and breaks the relative path. `pnpm dlx tsx` also -works. +Invoke `tsx` by path because `toolkit-docs-generator` has no package for +`pnpm exec`. Seeing a chunk on a page needs generated JSON. Either wait for the automated generation PR, or generate that one toolkit with Engine credentials and run diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index af837b7fd..737503e25 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -129,7 +129,9 @@ pnpm dlx tsx src/cli/index.ts generate \ ## Local usage -Run these commands from the `toolkit-docs-generator` directory. +Run these commands from the `toolkit-docs-generator` directory. Invoke `tsx` by +path because this directory has no package for `pnpm exec`; the sidebar sync +command below runs from the repo root instead. Generate a single toolkit: From 530346967a80d06edb42b9c9895ba5e613899dec Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 13 Aug 2026 13:15:10 -0400 Subject: [PATCH 8/8] fix: avoid mislabeling post-generation failures --- .github/workflows/generate-toolkit-docs.yml | 8 ++++---- toolkit-docs-generator/README.md | 2 +- .../tests/workflows/generate-toolkit-docs.test.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/generate-toolkit-docs.yml b/.github/workflows/generate-toolkit-docs.yml index ff10d2abc..3cbdcaa35 100644 --- a/.github/workflows/generate-toolkit-docs.yml +++ b/.github/workflows/generate-toolkit-docs.yml @@ -191,8 +191,8 @@ jobs: name: Alert on failure # Every red run gets a message. Which message depends on whether generation # itself broke or something after it did: those need different people to do - # different things, and calling a failed sidebar sync a "generation failure" - # sends someone hunting for a validation error that doesn't exist. + # different things, and calling a post-generation failure a "generation + # failure" sends someone hunting for a validation error that doesn't exist. if: ${{ !cancelled() && needs.generate.result == 'failure' }} needs: generate runs-on: ubuntu-latest @@ -212,8 +212,8 @@ jobs: fi if [ "$GENERATION_SUCCEEDED" = "true" ]; then - headline=":warning: Toolkit docs were generated but never published" - detail="Generation succeeded, so the toolkit JSON is fine — a step after it failed (sidebar sync, PR creation, or the Slack report). Today's docs are not on their way to the site until this run's work lands, and there may be no auto-PR to merge." + headline=":warning: Toolkit docs generated, but the workflow failed afterward" + detail="Generation succeeded, so the toolkit JSON is fine. A later workflow step failed — inspect the run to determine whether sidebar sync, PR creation, artifact upload, or Slack notification needs attention." else headline=":rotating_light: Toolkit docs generation failed" detail="The failed run contains the exact file path and validation error." diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md index 737503e25..bc971cb0f 100644 --- a/toolkit-docs-generator/README.md +++ b/toolkit-docs-generator/README.md @@ -287,7 +287,7 @@ questions: | --- | --- | --- | | Toolkits missing or stale | Generation recovered from a per-toolkit failure | Follow the fix named in the entry | | Toolkit docs generation failed | The generate step itself broke | Open the run for the path and validation error | -| Generated but never published | Generation was fine, a later step broke | Check sidebar sync and PR creation — there may be no auto-PR to merge | +| Generated, but workflow failed afterward | Generation was fine, a later step broke | Open the run to determine whether sidebar sync, PR creation, artifact upload, or Slack notification needs attention | The first is reported from inside the generate job, gated on the generate step rather than on the steps after it, so a broken sidebar sync or PR creation does diff --git a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts index d8850b6af..bad7b93cf 100644 --- a/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts +++ b/toolkit-docs-generator/tests/workflows/generate-toolkit-docs.test.ts @@ -110,7 +110,7 @@ test("porter workflow only claims generation failed when generation failed", () ); expect(workflowContents).toContain('if [ "$GENERATION_SUCCEEDED" = "true" ]'); expect(workflowContents).toContain( - "Toolkit docs were generated but never published" + "Toolkit docs generated, but the workflow failed afterward" ); expect(workflowContents).toContain("Toolkit docs generation failed"); });