PR#1415 — Merge main into staging

.github/scripts/select_docs_preview_targets.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Select pages that are safe to render incrementally for a PR preview."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+
+PAGE_SUFFIXES = {".qmd", ".md", ".ipynb"}
+ASSET_SUFFIXES = {".avif", ".gif", ".jpeg", ".jpg", ".pdf", ".png", ".svg", ".webp"}
+UNSAFE_TOP_LEVEL = {
+    "_extensions",
+    "_freeze",
+    "_source",
+    "environments",
+    "llm",
+    "scripts",
+}
+STATUS_MAP = {
+    "added": "A",
+    "modified": "M",
+    "removed": "D",
+    "renamed": "R",
+    "copied": "C",
+    "changed": "T",
+}
+
+
+@dataclass(frozen=True)
+class Selection:
+    targets: tuple[str, ...] = ()
+    assets: tuple[str, ...] = ()
+    fallback_reason: str | None = None
+
+    @property
+    def is_targeted(self) -> bool:
+        return self.fallback_reason is None
+
+
+def select(changes: list[tuple[str, tuple[str, ...]]]) -> Selection:
+    targets: set[str] = set()
+    assets: set[str] = set()
+
+    for status, paths in changes:
+        if status not in {"A", "M"} or len(paths) != 1:
+            return Selection(fallback_reason=f"{status} change requires a full render")
+
+        path = PurePosixPath(paths[0])
+        if not path.parts or path.parts[0] != "site" or len(path.parts) < 2:
+            return Selection(fallback_reason=f"{path} is outside targetable site content")
+
+        relative = PurePosixPath(*path.parts[1:])
+        if relative.parts[0] in UNSAFE_TOP_LEVEL:
+            return Selection(fallback_reason=f"{path} can affect generated or global content")
+        if any(part.startswith("_") for part in relative.parts):
+            return Selection(fallback_reason=f"{path} is Quarto metadata or shared content")
+
+        suffix = relative.suffix.lower()
+        if suffix in PAGE_SUFFIXES:
+            targets.add(relative.as_posix())
+        elif suffix in ASSET_SUFFIXES:
+            assets.add(relative.as_posix())
+        else:
+            return Selection(fallback_reason=f"{path} is not a targetable page or asset")
+
+    if not targets:
+        return Selection(fallback_reason="no changed renderable pages were found")
+
+    return Selection(tuple(sorted(targets)), tuple(sorted(assets)))
+
+
+def parse_changed_files(output: str) -> list[tuple[str, tuple[str, ...]]]:
+    changes: list[tuple[str, tuple[str, ...]]] = []
+    for line in output.splitlines():
+        fields = line.split("\t")
+        if len(fields) < 2:
+            raise ValueError(f"Unexpected changed-file line: {line!r}")
+        status = STATUS_MAP.get(fields[0], fields[0][0].upper())
+        changes.append((status, tuple(field for field in fields[1:] if field)))
+    return changes
+
+
+def write_lines(path: Path, values: tuple[str, ...]) -> None:
+    path.write_text("".join(f"{value}\n" for value in values))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--changes", default="-")
+    parser.add_argument("--targets", type=Path, required=True)
+    parser.add_argument("--assets", type=Path, required=True)
+    args = parser.parse_args()
+
+    if args.changes == "-":
+        changed_files = sys.stdin.read()
+    else:
+        changed_files = Path(args.changes).read_text()
+
+    selection = select(parse_changed_files(changed_files))
+    if not selection.is_targeted:
+        print(f"Full render required: {selection.fallback_reason}")
+        return 3
+
+    write_lines(args.targets, selection.targets)
+    write_lines(args.assets, selection.assets)
+    print("Targeted render pages:")
+    print("\n".join(selection.targets))
+    if selection.assets:
+        print("Targeted preview assets:")
+        print("\n".join(selection.assets))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/test_select_docs_preview_targets.py

@@ -0,0 +1,65 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import unittest
+
+from select_docs_preview_targets import parse_changed_files, select
+
+
+class SelectDocsPreviewTargetsTest(unittest.TestCase):
+    def test_selects_changed_pages_and_assets(self):
+        result = select(
+            [
+                ("M", ("site/guide/example.qmd",)),
+                ("A", ("site/guide/images/example.png",)),
+            ]
+        )
+
+        self.assertTrue(result.is_targeted)
+        self.assertEqual(result.targets, ("guide/example.qmd",))
+        self.assertEqual(result.assets, ("guide/images/example.png",))
+
+    def test_global_quarto_change_requires_full_render(self):
+        result = select([("M", ("site/_quarto.yml",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_shared_metadata_requires_full_render(self):
+        result = select([("M", ("site/releases/_metadata.yml",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_deleted_page_requires_full_render(self):
+        result = select([("D", ("site/guide/old.qmd",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_non_site_change_requires_full_render(self):
+        result = select([("M", (".github/workflows/example.yaml",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_asset_only_change_requires_full_render(self):
+        result = select([("M", ("site/guide/images/example.png",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_generated_corpus_change_requires_full_render(self):
+        result = select([("M", ("site/llm/AGENTS.md",))])
+
+        self.assertFalse(result.is_targeted)
+
+    def test_parses_renames_for_safe_fallback(self):
+        changes = parse_changed_files(
+            "renamed\tsite/guide/new.qmd\tsite/guide/old.qmd\n"
+        )
+
+        self.assertEqual(
+            changes,
+            [("R", ("site/guide/new.qmd", "site/guide/old.qmd"))],
+        )
+        self.assertFalse(select(changes).is_targeted)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/deploy-docs-prod.yaml

@@ -37,21 +37,31 @@ jobs:
         run: |
           set -euo pipefail
           name="docs-production-$(git rev-parse 'HEAD^{tree}')"
-          run_id=$(gh api "repos/${{ github.repository }}/actions/artifacts?name=$name&per_page=100" \
-            --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | reverse | .[0].workflow_run.id // empty')
-
           echo "name=$name" >> "$GITHUB_OUTPUT"
-          if [[ -n "$run_id" ]]; then
-            echo "Found $name in workflow run $run_id"
-            echo "found=true" >> "$GITHUB_OUTPUT"
-            echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
-          else
-            echo "No matching artifact found; falling back to a full production build."
-            echo "found=false" >> "$GITHUB_OUTPUT"
+
+          run_id=""
+          while read -r candidate; do
+            [[ -z "$candidate" ]] && continue
+            run=$(gh api "repos/${{ github.repository }}/actions/runs/$candidate")
+            run_path=$(jq -r .path <<< "$run")
+            conclusion=$(jq -r .conclusion <<< "$run")
+            if [[ "$run_path" == ".github/workflows/deploy-docs-staging.yaml" && "$conclusion" == "success" ]]; then
+              run_id="$candidate"
+              break
+            fi
+            echo "Ignoring $name from untrusted or unsuccessful workflow run $candidate ($run_path: $conclusion)"
+          done < <(gh api "repos/${{ github.repository }}/actions/artifacts?name=$name&per_page=100" \
+            --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | reverse | .[].workflow_run.id')
+
+          if [[ -z "$run_id" ]]; then
+            echo "::error::No fully validated production artifact named $name was produced by a successful staging workflow. Production was not modified."
+            exit 1
           fi
 
+          echo "Found validated $name in staging workflow run $run_id"
+          echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
+
       - name: Download prebuilt production docs
-        if: steps.production-artifact.outputs.found == 'true'
         uses: actions/download-artifact@v4
         with:
           name: ${{ steps.production-artifact.outputs.name }}
@@ -61,35 +71,15 @@ jobs:
           run-id: ${{ steps.production-artifact.outputs.run_id }}
 
       - name: Extract prebuilt production docs
-        if: steps.production-artifact.outputs.found == 'true'
         run: |
           mkdir -p site/_site
           tar --zstd -xf "$RUNNER_TEMP/production-artifact/docs-production.tar.zst" -C site/_site
 
-      # Reclaim space only when the prebuilt artifact is unavailable and the
-      # workflow must perform the original full-site build.
-      - name: Free space + create reserve
-        if: steps.production-artifact.outputs.found != 'true'
-        uses: ./.github/actions/free-disk-space
-        with:
-          remove_dotnet: "true"
-          remove_android: "true"
-          remove_haskell: "true"
-          prune_docker: "true"
-          apt_cleanup: "true"
-          create_reserve_gb: "3"
-
-      - name: Build production docs site
-        if: steps.production-artifact.outputs.found != 'true'
-        uses: ./.github/actions/build-docs-site
-        with:
-          profile: production
-          docs_ci_ro_pat: ${{ secrets.DOCS_CI_RO_PAT }}
-          quarto_version: ${{ vars.QUARTO_VERSION }}
-          library_ref: main
-          installation_ref: main
-          release_notes_ref: main
-          backend_ref: main
+      - name: Verify production artifact contents
+        run: |
+          test -s site/_site/index.html
+          test -s site/_site/search.json
+          test -s site/_site/listings.json
 
       # Prod bucket is in us-east-1
       - name: Configure AWS credentials

.github/workflows/deploy-docs-staging.yaml

@@ -73,6 +73,10 @@ jobs:
           git switch --detach origin/prod
           git merge --no-commit --no-ff "$source_sha"
 
+      - name: Verify copyright headers
+        if: matrix.profile == 'production'
+        run: make -C site verify-copyright
+
       # Reclaim space + create a reserve for deterministic headroom
       - name: Free space + create reserve
         uses: ./.github/actions/free-disk-space
@@ -95,6 +99,49 @@ jobs:
           release_notes_ref: ${{ needs.resolve-sources.outputs.release_notes }}
           backend_ref: ${{ needs.resolve-sources.outputs.backend }}
 
+      - name: Test production render for warnings or errors
+        if: matrix.profile == 'production'
+        run: |
+          if grep -q 'WARN\|WARNING\|ERROR:' site/render_errors.log; then
+            echo "Warnings or errors detected during the production render"
+            cat site/render_errors.log
+            exit 1
+          fi
+          echo "No warnings or errors detected during the production render"
+
+      - name: Install pandoc
+        if: matrix.profile == 'production'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pandoc
+
+      - name: Verify chatbot product map is up to date
+        if: matrix.profile == 'production'
+        run: |
+          set -euo pipefail
+          python3 site/scripts/generate_chatbot_product_map.py
+          git diff --exit-code -- \
+            site/llm/chatbot-product-map.md \
+            site/llm/chatbot-product-map-frontend-snapshot.json
+
+      - name: Test chatbot product map generator
+        if: matrix.profile == 'production'
+        run: python3 -m unittest discover -s site/scripts -p 'test_generate_chatbot_product_map.py' -v
+
+      - name: Validate LLM markdown render
+        if: matrix.profile == 'production'
+        run: bash llm/render.sh && bash llm/clean.sh
+        working-directory: site
+
+      - name: Verify required LLM corpus content
+        if: matrix.profile == 'production'
+        run: |
+          test -f site/llm/_llm-output/chatbot-product-map.md
+          test -f site/llm/_llm-output/AGENTS.md
+          test -f site/llm/_llm-output/about/using-the-documentation.md
+          test ! -f site/llm/_llm-output/about/contributing/validmind-community.md
+          test ! -d site/llm/_llm-output/about/contributing/style-guide
+
       - name: Add robots.txt for staging
         if: matrix.profile == 'staging'
         run: cp site/environments/robots-staging.txt site/_site/robots.txt
@@ -140,6 +187,7 @@ jobs:
             site/_source/backend
             site/render_errors.log
             site/_freeze
+            site/llm/_llm-output
             dev.env
             valid.env
 

.github/workflows/full-docs-validation.yaml

@@ -1,14 +1,12 @@
 name: Full docs validation
 
 on:
-  merge_group:
-    types: [checks_requested]
   pull_request:
     types: [labeled]
   workflow_dispatch:
 
 concurrency:
-  group: full-docs-validation-${{ github.event.pull_request.number || github.event.merge_group.head_sha || github.ref }}
+  group: full-docs-validation-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'run' }}
   cancel-in-progress: true
 
 permissions:

.github/workflows/validate-docs-merge-group.yaml

@@ -0,0 +1,18 @@
+name: Validate docs merge group
+
+on:
+  merge_group:
+    types: [checks_requested]
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: validate
+    runs-on: ubuntu-latest
+    steps:
+      - name: Confirm queued revision
+        run: |
+          echo "The pull request revision passed its targeted preview validation."
+          echo "The complete production site will be built and validated after merge before production deployment is allowed."