From 8af4994f1446c02cf97a5277a3c8a229a9ebb053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Noco=C5=84?= Date: Tue, 14 Jul 2026 09:56:02 +0200 Subject: [PATCH 1/4] Added "View as Markdown" buttons --- docs/css/custom.css | 72 ++++++++- docs/css/release-notes.css | 1 - .../api_reference/api_reference.md | 1 + .../api_reference/content_api.md | 1 + .../api_reference/recommendation_api.md | 1 + .../api_reference/tracking_api.md | 1 + .../personalization/api_reference/user_api.md | 1 + .../attribute_search_in_elasticsearch.md | 1 + .../personalization/enable_personalization.md | 1 + docs/personalization/how_it_works.md | 1 + ...importing_historical_user_tracking_data.md | 1 + .../integrate_recommendation_service.md | 1 + .../legacy_recommendation_api.md | 1 + docs/personalization/personalization.md | 1 + docs/personalization/personalization_guide.md | 1 + .../recommendation_integration.md | 1 + docs/personalization/tracking_integration.md | 1 + .../tracking_with_ibexa-tracker.md | 1 + mkdocs.yml | 3 + plugins.yml | 11 ++ requirements.txt | 1 + theme/assets/page-actions.js | 55 +++++++ theme/main.html | 48 +++++- theme/partials/header.html | 5 - theme/partials/source.html | 7 - update_llmstxt_config.py | 150 ++++++++++++++++++ 26 files changed, 354 insertions(+), 15 deletions(-) create mode 100644 theme/assets/page-actions.js delete mode 100644 theme/partials/source.html create mode 100644 update_llmstxt_config.py diff --git a/docs/css/custom.css b/docs/css/custom.css index fe9b85528a..0d7e34ea2e 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -158,7 +158,7 @@ body { } .md-typeset h1 { - margin: 0 0 1rem; + margin: 0 0 0.5rem; font-size: 24px; line-height: 34px; } @@ -649,3 +649,73 @@ div.path { [hidden] { display: none !important; } + +.page-actions { + display: none; + gap: 0; + margin: 12px 0 0 0; + padding-bottom: 6px; + border-bottom: 1px solid var(--ibexa-snow); +} + +.md-typeset .page-action-btn { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 4px 12px; + background: transparent; + border: none; + text-decoration: none; + color: var(--ibexa-dusk-black); + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: color 0.15s ease; + white-space: nowrap; + position: relative; +} + +.page-action-btn+.page-action-btn::before { + content: ''; + position: absolute; + left: 0; + top: 50%; + transform: translateY(-50%); + height: 16px; + width: 1px; + background: var(--ibexa-snow); +} + +.page-action-btn svg { + width: 14px; + height: 14px; + fill: currentColor; + flex-shrink: 0; +} + +.page-action-btn.external::after, +.page-action-btn[rel="nofollow"]::after { + display: none; +} + +@media (max-width: 768px) { + .page-actions { + gap: 0; + } +} + +.page-actions--visible { + display: flex; +} + +.page-action-btn.page-action-btn--success { + background: #d4edda; + border-color: #c3e6cb; + color: #155724; +} + +.page-action-btn.page-action-btn--info { + background: color-mix(in srgb, var(--turquoise-pearl) 25%, white); + border-color: color-mix(in srgb, var(--turquoise-pearl) 40%, white); + color: var(--sherpa-blue); +} diff --git a/docs/css/release-notes.css b/docs/css/release-notes.css index 8663f96563..bbf47f8821 100644 --- a/docs/css/release-notes.css +++ b/docs/css/release-notes.css @@ -2,7 +2,6 @@ display: flex; justify-content: space-between; align-items: center; - border-bottom: 1px solid #DCDDDE; padding: 8px 0; } .release-notes-header h1 { diff --git a/docs/personalization/api_reference/api_reference.md b/docs/personalization/api_reference/api_reference.md index 7910d07941..add35808d6 100644 --- a/docs/personalization/api_reference/api_reference.md +++ b/docs/personalization/api_reference/api_reference.md @@ -1,6 +1,7 @@ --- description: Explore Personalization API sets that let you manage item data, track events, combine tracking data with users and render recommendations. page_type: landing_page +exclude_from_llmstxt: true --- # Personalization API diff --git a/docs/personalization/api_reference/content_api.md b/docs/personalization/api_reference/content_api.md index a772bac0c0..9c77701375 100644 --- a/docs/personalization/api_reference/content_api.md +++ b/docs/personalization/api_reference/content_api.md @@ -1,5 +1,6 @@ --- description: Personalization server can use external information about the items. Use HTTP methods to create, update or get items from the data store. +exclude_from_llmstxt: true --- # Content API diff --git a/docs/personalization/api_reference/recommendation_api.md b/docs/personalization/api_reference/recommendation_api.md index 470c52134c..b51c794a9e 100644 --- a/docs/personalization/api_reference/recommendation_api.md +++ b/docs/personalization/api_reference/recommendation_api.md @@ -1,6 +1,7 @@ --- description: Use HTTP GET request method to render recommendations. month_change: false +exclude_from_llmstxt: true --- # Recommendation API diff --git a/docs/personalization/api_reference/tracking_api.md b/docs/personalization/api_reference/tracking_api.md index ba7f8f746e..e02bba7f54 100644 --- a/docs/personalization/api_reference/tracking_api.md +++ b/docs/personalization/api_reference/tracking_api.md @@ -1,5 +1,6 @@ --- description: Allows to track items based on an ID. It covers many content types with the same ID configured for tracking. +exclude_from_llmstxt: true --- # Tracking API diff --git a/docs/personalization/api_reference/user_api.md b/docs/personalization/api_reference/user_api.md index b6240fc651..9d7491689f 100644 --- a/docs/personalization/api_reference/user_api.md +++ b/docs/personalization/api_reference/user_api.md @@ -1,5 +1,6 @@ --- description: Use HTTP methods to correlate metadata with user data and combine users into clusters of certain type. +exclude_from_llmstxt: true --- # User API diff --git a/docs/personalization/attribute_search_in_elasticsearch.md b/docs/personalization/attribute_search_in_elasticsearch.md index 6c83b83cb1..34288232cd 100644 --- a/docs/personalization/attribute_search_in_elasticsearch.md +++ b/docs/personalization/attribute_search_in_elasticsearch.md @@ -1,5 +1,6 @@ --- description: Attribute search uses Elasticsearch database to display dynamically taken values in scenario and model previews. +exclude_from_llmstxt: true --- # Attribute search in Elasticsearch database diff --git a/docs/personalization/enable_personalization.md b/docs/personalization/enable_personalization.md index bafb0e8349..2bc77cfb57 100644 --- a/docs/personalization/enable_personalization.md +++ b/docs/personalization/enable_personalization.md @@ -1,6 +1,7 @@ --- description: Configure your project files to enable Personalization and set up items you want to track. month_change: false +exclude_from_llmstxt: true --- # Enable Personalization diff --git a/docs/personalization/how_it_works.md b/docs/personalization/how_it_works.md index ee28481dce..5038284a70 100644 --- a/docs/personalization/how_it_works.md +++ b/docs/personalization/how_it_works.md @@ -1,5 +1,6 @@ --- description: Integrate recommendation service into your website. +exclude_from_llmstxt: true --- # How Personalization works diff --git a/docs/personalization/importing_historical_user_tracking_data.md b/docs/personalization/importing_historical_user_tracking_data.md index fd655a8e59..c3e3921f95 100644 --- a/docs/personalization/importing_historical_user_tracking_data.md +++ b/docs/personalization/importing_historical_user_tracking_data.md @@ -1,5 +1,6 @@ --- description: Use historical user tracking data to build user profiles and generate better recommendations. +exclude_from_llmstxt: true --- # Importing historical user tracking data diff --git a/docs/personalization/integrate_recommendation_service.md b/docs/personalization/integrate_recommendation_service.md index 740a45ae7b..4b469d79ef 100644 --- a/docs/personalization/integrate_recommendation_service.md +++ b/docs/personalization/integrate_recommendation_service.md @@ -1,6 +1,7 @@ --- description: Integrate recommendation service into your website. month_change: false +exclude_from_llmstxt: true --- # Integrate recommendation service diff --git a/docs/personalization/legacy_recommendation_api.md b/docs/personalization/legacy_recommendation_api.md index 2fcb224277..459a15f33d 100644 --- a/docs/personalization/legacy_recommendation_api.md +++ b/docs/personalization/legacy_recommendation_api.md @@ -1,5 +1,6 @@ --- description: An old method of fetching recommendations from the system using recommendation requests. +exclude_from_llmstxt: true --- # Legacy Recommendation API diff --git a/docs/personalization/personalization.md b/docs/personalization/personalization.md index ab9309cb41..27b174f6b8 100644 --- a/docs/personalization/personalization.md +++ b/docs/personalization/personalization.md @@ -1,6 +1,7 @@ --- description: Personalization tracks consumed content and suggests targeted content to your website visitors. page_type: landing_page +exclude_from_llmstxt: true --- # Personalization diff --git a/docs/personalization/personalization_guide.md b/docs/personalization/personalization_guide.md index 583de4f143..cd5e03a868 100644 --- a/docs/personalization/personalization_guide.md +++ b/docs/personalization/personalization_guide.md @@ -1,5 +1,6 @@ --- description: Discover Personalization - a cloud-based service that tracks and analyzes customer behaviors. +exclude_from_llmstxt: true --- # Personalization product guide diff --git a/docs/personalization/recommendation_integration.md b/docs/personalization/recommendation_integration.md index 769f3f1671..d53c930419 100644 --- a/docs/personalization/recommendation_integration.md +++ b/docs/personalization/recommendation_integration.md @@ -1,5 +1,6 @@ --- description: Methods for REST call with Personalization server. +exclude_from_llmstxt: true --- # Recommendation integration diff --git a/docs/personalization/tracking_integration.md b/docs/personalization/tracking_integration.md index 3dab342b8f..139e110f7d 100644 --- a/docs/personalization/tracking_integration.md +++ b/docs/personalization/tracking_integration.md @@ -1,6 +1,7 @@ --- description: See the methods of event tracking integration using tracking from server or from client-side. month_change: false +exclude_from_llmstxt: true --- # Tracking integration diff --git a/docs/personalization/tracking_with_ibexa-tracker.md b/docs/personalization/tracking_with_ibexa-tracker.md index 8537528f69..0e1d9a81ec 100644 --- a/docs/personalization/tracking_with_ibexa-tracker.md +++ b/docs/personalization/tracking_with_ibexa-tracker.md @@ -1,6 +1,7 @@ --- description: Integrate tracking with a Google-style JavaScript. month_change: false +exclude_from_llmstxt: true --- # Track events with ibexa-tracker.js diff --git a/mkdocs.yml b/mkdocs.yml index 0dae097323..a74d76ae28 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1107,3 +1107,6 @@ markdown_extensions: custom_checkbox: true - pymdownx.tilde - pymdownx.details + +hooks: + - hooks.py diff --git a/plugins.yml b/plugins.yml index bb7c1d7725..d553812608 100644 --- a/plugins.yml +++ b/plugins.yml @@ -592,3 +592,14 @@ plugins: 'ai_actions/extend_ai_actions.md': 'ai/ai_actions/extend_ai_actions.md' 'cdp/cdp_activation/cdp_add_clientside_tracking.md': 'cdp/cdp_activation/cdp_add_tracking.md' + + + - llmstxt: + preprocess: llmstxt_preprocess.py + + autoclean: false + full_output: llms-full.txt + sections: + # Built automatically when running MkDocs build based on MkDocs nav configuration, don't change + Ibexa DXP developer documentation: + - index.md diff --git a/requirements.txt b/requirements.txt index c271f19622..eb13f097ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ mkdocs-macros-plugin==1.3.7 mkdocs-redirects==1.2.2 mkdocs-autolinks-plugin==0.7.1 Jinja2==3.1.6 +mkdocs-llmstxt diff --git a/theme/assets/page-actions.js b/theme/assets/page-actions.js new file mode 100644 index 0000000000..ea34914d80 --- /dev/null +++ b/theme/assets/page-actions.js @@ -0,0 +1,55 @@ +/** + * Page Actions JavaScript + * Handles functionality for page action buttons (Copy as Markdown, View as Markdown, Edit on GitHub) + */ + +async function copyPageForLLM() { + const mdPath = document.querySelector('meta[name="markdown-path"]').content; + + try { + const response = await fetch(mdPath); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + const markdownContent = await response.text(); + await navigator.clipboard.writeText(markdownContent); + showButtonFeedback('success', 'Copied!'); + } catch (error) { + console.error('Failed to copy content:', error); + window.open(mdPath, '_blank'); + showButtonFeedback('info', 'Opened in tab'); + } +} + +function showButtonFeedback(type, message) { + const button = document.querySelector('button[onclick="copyPageForLLM()"]'); + if (!button) return; + + const originalHTML = button.innerHTML; + button.innerHTML = `${message}`; + + if (type === 'success') { + button.classList.add('page-action-btn--success'); + } else if (type === 'info') { + button.classList.add('page-action-btn--info'); + } + + setTimeout(() => { + button.innerHTML = originalHTML; + button.classList.remove('page-action-btn--success', 'page-action-btn--info'); + }, 2000); +} + +document.addEventListener('DOMContentLoaded', function() { + const pageActions = document.getElementById('page-actions'); + const firstH1 = document.querySelector('.bootstrap-iso h1, h1'); + + if (pageActions && firstH1) { + // If h1 is inside a special header container (e.g. release-notes-header), + // insert page-actions after that container so it doesn't disrupt the header layout. + const headerContainer = firstH1.closest('.release-notes-header'); + const anchor = headerContainer || firstH1; + anchor.insertAdjacentElement('afterend', pageActions); + pageActions.classList.add('page-actions--visible'); + } +}); diff --git a/theme/main.html b/theme/main.html index 66ce7dc87d..00f47b7a81 100644 --- a/theme/main.html +++ b/theme/main.html @@ -13,6 +13,23 @@ + + {% if config.repo_url and page.edit_url %} + + + {% endif %} + + + + + + {% if page and not page.meta.exclude_from_llmstxt %} + {% set md_href = page.file.dest_uri | replace('.html', '.md') %} + + {% endif %} + + + {% endblock %} {% block site_nav %} {% if nav %} @@ -75,7 +92,36 @@ {% endif %} {% include "partials/eol_warning.html" %} - {{ page.content }} + +
+ {{ page.content }} + + + {% if config.repo_url and page.edit_url and not page.is_homepage and not page.meta.exclude_from_llmstxt %} + + {% endif %} +
+ {% include "partials/tags.html" %} {% endblock %} diff --git a/theme/partials/header.html b/theme/partials/header.html index ba5ca0931d..80a4345297 100644 --- a/theme/partials/header.html +++ b/theme/partials/header.html @@ -20,10 +20,5 @@ {% include ".icons/material/magnify.svg" %} {% include "partials/search.html" %} - {% if config.repo_url %} -
- {% include "partials/source.html" %} -
- {% endif %} diff --git a/theme/partials/source.html b/theme/partials/source.html deleted file mode 100644 index 0948754c07..0000000000 --- a/theme/partials/source.html +++ /dev/null @@ -1,7 +0,0 @@ -{% import "partials/language.html" as lang with context %} - -
- {% include ".icons/fontawesome/brands/github-alt.svg" %} -
- View on GitHub -
diff --git a/update_llmstxt_config.py b/update_llmstxt_config.py new file mode 100644 index 0000000000..8b6d35d4b3 --- /dev/null +++ b/update_llmstxt_config.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +""" +Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav structure. +This script converts the mkdocs navigation into a format suitable for the llmstxt plugin. + +Files can be excluded from the llmstxt output by adding the following to their YAML frontmatter: + + exclude_from_llmstxt: true + +Files without this property are included by default. +""" + +import re +import yaml +from pathlib import Path + + +def read_frontmatter(file_path): + """ + Parse YAML frontmatter from a markdown file. + Returns a dict of frontmatter values, or an empty dict if none is found. + Handles files that begin with HTML comments before the frontmatter block. + """ + try: + content = Path(file_path).read_text(encoding='utf-8') + except (OSError, UnicodeDecodeError): + return {} + + # Strip leading HTML comments (e.g. ) + content = re.sub(r'\A(\s*\s*)+', '', content, flags=re.DOTALL) + + if not content.startswith('---'): + return {} + + end = content.find('\n---', 3) + if end == -1: + return {} + + try: + return yaml.safe_load(content[3:end]) or {} + except yaml.YAMLError: + return {} + + +def is_excluded(file_path, docs_dir): + """ + Return True if the file has 'exclude_from_llmstxt: true' in its frontmatter. + file_path is relative to docs_dir (as written in mkdocs nav). + """ + full_path = Path(docs_dir) / file_path + fm = read_frontmatter(full_path) + return fm.get('exclude_from_llmstxt', False) is True + + +def convert_nav_to_llmstxt_sections(nav_list, docs_dir): + """ + Convert mkdocs nav list to llmstxt sections format. + Returns a dict mapping top-level section names to flat lists of file paths. + Files with 'exclude_from_llmstxt: true' in their frontmatter are skipped. + Sections that contain no remaining files are omitted. + """ + sections = {} + + def extract_files(item): + """Recursively extract included file paths from a nav item.""" + files = [] + if isinstance(item, str): + if not item.endswith('.html') and not is_excluded(item, docs_dir): + files.append(item) + elif isinstance(item, list): + for subitem in item: + files.extend(extract_files(subitem)) + elif isinstance(item, dict): + for value in item.values(): + if isinstance(value, str): + if not value.endswith('.html') and not is_excluded(value, docs_dir): + files.append(value) + elif isinstance(value, list): + for subitem in value: + files.extend(extract_files(subitem)) + return files + + for item in nav_list: + if isinstance(item, dict): + for section_name, section_content in item.items(): + files = extract_files({section_name: section_content}) + if files: + sections[section_name] = files + elif isinstance(item, str): + if not item.endswith('.html') and not is_excluded(item, docs_dir): + sections.setdefault('Ibexa Developer Documentation', []).append(item) + + return sections + + +def update_plugins_yml(plugins_path, mkdocs_path): + """ + Update the llmstxt plugin configuration in plugins.yml based on mkdocs.yml nav. + """ + with open(plugins_path, 'r') as f: + plugins_data = yaml.safe_load(f) + + with open(mkdocs_path, 'r') as f: + mkdocs_data = yaml.safe_load(f) + + # docs/ directory is resolved relative to mkdocs.yml location + docs_dir = Path(mkdocs_path).parent / mkdocs_data.get('docs_dir', 'docs') + + nav = mkdocs_data.get('nav', []) + new_sections = convert_nav_to_llmstxt_sections(nav, docs_dir) + + plugins_list = plugins_data.get('plugins', []) + for plugin in plugins_list: + if isinstance(plugin, dict) and 'llmstxt' in plugin: + plugin['llmstxt']['sections'] = new_sections + print(f"✓ Updated llmstxt plugin configuration") + print(f" Total sections: {len(new_sections)}") + break + else: + print("✗ llmstxt plugin not found in plugins.yml") + return False + + with open(plugins_path, 'w') as f: + yaml.dump(plugins_data, f, default_flow_style=False, sort_keys=False, + allow_unicode=True, width=120) + + print(f"✓ Updated {plugins_path}") + return True + + +if __name__ == '__main__': + script_dir = Path(__file__).parent + plugins_path = script_dir / 'plugins.yml' + mkdocs_path = script_dir / 'mkdocs.yml' + + if not plugins_path.exists(): + print(f"✗ plugins.yml not found at {plugins_path}") + exit(1) + + if not mkdocs_path.exists(): + print(f"✗ mkdocs.yml not found at {mkdocs_path}") + exit(1) + + print("Updating llmstxt configuration...") + print(f"Reading from: {mkdocs_path}") + print(f"Updating: {plugins_path}") + print() + + success = update_plugins_yml(plugins_path, mkdocs_path) + exit(0 if success else 1) From 6fde75e605cd6cd064aa5bcb5a1c30388af66612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Noco=C5=84?= Date: Tue, 14 Jul 2026 10:52:50 +0200 Subject: [PATCH 2/4] Review feedback + added tests --- .github/workflows/build.yaml | 19 + .gitignore | 2 +- docs/css/custom.css | 2 +- docs/css/navigation.css | 2 +- docs/css/release-notes.css | 3 +- docs/index.md | 4 +- .../embed_and_list_content/list_content.md | 2 +- hooks.py | 90 ++++ llmstxt_preprocess.py | 488 ++++++++++++++++++ mkdocs.yml | 3 + pytest.ini | 2 + requirements-dev.txt | 2 + tests/python/conftest.py | 5 + tests/python/test_edition_badges.py | 37 ++ tests/python/test_image_urls.py | 54 ++ tests/python/test_preprocess.py | 189 +++++++ tests/python/test_renumber.py | 140 +++++ update_llmstxt_config.py | 3 + 18 files changed, 1039 insertions(+), 8 deletions(-) create mode 100644 hooks.py create mode 100644 llmstxt_preprocess.py create mode 100644 pytest.ini create mode 100644 requirements-dev.txt create mode 100644 tests/python/conftest.py create mode 100644 tests/python/test_edition_badges.py create mode 100644 tests/python/test_image_urls.py create mode 100644 tests/python/test_preprocess.py create mode 100644 tests/python/test_renumber.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d7940c9922..beab4188ca 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -65,6 +65,25 @@ jobs: exit 1 fi + python-tests: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: "3.13" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + + - name: Run pytest + run: pytest + markdownlint: runs-on: ubuntu-latest if: github.event_name == 'pull_request' diff --git a/.gitignore b/.gitignore index c0ebcf0350..f30fd66606 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ /vendor/ **/.DS_Store __pycache__/* -hooks/__pycache__/* +**/__pycache__ /site/ **/.idea/ .php-cs-fixer.cache diff --git a/docs/css/custom.css b/docs/css/custom.css index 0d7e34ea2e..7645062503 100644 --- a/docs/css/custom.css +++ b/docs/css/custom.css @@ -653,7 +653,7 @@ div.path { .page-actions { display: none; gap: 0; - margin: 12px 0 0 0; + margin: 12px 0 2rem 0; padding-bottom: 6px; border-bottom: 1px solid var(--ibexa-snow); } diff --git a/docs/css/navigation.css b/docs/css/navigation.css index a62f53f0b4..d6698cfb0e 100644 --- a/docs/css/navigation.css +++ b/docs/css/navigation.css @@ -393,7 +393,7 @@ margin-left: 0; margin-top: 0; margin-bottom: 0; - padding-bottom: 2rem; + padding-bottom: 0.5rem; } ul.breadcrumbs li.breadcrumb-item { diff --git a/docs/css/release-notes.css b/docs/css/release-notes.css index bbf47f8821..a656953848 100644 --- a/docs/css/release-notes.css +++ b/docs/css/release-notes.css @@ -2,10 +2,9 @@ display: flex; justify-content: space-between; align-items: center; - padding: 8px 0; + padding: 0; } .release-notes-header h1 { - line-height: 1.5; margin: 0; } diff --git a/docs/index.md b/docs/index.md index 591499ccf1..05f88a5ecc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -82,7 +82,7 @@ Release notes
- The latest release +
@@ -97,7 +97,7 @@ Discover other LTS Updates
- LTS Update +
diff --git a/docs/templating/embed_and_list_content/list_content.md b/docs/templating/embed_and_list_content/list_content.md index 3b2344aa6b..cacd5aa84b 100644 --- a/docs/templating/embed_and_list_content/list_content.md +++ b/docs/templating/embed_and_list_content/list_content.md @@ -64,7 +64,7 @@ Then, in the content view configuration, add the configuration under `content_qu The `match` configuration matches both the content type and the identifier of the Content query field. -Finally, in the template `templates/themes//content_query/blog_posts.html.twig`, render all results of the query: ``` html+twig [[= include_file('code_samples/front/list_content/templates/themes/my_theme/full/blog_post.html.twig') =]] diff --git a/hooks.py b/hooks.py new file mode 100644 index 0000000000..7d1749e50f --- /dev/null +++ b/hooks.py @@ -0,0 +1,90 @@ +""" +MkDocs hooks for Ibexa developer documentation. + +- Keeps the llmstxt plugin's ``sections`` config in sync with the ``nav`` + defined in ``mkdocs.yml``. +- Post-processes the Markdown generated by the llmstxt plugin. + +All content transformations live in ``llmstxt_preprocess.py``; this module is +only MkDocs event glue. +""" + +from __future__ import annotations + +import sys +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +_here = Path(__file__).parent +if str(_here) not in sys.path: + sys.path.insert(0, str(_here)) + +from llmstxt_preprocess import ( + absolutize_image_urls, + editions_from_frontmatter, + inject_edition_badges, + renumber_ordered_lists, +) +from update_llmstxt_config import convert_nav_to_llmstxt_sections + +if TYPE_CHECKING: + from mkdocs.config.defaults import MkDocsConfig + from mkdocs.structure.pages import Page + + +def on_config(config: "MkDocsConfig") -> None: + """Populate llmstxt sections from nav before the build starts. + + The llmstxt plugin reads ``config.sections`` in its ``on_files`` event, + which fires after ``on_config``, so injecting here is the right place. + """ + nav = config.get("nav") + if not nav: + return + + llmstxt = config["plugins"].get("llmstxt") + if llmstxt is None: + return + + docs_dir = Path(config["docs_dir"]) + llmstxt.config.sections = convert_nav_to_llmstxt_sections(nav, docs_dir) + + +def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs) -> None: + """Reformat the Markdown content generated by the llmstxt plugin for this page. + + Hooks run after plugins for every event, so at this point the llmstxt + plugin has already generated Markdown and stored it in ``_md_pages``. + We reformat here so the plugin writes the corrected content in its own + ``on_post_build`` (which also runs before ours, for the same reason). + """ + llmstxt = config["plugins"].get("llmstxt") + if llmstxt is None: + return + + src_uri = page.file.src_uri + page_info = llmstxt._md_pages.get(src_uri) + if page_info is None: + return + + editions = editions_from_frontmatter(_read_frontmatter(page, config)) + content = inject_edition_badges(page_info.content, editions) + content = renumber_ordered_lists(content) + # Same base URL and page directory the plugin uses for making link hrefs absolute. + page_dir = PurePosixPath(page.file.dest_uri).parent.as_posix() + content = absolutize_image_urls(content, llmstxt._base_url, page_dir) + if content != page_info.content: + llmstxt._md_pages[src_uri] = page_info._replace(content=content) + + +def _read_frontmatter(page: "Page", config: "MkDocsConfig") -> dict: + """Read the page's YAML frontmatter from its source file.""" + src_path = Path(config["docs_dir"]) / page.file.src_path + try: + from mkdocs.utils import meta as mkdocs_meta + with open(src_path, encoding="utf-8") as f: + raw = f.read() + _, frontmatter = mkdocs_meta.get_data(raw) + return frontmatter + except Exception: + return {} diff --git a/llmstxt_preprocess.py b/llmstxt_preprocess.py new file mode 100644 index 0000000000..6aad25f4ea --- /dev/null +++ b/llmstxt_preprocess.py @@ -0,0 +1,488 @@ +""" +Markdown conversion logic for the llmstxt plugin. + +This module is the single place holding all content transformations used to +produce the Markdown version of the documentation: + +- ``preprocess(soup, output)`` — HTML-level transforms invoked by the + mkdocs-llmstxt plugin (configured in ``plugins.yml``) before the HTML is + converted to Markdown. +- Markdown post-processing helpers (``renumber_ordered_lists``, + ``inject_edition_badges``, ``editions_from_frontmatter``) applied by + ``hooks.py`` to the Markdown the plugin generated. + +Note: the plugin loads this file by path (``spec_from_file_location``) while +``hooks.py`` imports it as a regular module, so two module objects may coexist. +Keep this module stateless (constants and pure functions only). +""" + +import html as html_module +import re +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup as Soup, NavigableString + +PILL_CLASS_TO_EDITION = { + "pill--lts-update": "LTS Update", + "pill--experience": "Experience", + "pill--commerce": "Commerce", + "pill--headless": "Headless", + "pill--new-feature": "New feature", + "pill--first-release": "First release", +} + +FRONTMATTER_EDITION_DISPLAY = { + "lts-update": "LTS Update", + "experience": "Experience", + "commerce": "Commerce", + "headless": "Headless", +} + + +def preprocess(soup: Soup, output: str) -> None: + """ + Preprocess HTML to improve markdown conversion. + + Runs with autoclean disabled so we can control the order: + 1. Expand tabbed sets with labels before autoclean removes tabbed-labels. + 2. Run autoclean-equivalent cleanup. + 3. Replace inline edition badge spans with readable text. + 4. Remove release notes filter UI. + 5. Convert card macros to markdown lists. + + Note: frontmatter edition injection is handled in hooks.py on_page_content, + where page.file.src_path is available directly. + """ + _process_tabbed_sets(soup) + _autoclean(soup) + _process_inline_pills(soup) + _process_release_note_tags(soup) + _process_release_note_dates(soup) + _process_release_notes_filters(soup) + _process_cards(soup) + _process_info_tiles(soup) + _process_tables(soup) + _process_admonitions(soup) + + +# --------------------------------------------------------------------------- +# Tables +# --------------------------------------------------------------------------- + +def _process_tables(soup: Soup) -> None: + """Replace ✔ (U+2714) with 'Yes' in table cells for readability.""" + CHECK = "✔" + for cell in soup.find_all(["td", "th"]): + for text_node in cell.find_all(string=lambda s: CHECK in s): + text_node.replace_with(NavigableString(text_node.replace(CHECK, "Yes"))) + + +# --------------------------------------------------------------------------- +# Tabbed sets +# --------------------------------------------------------------------------- + +def _process_tabbed_sets(soup: Soup) -> None: + """Prepend each tab label as bold text before its content block.""" + for tabbed_set in soup.find_all("div", class_="tabbed-set"): + labels_div = tabbed_set.find("div", class_="tabbed-labels") + content_div = tabbed_set.find("div", class_="tabbed-content") + + if not labels_div or not content_div: + tabbed_set.unwrap() + continue + + labels = [label.get_text(strip=True) for label in labels_div.find_all("label")] + blocks = content_div.find_all("div", class_="tabbed-block", recursive=False) + + wrapper = soup.new_tag("div") + for i, block in enumerate(blocks): + if i < len(labels): + label_tag = soup.new_tag("p") + strong = soup.new_tag("strong") + strong.string = labels[i] + label_tag.append(strong) + wrapper.append(label_tag) + for child in list(block.children): + wrapper.append(child.extract()) + + tabbed_set.replace_with(wrapper) + + +# --------------------------------------------------------------------------- +# Autoclean equivalent (mirrors mkdocs-llmstxt autoclean, minus tabbed-labels) +# --------------------------------------------------------------------------- + +def _autoclean(soup: Soup) -> None: + """Replicate the plugin's autoclean so we can run it after tab processing.""" + + # Unwrap links that wrap an image (e.g. lightbox links) so the image + # itself decides its fate below. + for link in soup.find_all("a"): + img = link.find("img") + if img: + link.replace_with(img.extract()) + + # Keep images that have alt text so markdownify emits ![alt](src) + # (URLs are made absolute later by absolutize_image_urls); drop the rest, + # since an image without a description tells an AI agent nothing. + # Done in its own pass: mutating the tree from inside a find_all() predicate + # makes the traversal skip the elements that follow the mutated one. + for img in soup.find_all("img"): + alt = (img.get("alt") or "").strip() + if not alt: + img.decompose() + + def _should_remove(tag) -> bool: + if tag.name == "svg": + return True + classes = tag.get("class") or () + if tag.name == "a" and "headerlink" in classes: + return True + if "twemoji" in classes: + return True + # tabbed-labels are already consumed by _process_tabbed_sets, but + # handle any stragglers defensively. + if "tabbed-labels" in classes: + return True + return False + + for element in soup.find_all(_should_remove): + element.decompose() + + for element in soup.find_all("autoref"): + element.replace_with(NavigableString(element.get_text())) + + # Insert ", " between adjacent elements in table cells. + # html.parser silently drops
(invalid void-closing tag), so adjacent + # blocks have no separator and markdownify concatenates their backticks. + for td in soup.find_all(["td", "th"]): + children = list(td.children) + for i in range(len(children) - 1): + curr = children[i] + nxt = children[i + 1] + if getattr(curr, "name", None) == "code" and getattr(nxt, "name", None) == "code": + curr.insert_after(NavigableString(", ")) + + for element in soup.find_all("div", attrs={"class": "doc-md-description"}): + element.replace_with(NavigableString(element.get_text().strip())) + + for element in soup.find_all("span", attrs={"class": "doc-labels"}): + element.decompose() + + # Flatten line-numbered code blocks to a plain
, keeping the
+    # language-* class (emitted thanks to pygments_lang_class) on the 
,
+    # where markdownify's code_language_callback looks for it.
+    for element in soup.find_all("table", attrs={"class": "highlighttable"}):
+        code_elem = element.find("code")
+        if code_elem:
+            classes = code_elem.get("class") or ()
+            language = next((c for c in classes if c.startswith("language-")), "")
+            attr = f' class="{language}"' if language else ""
+            element.replace_with(
+                Soup(f"{html_module.escape(code_elem.get_text())}
", "html.parser") + ) + +# --------------------------------------------------------------------------- +# Inline edition badge spans (from snippet includes) +# --------------------------------------------------------------------------- + +def _process_inline_pills(soup: Soup) -> None: + """Replace inline edition pill spans with readable text, e.g. '(Experience)'.""" + for span in soup.find_all("span", class_="pill--inline"): + span_classes = span.get("class", []) + for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items(): + if pill_cls in span_classes: + span.replace_with(soup.new_string(f" ({edition_name})")) + break + + +def _process_release_note_tags(soup: Soup) -> None: + """Append edition labels from release-note__tags divs to their preceding heading. + + Release notes use a
block after each

+ containing empty
elements rendered via CSS. + This converts them to a readable parenthetical on the heading, e.g.: + ## Google Gemini connector v5.0.7 (Headless, Experience, LTS Update, New feature) + """ + for tags_div in soup.find_all("div", class_="release-note__tags"): + editions = [] + for pill_div in tags_div.find_all("div"): + classes = pill_div.get("class", []) + for pill_cls, name in PILL_CLASS_TO_EDITION.items(): + if pill_cls in classes: + editions.append(name) + break + + heading = tags_div.find_previous_sibling(["h1", "h2", "h3", "h4"]) + if heading and editions: + # Insert before the permalink anchor so it's part of the heading text + anchor = heading.find("a", class_="headerlink") + label = NavigableString(f" ({', '.join(editions)})") + if anchor: + anchor.insert_before(label) + else: + heading.append(label) + + tags_div.decompose() + + +def _process_release_note_dates(soup: Soup) -> None: + """Prefix release-note dates with 'Release date: ' so the bare date line stays understandable.""" + for date_div in soup.find_all("div", class_="release-note__date"): + date_text = date_div.get_text(strip=True) + if date_text: + date_div.string = f"Release date: {date_text}" + + +def _process_info_tiles(soup: Soup) -> None: + """Simplify info-tile links to a single clean link text. + + Info tiles have a 'Details' label div and a separate content div. + After SVG removal, markdownify produces broken multi-line link text. + Replace the whole content with just the meaningful text. + """ + for tile in soup.find_all("a", class_="info-tile"): + # The label div ("Details") can be discarded + label = tile.find("div", class_="info-tile__details") + if label: + label.decompose() + + # Flatten remaining content to plain text + text = tile.get_text(separator=" ", strip=True) + tile.clear() + tile.append(soup.new_string(text)) + + +# --------------------------------------------------------------------------- +# Admonitions +# --------------------------------------------------------------------------- + +def _process_admonitions(soup: Soup) -> None: + """Convert admonition divs to blockquotes with a bold 'Type: Title' heading. + + Input:
+

Recommended versions

+

Body text...

+
+ + Output:
+

Caution: Recommended versions

+

Body text...

+
+ """ + for admonition in soup.find_all("div", class_="admonition"): + classes = admonition.get("class", []) + admonition_type = next((c for c in classes if c != "admonition"), None) + + title_elem = admonition.find("p", class_="admonition-title") + title_text = title_elem.get_text(strip=True) if title_elem else "" + + blockquote = soup.new_tag("blockquote") + + # Bold title paragraph + title_p = soup.new_tag("p") + strong = soup.new_tag("strong") + prefix = f"{admonition_type.capitalize()}: " if admonition_type else "" + strong.string = f"{prefix}{title_text}" + title_p.append(strong) + blockquote.append(title_p) + + # Body: all children except the title + for child in list(admonition.children): + if child == title_elem: + continue + blockquote.append(child.extract()) + + admonition.replace_with(blockquote) + + +def _process_release_notes_filters(soup: Soup) -> None: + """Remove interactive release-notes filter UI elements.""" + for container in soup.find_all("div", class_="release-notes-filters"): + container.decompose() + + +# --------------------------------------------------------------------------- +# Card macros +# --------------------------------------------------------------------------- + +def _process_cards(soup: Soup) -> None: + """Convert card macro HTML structures into markdown-friendly lists with links.""" + for cards_div in soup.find_all("div", class_=lambda c: c and c.startswith("cards ")): + card_wrappers = cards_div.find_all("div", class_="card-wrapper") + + if not card_wrappers: + continue + + ul = soup.new_tag("ul") + + for card_wrapper in card_wrappers: + link = card_wrapper.find("a", class_="card") + if not link: + continue + + href = link.get("href", "") + if href.startswith("//"): + href = "https:" + href + + title_elem = link.find("p", class_="title") + description_elem = link.find("p", class_="description") + + if not title_elem: + continue + + title = title_elem.get_text(strip=True) + description = description_elem.get_text(strip=True) if description_elem else "" + + li = soup.new_tag("li") + link_tag = soup.new_tag("a", href=href) + link_tag.string = title + li.append(link_tag) + + if description: + li.append(soup.new_string(": ")) + li.append(soup.new_string(description)) + + ul.append(li) + + cards_div.replace_with(ul) + + +# --------------------------------------------------------------------------- +# Markdown post-processing (applied by hooks.py to the generated Markdown) +# --------------------------------------------------------------------------- + +def editions_from_frontmatter(frontmatter: dict) -> list: + """Map ``edition``/``editions`` frontmatter values to display names.""" + + def _to_list(value): + if isinstance(value, list): + return value + if isinstance(value, str): + return value.split() + return [] + + all_editions = _to_list(frontmatter.get("edition")) + _to_list(frontmatter.get("editions") or []) + return [FRONTMATTER_EDITION_DISPLAY.get(e, e) for e in all_editions if e] + + +def inject_edition_badges(content: str, editions: list) -> str: + """Insert an 'Editions: X, Y' line after the first h1 heading.""" + if not editions: + return content + + badge_line = "Editions: " + ", ".join(editions) + + lines = content.split("\n") + for i, line in enumerate(lines): + if line.startswith("# "): + lines.insert(i + 1, "") + lines.insert(i + 2, badge_line) + return "\n".join(lines) + + return badge_line + "\n\n" + content + + +# ![alt](url) or ![alt](url "title") +_IMAGE_RE = re.compile(r'!\[([^\]]*)\]\(([^)\s]+)((?:\s+"[^"]*")?)\)') + + +def absolutize_image_urls(content: str, base_url: str, page_dir: str) -> str: + """Rewrite relative image URLs in Markdown to absolute ones. + + The llmstxt plugin makes link hrefs absolute but not image srcs, so the + ``![alt](src)`` references generated by markdownify keep their + page-relative paths, which break in ``llms-full.txt``. This mirrors the + URL logic the plugin applies to links. + + ``base_url`` is the site base URL (trailing slash), ``page_dir`` the + page's directory relative to the site root (e.g. ``"cdp/cdp"``). + """ + + def _absolutize(match: re.Match) -> str: + alt, url, title = match.groups() + if not url.startswith(("/", "#")) and not urlparse(url).scheme: + relative_base = urljoin(base_url, page_dir + "/") if page_dir else base_url + url = urljoin(relative_base, url) + return f"![{alt}]({url}{title})" + + return _IMAGE_RE.sub(_absolutize, content) + + +_ORDERED_MARKER_RE = re.compile(r"^(\s*)(\d+)\. (.*)$") +_FENCE_OPEN_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") +# Width of an ordered-list marker ("1. ") — mdformat normalizes markers to +# "1. " and indents item content by exactly this much. +_MARKER_WIDTH = 3 + + +def renumber_ordered_lists(content: str) -> str: + """Replace repeated '1.' ordered-list markers with sequential numbers. + + The llmstxt plugin runs mdformat on the converted Markdown, which keeps + the first marker of each ordered list (the list's start number) and + rewrites all following markers to "1." (its default numbering style). + This restores sequential numbers while keeping track of nesting: + + - a stack of (marker indent, counter) pairs tracks open ordered lists; + the counter starts at the list's first marker number, so intentional + start values (
    , or literal "2\\." paragraphs in the + source) are preserved, + - blank lines and lines indented to an item's content column (marker + indent + 3) are item continuation and don't interrupt numbering, + - a less-indented line closes the lists it's not a continuation of, + - fenced code blocks are passed through untouched. + + Known limitations (acceptable for LLM-oriented output): markers of items + >= 10 are one character wider than the 3-space content indent, and lists + inside blockquotes keep their "1." markers. + """ + lines = content.split("\n") + result = [] + stack = [] # [marker_indent, counter] per open ordered list, outermost first + fence = None # (fence_char, fence_length) while inside a fenced code block + + def close_lists(indent: int) -> None: + # A line not indented to the content column of an open list closes it. + while stack and indent < stack[-1][0] + _MARKER_WIDTH: + stack.pop() + + for line in lines: + if fence is not None: + stripped = line.strip() + if stripped and set(stripped) == {fence[0]} and len(stripped) >= fence[1]: + fence = None + result.append(line) + continue + + fence_match = _FENCE_OPEN_RE.match(line) + if fence_match: + close_lists(len(fence_match.group(1))) + marker = fence_match.group(2) + fence = (marker[0], len(marker)) + result.append(line) + continue + + if not line.strip(): + # Blank lines separate loose-list items and item paragraphs; they + # never terminate a list on their own in mdformat output. + result.append(line) + continue + + marker_match = _ORDERED_MARKER_RE.match(line) + if marker_match: + indent = len(marker_match.group(1)) + while stack and stack[-1][0] > indent: + stack.pop() + if stack and stack[-1][0] == indent: + stack[-1][1] += 1 + else: + # A new list keeps its first marker number as the start. + stack.append([indent, int(marker_match.group(2))]) + result.append(f"{marker_match.group(1)}{stack[-1][1]}. {marker_match.group(3)}") + continue + + close_lists(len(line) - len(line.lstrip())) + result.append(line) + + return "\n".join(result) diff --git a/mkdocs.yml b/mkdocs.yml index a74d76ae28..1ad95f0365 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1091,6 +1091,9 @@ markdown_extensions: - pymdownx.highlight: guess_lang: false linenums: true + # Emit language-* classes so the Markdown conversion (llmstxt) can + # restore the language on fenced code blocks. + pygments_lang_class: true extend_pygments_lang: - name: php lang: php diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..5b197e276e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +testpaths = tests/python diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000000..1d592acc1d --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==8.* diff --git a/tests/python/conftest.py b/tests/python/conftest.py new file mode 100644 index 0000000000..b9018a2f6b --- /dev/null +++ b/tests/python/conftest.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +# Make repo-root modules (llmstxt_preprocess.py) importable. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) diff --git a/tests/python/test_edition_badges.py b/tests/python/test_edition_badges.py new file mode 100644 index 0000000000..5e39f2c98e --- /dev/null +++ b/tests/python/test_edition_badges.py @@ -0,0 +1,37 @@ +from llmstxt_preprocess import editions_from_frontmatter, inject_edition_badges + + +def test_inserted_after_first_h1(): + content = "# Title\n\nBody text." + assert inject_edition_badges(content, ["Commerce"]) == "# Title\n\nEditions: Commerce\n\nBody text." + + +def test_prepended_when_no_h1(): + content = "Body text." + assert inject_edition_badges(content, ["Commerce"]) == "Editions: Commerce\n\nBody text." + + +def test_no_editions_leaves_content_unchanged(): + content = "# Title\n\nBody text." + assert inject_edition_badges(content, []) == content + + +def test_frontmatter_edition_string(): + assert editions_from_frontmatter({"edition": "commerce experience"}) == ["Commerce", "Experience"] + + +def test_frontmatter_editions_list(): + assert editions_from_frontmatter({"editions": ["headless", "lts-update"]}) == ["Headless", "LTS Update"] + + +def test_frontmatter_edition_and_editions_merged(): + result = editions_from_frontmatter({"edition": "commerce", "editions": ["headless"]}) + assert result == ["Commerce", "Headless"] + + +def test_unknown_edition_passes_through(): + assert editions_from_frontmatter({"edition": "custom"}) == ["custom"] + + +def test_empty_frontmatter(): + assert editions_from_frontmatter({}) == [] diff --git a/tests/python/test_image_urls.py b/tests/python/test_image_urls.py new file mode 100644 index 0000000000..d81118c173 --- /dev/null +++ b/tests/python/test_image_urls.py @@ -0,0 +1,54 @@ +from llmstxt_preprocess import absolutize_image_urls + +BASE = "https://doc.ibexa.co/en/latest/" + + +def test_relative_src_resolved_against_page_dir(): + content = "![CDP panel](img/cdp.png)" + assert absolutize_image_urls(content, BASE, "cdp/cdp") == ( + "![CDP panel](https://doc.ibexa.co/en/latest/cdp/cdp/img/cdp.png)" + ) + + +def test_parent_relative_src(): + content = "![Diagram](../../img/flow.png)" + assert absolutize_image_urls(content, BASE, "cdp/cdp") == ( + "![Diagram](https://doc.ibexa.co/en/latest/img/flow.png)" + ) + + +def test_root_page_uses_base_url(): + content = "![Logo](img/logo.png)" + assert absolutize_image_urls(content, BASE, ".") == ( + "![Logo](https://doc.ibexa.co/en/latest/img/logo.png)" + ) + + +def test_already_absolute_url_untouched(): + content = "![External](https://example.com/x.png)" + assert absolutize_image_urls(content, BASE, "cdp/cdp") == content + + +def test_site_absolute_path_untouched(): + # Matches the plugin's behavior for link hrefs starting with "/". + content = "![Rooted](/assets/x.png)" + assert absolutize_image_urls(content, BASE, "cdp/cdp") == content + + +def test_title_preserved(): + content = '![Alt](img/x.png "A title")' + assert absolutize_image_urls(content, BASE, "a/b") == ( + '![Alt](https://doc.ibexa.co/en/latest/a/b/img/x.png "A title")' + ) + + +def test_multiple_images_and_surrounding_text(): + content = "Before ![One](a.png) middle ![Two](b.png) after" + result = absolutize_image_urls(content, BASE, "dir") + assert "![One](https://doc.ibexa.co/en/latest/dir/a.png)" in result + assert "![Two](https://doc.ibexa.co/en/latest/dir/b.png)" in result + + +def test_regular_links_untouched(): + content = "[A link](img/x.png) and text" + assert absolutize_image_urls(content, BASE, "dir") == content diff --git a/tests/python/test_preprocess.py b/tests/python/test_preprocess.py new file mode 100644 index 0000000000..cd3a57856c --- /dev/null +++ b/tests/python/test_preprocess.py @@ -0,0 +1,189 @@ +import mdformat +from bs4 import BeautifulSoup +from mkdocs_llmstxt._internal.plugin import _converter + +import llmstxt_preprocess +from llmstxt_preprocess import renumber_ordered_lists + + +def to_markdown(html: str) -> str: + """Replicate the mkdocs-llmstxt conversion pipeline for a HTML snippet. + + Uses the plugin's own MarkdownConverter instance so options (bullets, + heading style, code language callback) always match what ships. + """ + soup = BeautifulSoup(html, "html.parser") + llmstxt_preprocess.preprocess(soup, "") + return mdformat.text( + _converter.convert_soup(soup), + options={"wrap": "no"}, + extensions=("tables",), + ) + + +def preprocessed(html: str) -> BeautifulSoup: + soup = BeautifulSoup(html, "html.parser") + llmstxt_preprocess.preprocess(soup, "") + return soup + + +def test_checkmark_only_cell_becomes_yes(): + html = "
    FeatureIncluded
    Search
    " + result = to_markdown(html) + assert "✔" not in result + assert "| Yes" in result + + +def test_checkmark_mixed_with_other_nodes(): + # Regression: raised NameError (NavigableString not imported) before the fix. + html = "
    H
    option
    " + result = to_markdown(html) + assert "Yes `option`" in result + + +def test_release_note_date_gets_prefix(): + html = '

    Connector v5.0.7

    2026-04-20
    ' + assert "Release date: 2026-04-20" in to_markdown(html) + + +def test_release_note_tags_appended_to_heading(): + html = ( + '

    Connector v5.0.7

    ' + '
    ' + '
    ' + '
    ' + "
    " + ) + assert "## Connector v5.0.7 (Headless, Experience)" in to_markdown(html) + + +def test_admonition_becomes_blockquote(): + html = ( + '
    ' + '

    Recommended versions

    ' + "

    Body text.

    " + "
    " + ) + result = to_markdown(html) + assert "> **Caution: Recommended versions**" in result + assert "> Body text." in result + + +def test_tabbed_set_labels_become_bold_text(): + html = ( + '
    ' + '
    ' + '
    ' + '

    Content A

    ' + '

    Content B

    ' + "
    " + ) + result = to_markdown(html) + assert "**Tab A**\n\nContent A\n\n**Tab B**\n\nContent B" in result + + +def test_cards_become_link_list(): + html = ( + '" + ) + assert "- [Page title](https://example.com/page/): Page description" in to_markdown(html) + + +def test_inline_pill_becomes_parenthetical(): + html = '

    Feature is available.

    ' + assert "Feature (Experience) is available." in to_markdown(html) + + +def test_ol_start_attribute_preserved(): + #
      (a list interrupted by other content) keeps its numbering. + html = "
      1. a
      2. b

      note

      1. c
      2. d
      " + result = renumber_ordered_lists(to_markdown(html)) + assert "1. a\n2. b" in result + assert "3. c\n4. d" in result + + +def test_escaped_literal_numbers_keep_their_value(): + # 'N\.' paragraphs in the source render as plain text and must keep N. + html = "

      2. To expose a single catalog to all users:

      " + assert renumber_ordered_lists(to_markdown(html)).startswith("2. To expose") + + +def test_info_tile_flattened_to_link_text(): + html = ( + '' + '
      Details
      ' + "
      Meaningful text
      " + "
      " + ) + assert "[Meaningful text](page/)" in to_markdown(html) + + +def test_image_with_alt_kept_as_markdown_image(): + assert "![Request lifecycle](x.png)" in to_markdown('Request lifecycle') + # Images without a description tell an AI agent nothing — dropped. + assert to_markdown('').strip() == "" + + +def test_headerlink_removed(): + html = '

      Title

      ' + assert to_markdown(html).strip() == "## Title" + + +def test_headerlink_after_image_removed(): + # Regression: replacing the from inside the find_all() predicate made + # the traversal skip the following elements, leaving their headerlinks in + # (e.g. '## How it works[¶](#how-it-works "Permanent link")' on the CDP page). + html = ( + '

      Ibexa CDP control panel

      ' + '

      How it works' + '

      ' + ) + result = to_markdown(html) + assert "![Ibexa CDP control panel](cdp.png)" in result + assert "## How it works" in result + assert "Permanent link" not in result + assert "¶" not in result + + +def test_image_wrapping_link_unwrapped(): + # Lightbox-style links around images are unwrapped to a plain image. + html = 'Thumb

      Kept

      ' + result = to_markdown(html) + assert "![Thumb](small.png)" in result + assert "[![" not in result + assert "Kept" in result + + +def test_line_numbered_code_block_keeps_language(): + # Structure emitted by pymdownx.highlight with linenums + pygments_lang_class. + html = ( + '
      ' + '' + '
      1
      '
      +        'apt-get install composer\n'
      +        "
      " + ) + result = to_markdown(html) + assert "```bash\napt-get install composer\n```" in result + # Line numbers must not leak into the output. + assert "\n1\n" not in result + + +def test_line_numbered_code_block_without_language(): + html = ( + '
      ' + '' + "
      plain text\n
      " + ) + result = to_markdown(html) + assert "```\nplain text\n```" in result + + +def test_release_notes_filters_removed(): + html = '

      Kept

      ' + result = to_markdown(html) + assert "Filter" not in result + assert "Kept" in result diff --git a/tests/python/test_renumber.py b/tests/python/test_renumber.py new file mode 100644 index 0000000000..1c1629c6ca --- /dev/null +++ b/tests/python/test_renumber.py @@ -0,0 +1,140 @@ +from llmstxt_preprocess import renumber_ordered_lists + + +def test_flat_list(): + assert renumber_ordered_lists("1. one\n1. two\n1. three") == "1. one\n2. two\n3. three" + + +def test_nested_bullets_do_not_reset_numbering(): + # Regression: https://github.com/ibexa/documentation-developer/pull/3161 + # numbering restarted at 1 after the nested "- " bullets. + content = "\n".join([ + "1. Sets a variable with the desired Varnish version", + "1. Copies and customizes `parameters.vcl` file:", + " - sets `web` container as the backend host", + ' - adds "all IPs" CIDR notation to `debuggers` list', + " - on Varnish 7, enable logging of access control list matching", + "1. Sets main `varnish*.vcl` file to use", + "1. Copies the main VCL file", + "1. Sets the Varnish version to use", + "1. Adds the Varnish container", + "1. Sets Varnish as the HTTP cache server", + "1. Restarts the DDEV cluster", + ]) + result = renumber_ordered_lists(content) + markers = [line.split(".")[0] for line in result.split("\n") if not line.startswith(" ")] + assert markers == ["1", "2", "3", "4", "5", "6", "7", "8"] + # Nested bullets are untouched + assert " - sets `web` container as the backend host" in result + + +def test_nested_ordered_lists(): + content = "\n".join([ + "1. a", + " 1. a1", + " 1. a2", + "1. b", + " 1. b1", + ]) + expected = "\n".join([ + "1. a", + " 1. a1", + " 2. a2", + "2. b", + " 1. b1", + ]) + assert renumber_ordered_lists(content) == expected + + +def test_loose_list_keeps_numbering_across_blank_lines(): + content = "1. first\n\n1. second\n\n1. third" + assert renumber_ordered_lists(content) == "1. first\n\n2. second\n\n3. third" + + +def test_continuation_paragraph_keeps_numbering(): + content = "1. first\n\n continuation paragraph\n\n1. second" + assert renumber_ordered_lists(content) == "1. first\n\n continuation paragraph\n\n2. second" + + +def test_fence_inside_item_keeps_numbering_and_content(): + content = "\n".join([ + "1. first", + "", + " ```bash", + " 1. not a marker", + " echo hi", + " ```", + "", + "1. second", + ]) + result = renumber_ordered_lists(content) + assert " 1. not a marker" in result + assert result.endswith("2. second") + + +def test_top_level_fence_resets_list(): + content = "\n".join([ + "1. first", + "1. second", + "", + "```text", + "1. fake", + "```", + "", + "1. new list", + ]) + result = renumber_ordered_lists(content).split("\n") + assert result[0] == "1. first" + assert result[1] == "2. second" + assert result[4] == "1. fake" # fence content untouched + assert result[7] == "1. new list" + + +def test_top_level_paragraph_resets_list(): + content = "1. first\n1. second\n\nA paragraph.\n\n1. new list" + assert renumber_ordered_lists(content) == "1. first\n2. second\n\nA paragraph.\n\n1. new list" + + +def test_keeps_start_number_and_is_idempotent(): + # mdformat keeps the first marker (the list's start) and rewrites the + # rest to "1." — the start must survive renumbering. + assert renumber_ordered_lists("4. a\n1. b\n1. c") == "4. a\n5. b\n6. c" + assert renumber_ordered_lists("3. a\n7. b") == "3. a\n4. b" + content = "1. one\n1. two\n - bullet\n1. three" + once = renumber_ordered_lists(content) + assert renumber_ordered_lists(once) == once + assert renumber_ordered_lists("4. a\n5. b\n6. c") == "4. a\n5. b\n6. c" + + +def test_intentional_paragraph_numbers_kept(): + # Sources sometimes use escaped literal numbers ("1\.", "2\.") that end up + # as separate single-item lists in the converted Markdown — their numbers + # are intentional and must not be reset to 1. + content = "\n".join([ + "1. All products available for all users:", + "", + "```yaml", + "catalog: ~", + "```", + "", + "2. To expose a single catalog:", + "", + "```yaml", + "catalog: custom_catalog", + "```", + "", + "3. Specific catalog for the defined customer group", + ]) + assert renumber_ordered_lists(content) == content + + +def test_escaped_number_is_not_a_marker(): + content = "1\\. not a list" + assert renumber_ordered_lists(content) == content + + +def test_tilde_fence_and_unclosed_fence(): + content = "~~~\n1. fake\n~~~" + assert renumber_ordered_lists(content) == content + unclosed = "```\n1. fake\n1. still fake" + assert renumber_ordered_lists(unclosed) == unclosed diff --git a/update_llmstxt_config.py b/update_llmstxt_config.py index 8b6d35d4b3..b232392c38 100644 --- a/update_llmstxt_config.py +++ b/update_llmstxt_config.py @@ -7,6 +7,9 @@ exclude_from_llmstxt: true +This excludes the page from both llms.txt and llms-full.txt, skips generating its Markdown +version, and hides the page action buttons. There is no per-file control for excluding a page +from llms.txt only (both files are derived from the same `sections` config). Files without this property are included by default. """ From 9cddf6ec227bd943f9a4a8484bf82a7c15362277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Noco=C5=84?= Date: Tue, 14 Jul 2026 14:54:13 +0200 Subject: [PATCH 3/4] Before re-review --- hooks.py | 12 +++- llmstxt_preprocess.py | 85 ++++++++++++++++++++++++----- tests/python/test_edition_badges.py | 37 ------------- tests/python/test_page_metadata.py | 74 +++++++++++++++++++++++++ tests/python/test_preprocess.py | 27 +++++++-- tests/python/test_renumber.py | 2 - 6 files changed, 178 insertions(+), 59 deletions(-) delete mode 100644 tests/python/test_edition_badges.py create mode 100644 tests/python/test_page_metadata.py diff --git a/hooks.py b/hooks.py index 7d1749e50f..dfdbe4ba70 100644 --- a/hooks.py +++ b/hooks.py @@ -22,7 +22,8 @@ from llmstxt_preprocess import ( absolutize_image_urls, editions_from_frontmatter, - inject_edition_badges, + expand_macros, + inject_page_metadata, renumber_ordered_lists, ) from update_llmstxt_config import convert_nav_to_llmstxt_sections @@ -67,8 +68,13 @@ def on_page_content(html: str, *, page: "Page", config: "MkDocsConfig", **kwargs if page_info is None: return - editions = editions_from_frontmatter(_read_frontmatter(page, config)) - content = inject_edition_badges(page_info.content, editions) + frontmatter = _read_frontmatter(page, config) + editions = editions_from_frontmatter(frontmatter) + description = expand_macros(str(frontmatter.get("description") or ""), config.get("extra") or {}) + if "[[=" in description: + # Unresolved macros must not leak into the output. + description = "" + content = inject_page_metadata(page_info.content, description, editions) content = renumber_ordered_lists(content) # Same base URL and page directory the plugin uses for making link hrefs absolute. page_dir = PurePosixPath(page.file.dest_uri).parent.as_posix() diff --git a/llmstxt_preprocess.py b/llmstxt_preprocess.py index 6aad25f4ea..f0e3c7c6fb 100644 --- a/llmstxt_preprocess.py +++ b/llmstxt_preprocess.py @@ -186,14 +186,54 @@ def _should_remove(tag) -> bool: # Inline edition badge spans (from snippet includes) # --------------------------------------------------------------------------- +def _pill_edition(node) -> str: + """Return the edition name of an inline pill span, or '' if not one.""" + if getattr(node, "name", None) != "span": + return "" + classes = node.get("class") or [] + if "pill--inline" not in classes: + return "" + for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items(): + if pill_cls in classes: + return edition_name + return "" + + def _process_inline_pills(soup: Soup) -> None: - """Replace inline edition pill spans with readable text, e.g. '(Experience)'.""" + """Replace inline edition pill spans with readable text. + + Consecutive pills (possibly separated by whitespace) are merged into a + single parenthetical, e.g. ' (Experience, Commerce)' instead of + ' (Experience) (Commerce)'. + """ for span in soup.find_all("span", class_="pill--inline"): - span_classes = span.get("class", []) - for pill_cls, edition_name in PILL_CLASS_TO_EDITION.items(): - if pill_cls in span_classes: - span.replace_with(soup.new_string(f" ({edition_name})")) + if span.parent is None: # already consumed as part of a previous run + continue + edition = _pill_edition(span) + if not edition: + continue + + # Collect the run of pills that follow, skipping whitespace between them. + editions = [edition] + consumed = [] + node = span.next_sibling + pending_whitespace = [] + while node is not None: + if isinstance(node, NavigableString) and not node.strip(): + pending_whitespace.append(node) + node = node.next_sibling + continue + next_edition = _pill_edition(node) + if not next_edition: break + editions.append(next_edition) + consumed += pending_whitespace + [node] + pending_whitespace = [] + node = node.next_sibling + + for extra_node in consumed: + extra_node.extract() + span.replace_with(soup.new_string(f" ({', '.join(editions)})")) def _process_release_note_tags(soup: Soup) -> None: @@ -366,21 +406,40 @@ def _to_list(value): return [FRONTMATTER_EDITION_DISPLAY.get(e, e) for e in all_editions if e] -def inject_edition_badges(content: str, editions: list) -> str: - """Insert an 'Editions: X, Y' line after the first h1 heading.""" - if not editions: - return content +_MACRO_RE = re.compile(r"\[\[=\s*(\w+)\s*=\]\]") + + +def expand_macros(text: str, variables: dict) -> str: + """Expand simple ``[[= name =]]`` macro variables (mkdocs-macros syntax). + + Only plain scalar variables are substituted; unknown or complex macros are + left untouched so callers can detect and handle them. + """ - badge_line = "Editions: " + ", ".join(editions) + def _substitute(match: re.Match) -> str: + value = variables.get(match.group(1)) + return str(value) if isinstance(value, (str, int, float)) else match.group(0) + + return _MACRO_RE.sub(_substitute, text) + + +def inject_page_metadata(content: str, description: str = "", editions: list = ()) -> str: + """Insert the page description and an 'Editions: X, Y' line after the first h1 heading.""" + metadata_lines = [] + if description: + metadata_lines += ["", description] + if editions: + metadata_lines += ["", "Editions: " + ", ".join(editions)] + if not metadata_lines: + return content lines = content.split("\n") for i, line in enumerate(lines): if line.startswith("# "): - lines.insert(i + 1, "") - lines.insert(i + 2, badge_line) + lines[i + 1:i + 1] = metadata_lines return "\n".join(lines) - return badge_line + "\n\n" + content + return "\n".join(metadata_lines).lstrip("\n") + "\n\n" + content # ![alt](url) or ![alt](url "title") diff --git a/tests/python/test_edition_badges.py b/tests/python/test_edition_badges.py deleted file mode 100644 index 5e39f2c98e..0000000000 --- a/tests/python/test_edition_badges.py +++ /dev/null @@ -1,37 +0,0 @@ -from llmstxt_preprocess import editions_from_frontmatter, inject_edition_badges - - -def test_inserted_after_first_h1(): - content = "# Title\n\nBody text." - assert inject_edition_badges(content, ["Commerce"]) == "# Title\n\nEditions: Commerce\n\nBody text." - - -def test_prepended_when_no_h1(): - content = "Body text." - assert inject_edition_badges(content, ["Commerce"]) == "Editions: Commerce\n\nBody text." - - -def test_no_editions_leaves_content_unchanged(): - content = "# Title\n\nBody text." - assert inject_edition_badges(content, []) == content - - -def test_frontmatter_edition_string(): - assert editions_from_frontmatter({"edition": "commerce experience"}) == ["Commerce", "Experience"] - - -def test_frontmatter_editions_list(): - assert editions_from_frontmatter({"editions": ["headless", "lts-update"]}) == ["Headless", "LTS Update"] - - -def test_frontmatter_edition_and_editions_merged(): - result = editions_from_frontmatter({"edition": "commerce", "editions": ["headless"]}) - assert result == ["Commerce", "Headless"] - - -def test_unknown_edition_passes_through(): - assert editions_from_frontmatter({"edition": "custom"}) == ["custom"] - - -def test_empty_frontmatter(): - assert editions_from_frontmatter({}) == [] diff --git a/tests/python/test_page_metadata.py b/tests/python/test_page_metadata.py new file mode 100644 index 0000000000..5029316e42 --- /dev/null +++ b/tests/python/test_page_metadata.py @@ -0,0 +1,74 @@ +from llmstxt_preprocess import ( + editions_from_frontmatter, + expand_macros, + inject_page_metadata, +) + + +def test_editions_inserted_after_first_h1(): + content = "# Title\n\nBody text." + assert inject_page_metadata(content, editions=["Commerce"]) == ( + "# Title\n\nEditions: Commerce\n\nBody text." + ) + + +def test_description_inserted_after_first_h1(): + content = "# Title\n\nBody text." + assert inject_page_metadata(content, description="Configure the Storefront.") == ( + "# Title\n\nConfigure the Storefront.\n\nBody text." + ) + + +def test_description_comes_before_editions(): + content = "# Title\n\nBody text." + assert inject_page_metadata(content, description="A description.", editions=["Commerce"]) == ( + "# Title\n\nA description.\n\nEditions: Commerce\n\nBody text." + ) + + +def test_prepended_when_no_h1(): + content = "Body text." + assert inject_page_metadata(content, description="A description.", editions=["Commerce"]) == ( + "A description.\n\nEditions: Commerce\n\nBody text." + ) + + +def test_no_metadata_leaves_content_unchanged(): + content = "# Title\n\nBody text." + assert inject_page_metadata(content) == content + assert inject_page_metadata(content, description="", editions=[]) == content + + +def test_frontmatter_edition_string(): + assert editions_from_frontmatter({"edition": "commerce experience"}) == ["Commerce", "Experience"] + + +def test_frontmatter_editions_list(): + assert editions_from_frontmatter({"editions": ["headless", "lts-update"]}) == ["Headless", "LTS Update"] + + +def test_frontmatter_edition_and_editions_merged(): + result = editions_from_frontmatter({"edition": "commerce", "editions": ["headless"]}) + assert result == ["Commerce", "Headless"] + + +def test_unknown_edition_passes_through(): + assert editions_from_frontmatter({"edition": "custom"}) == ["custom"] + + +def test_empty_frontmatter(): + assert editions_from_frontmatter({}) == [] + + +def test_expand_macros_substitutes_scalars(): + variables = {"product_name": "Ibexa DXP", "product_name_cdp": "Ibexa CDP"} + assert expand_macros("Install [[= product_name_cdp =]] with [[= product_name =]].", variables) == ( + "Install Ibexa CDP with Ibexa DXP." + ) + + +def test_expand_macros_leaves_unknown_untouched(): + assert expand_macros("Uses [[= unknown_var =]].", {}) == "Uses [[= unknown_var =]]." + # Complex expressions are not simple variables — left for the caller to detect. + text = "See [[= include_file('x.md') =]]." + assert expand_macros(text, {"include_file": "nope"}) == text diff --git a/tests/python/test_preprocess.py b/tests/python/test_preprocess.py index cd3a57856c..d8ef5e96bf 100644 --- a/tests/python/test_preprocess.py +++ b/tests/python/test_preprocess.py @@ -35,7 +35,6 @@ def test_checkmark_only_cell_becomes_yes(): def test_checkmark_mixed_with_other_nodes(): - # Regression: raised NameError (NavigableString not imported) before the fix. html = "
      H
      option
      " result = to_markdown(html) assert "Yes `option`" in result @@ -97,6 +96,29 @@ def test_inline_pill_becomes_parenthetical(): assert "Feature (Experience) is available." in to_markdown(html) +def test_adjacent_inline_pills_merged(): + # Structure from update_from_5.0: pills separated by a space in a heading. + html = ( + '

      Database update ' + ' ' + '' + '

      ' + ) + result = to_markdown(html) + assert "### Database update (Experience, Commerce)" in result + assert ") (" not in result + + +def test_three_adjacent_inline_pills_merged(): + html = ( + "

      Feature" + ' ' + ' ' + ' is available.

      ' + ) + assert "Feature (Headless, Experience, Commerce) is available." in to_markdown(html) + + def test_ol_start_attribute_preserved(): #
        (a list interrupted by other content) keeps its numbering. html = "
        1. a
        2. b

        note

        1. c
        2. d
        " @@ -133,9 +155,6 @@ def test_headerlink_removed(): def test_headerlink_after_image_removed(): - # Regression: replacing the from inside the find_all() predicate made - # the traversal skip the following elements, leaving their headerlinks in - # (e.g. '## How it works[¶](#how-it-works "Permanent link")' on the CDP page). html = ( '

        Ibexa CDP control panel

        ' '

        How it works' diff --git a/tests/python/test_renumber.py b/tests/python/test_renumber.py index 1c1629c6ca..12e1a403fe 100644 --- a/tests/python/test_renumber.py +++ b/tests/python/test_renumber.py @@ -6,8 +6,6 @@ def test_flat_list(): def test_nested_bullets_do_not_reset_numbering(): - # Regression: https://github.com/ibexa/documentation-developer/pull/3161 - # numbering restarted at 1 after the nested "- " bullets. content = "\n".join([ "1. Sets a variable with the desired Varnish version", "1. Copies and customizes `parameters.vcl` file:", From c5820213843012027a0b71c87ccaf85914cc583d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Noco=C5=84?= Date: Wed, 15 Jul 2026 12:32:00 +0200 Subject: [PATCH 4/4] View on GitHub --- theme/main.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/theme/main.html b/theme/main.html index 00f47b7a81..cf88c5c920 100644 --- a/theme/main.html +++ b/theme/main.html @@ -112,11 +112,11 @@ View as Markdown - + - Edit on GitHub + View on GitHub

{% endif %}