From 6cc48d7b1d813fd56a1d3b853fb7f19ac9ddc13e Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 8 Jul 2026 00:42:00 +0200 Subject: [PATCH] feat: Generate llms.txt for AI agents A new Antora extension emits /llms.txt (https://llmstxt.org/) at build time. It indexes every page of the version that 'stable' points to, with the page description and a link to the AsciiDoc source on GitHub, and cross-links the machine-readable release/CRD data on the Stackable Hub. This gives AI agents a way to discover and navigate the docs without scraping HTML, and to locate the source file behind any page. --- antora-playbook.yml | 1 + local-antora-playbook.yml | 1 + only-dev-antora-playbook.yml | 1 + supplemental-ui/lib/llms-txt.js | 70 +++++++++++++++++++++++++++++++++ truly-local-playbook.yml | 1 + 5 files changed, 74 insertions(+) create mode 100644 supplemental-ui/lib/llms-txt.js diff --git a/antora-playbook.yml b/antora-playbook.yml index c39ba8a35..44f0ca8e6 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -31,6 +31,7 @@ antora: mermaid_library_url: https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs script_stem: mermaid-scripts - ./supplemental-ui/lib/stackable-operator-helpers.js + - ./supplemental-ui/lib/llms-txt.js content: sources: - url: . diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index adc786434..6994f2987 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -16,6 +16,7 @@ antora: mermaid_library_url: https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs script_stem: mermaid-scripts - ./supplemental-ui/lib/stackable-operator-helpers.js + - ./supplemental-ui/lib/llms-txt.js content: sources: - url: ./ diff --git a/only-dev-antora-playbook.yml b/only-dev-antora-playbook.yml index 81dc672d6..3bfd3e9c0 100644 --- a/only-dev-antora-playbook.yml +++ b/only-dev-antora-playbook.yml @@ -16,6 +16,7 @@ antora: mermaid_library_url: https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs script_stem: mermaid-scripts - ./supplemental-ui/lib/stackable-operator-helpers.js + - ./supplemental-ui/lib/llms-txt.js content: sources: - url: ./ diff --git a/supplemental-ui/lib/llms-txt.js b/supplemental-ui/lib/llms-txt.js new file mode 100644 index 000000000..75bbd4e10 --- /dev/null +++ b/supplemental-ui/lib/llms-txt.js @@ -0,0 +1,70 @@ +// Generates /llms.txt following https://llmstxt.org/ so that AI agents can +// discover the documentation without scraping HTML. +// +// For every component, only the version that 'stable' points to is indexed +// (the latest release, or nightly when building without release branches). +// Each entry carries the published page URL, the page description and a link +// to the AsciiDoc source on GitHub, so agents can both read a page and find +// the file to change. +// +// Useful links: +// Extensions: https://docs.antora.org/antora/latest/extend/extensions/ +// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/ +module.exports.register = function () { + this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => { + const siteUrl = (playbook.site.url || '').replace(/\/$/, '') + const lines = [ + '# Stackable Data Platform Documentation', + '', + '> Documentation for the Stackable Data Platform (SDP), a curated selection of', + '> open source data apps like Apache Kafka, Apache Druid, Apache Trino and', + '> Apache Spark, all working together seamlessly on Kubernetes.', + '', + 'Machine-readable release, component and CRD data is available from the', + 'Stackable Hub: https://hub.stackable.tech/llms.txt', + '', + 'Every entry below links the published page and, in parentheses, its AsciiDoc', + 'source on GitHub. To suggest a change to a page, edit that source file.', + ] + + for (const component of contentCatalog.getComponents()) { + const version = component.latest + const pages = contentCatalog + .findBy({ component: component.name, version: version.version, family: 'page' }) + .filter((page) => page.out) + .sort((a, b) => a.pub.url.localeCompare(b.pub.url)) + const byModule = new Map() + for (const page of pages) { + const module = page.src.module + if (!byModule.has(module)) byModule.set(module, []) + byModule.get(module).push(page) + } + for (const [module, modulePages] of byModule) { + lines.push('', `## ${component.title}: ${module === 'ROOT' ? 'general' : module}`, '') + for (const page of modulePages) { + const title = (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem) + .replace(/<[^>]+>/g, '') + const description = page.asciidoc && page.asciidoc.attributes['description'] + const source = sourceUrl(page.src) + let entry = `- [${title}](${siteUrl}${page.pub.url})` + if (description) entry += `: ${description.replace(/\s+/g, ' ').trim()}` + if (source) entry += ` (source: ${source})` + lines.push(entry) + } + } + } + + siteCatalog.addFile({ + contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), + out: { path: 'llms.txt' }, + }) + }) +} + +function sourceUrl (src) { + const origin = src.origin + if (!origin || !origin.url || !origin.url.startsWith('https://github.com/')) return undefined + const repo = origin.url.replace(/\.git$/, '') + const start = origin.startPath ? `${origin.startPath}/` : '' + return `${repo}/blob/${origin.refname}/${start}${src.path}` +} diff --git a/truly-local-playbook.yml b/truly-local-playbook.yml index 76c2729a3..55ece3052 100644 --- a/truly-local-playbook.yml +++ b/truly-local-playbook.yml @@ -16,6 +16,7 @@ antora: mermaid_library_url: https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs script_stem: mermaid-scripts - ./supplemental-ui/lib/stackable-operator-helpers.js + - ./supplemental-ui/lib/llms-txt.js content: sources: - url: ./