From 659790e8f681e1c130fe58ef0943caf1f3ee6209 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 19 Oct 2026 04:10:40 +0700 Subject: [PATCH] feat(seo): add /llms.txt and markdown versions of blog posts and legal pages --- app/Http/Controllers/BlogController.php | 25 ++++ app/Http/Controllers/Controller.php | 13 +- app/Http/Controllers/LegalController.php | 34 +++++ app/Http/Controllers/LlmsTxtController.php | 124 ++++++++++++++++ app/Services/Blog/BlogService.php | 5 + app/Services/Legal/LegalDocuments.php | 41 ++++-- app/Support/Content/MarkdownExport.php | 138 ++++++++++++++++++ app/Support/Seo/BlogPosts.php | 1 + app/Support/Seo/LegalPages.php | 1 + app/Support/Seo/PageEntry.php | 9 ++ app/Support/Seo/SeoContext.php | 18 ++- app/Support/Seo/WithoutRetiredPaths.php | 1 + docs/architecture.md | 24 +++ lang/de/blog.php | 5 + lang/en/blog.php | 7 + lang/es/blog.php | 5 + lang/fr/blog.php | 5 + lang/id/blog.php | 5 + lang/it/blog.php | 5 + lang/ja/blog.php | 5 + lang/ko/blog.php | 5 + lang/pt-BR/blog.php | 5 + lang/vi/blog.php | 5 + lang/zh-Hans/blog.php | 5 + lang/zh-Hant/blog.php | 5 + resources/js/components/seo/seo-head.tsx | 4 +- resources/js/types/shared-props.ts | 2 + routes/localized.php | 11 ++ routes/web.php | 3 + tests/Feature/Seo/HreflangReciprocityTest.php | 4 +- tests/Feature/Seo/LlmsTxtTest.php | 115 +++++++++++++++ tests/Feature/Seo/MarkdownVersionsTest.php | 133 +++++++++++++++++ 32 files changed, 750 insertions(+), 18 deletions(-) create mode 100644 app/Http/Controllers/LlmsTxtController.php create mode 100644 app/Support/Content/MarkdownExport.php create mode 100644 lang/de/blog.php create mode 100644 lang/en/blog.php create mode 100644 lang/es/blog.php create mode 100644 lang/fr/blog.php create mode 100644 lang/id/blog.php create mode 100644 lang/it/blog.php create mode 100644 lang/ja/blog.php create mode 100644 lang/ko/blog.php create mode 100644 lang/pt-BR/blog.php create mode 100644 lang/vi/blog.php create mode 100644 lang/zh-Hans/blog.php create mode 100644 lang/zh-Hant/blog.php create mode 100644 tests/Feature/Seo/LlmsTxtTest.php create mode 100644 tests/Feature/Seo/MarkdownVersionsTest.php diff --git a/app/Http/Controllers/BlogController.php b/app/Http/Controllers/BlogController.php index cca89cc0..1410b6e3 100644 --- a/app/Http/Controllers/BlogController.php +++ b/app/Http/Controllers/BlogController.php @@ -9,6 +9,7 @@ use App\Services\Blog\PostRelease; use App\Services\Blog\PostTopics; use App\Support\Content\ContentRepository; +use App\Support\Content\MarkdownExport; use App\Support\Localization\Locales; use App\Support\Seo\BlogPosts; use App\Support\Seo\PageRegistry; @@ -95,6 +96,30 @@ public function show(ContentRepository $content, PostTopics $topics, PostRelease ]); } + public function markdown(MarkdownExport $export, ContentRepository $content, string $slug): HttpResponse + { + $locale = App::getLocale(); + $post = $this->blog->find($slug, $locale); + $entry = $this->registry->find(BlogPosts::ROUTE, ['slug' => $slug]); + + if ($post === null || $entry === null || ! $this->isPublished($post)) { + abort(404); + } + + $copy = $content->has('blog', $locale) ? $content->page('blog', $locale) : []; + $correction = $this->correction($copy, $post, $locale); + $notes = [implode(' · ', array_filter([$post->dateFormatted($locale), $post->author]))]; + + if ($correction !== null) { + $notes[] = $export->callout(trans('blog.correction', ['date' => $correction['dateFormatted']], $locale), $correction['text']); + } + + return $this->markdownResponse( + $export->document($post->title, $post->description, $notes, $export->body($this->blog->source($post), $locale)), + $entry->url($locale), + ); + } + /** * The default language's posts as Atom, cached at the edge like a page. */ diff --git a/app/Http/Controllers/Controller.php b/app/Http/Controllers/Controller.php index 8677cd5c..67c0840f 100644 --- a/app/Http/Controllers/Controller.php +++ b/app/Http/Controllers/Controller.php @@ -2,7 +2,18 @@ namespace App\Http\Controllers; +use App\Http\Middleware\CacheHtmlAtEdge; +use Illuminate\Http\Response; + abstract class Controller { - // + // Cached at the edge like its page, which the canonical names as the one to index. + protected function markdownResponse(string $markdown, string $canonical): Response + { + return response($markdown, 200, [ + 'Content-Type' => 'text/markdown; charset=utf-8', + 'Cache-Control' => CacheHtmlAtEdge::CACHE_CONTROL, + 'Link' => '<' . $canonical . '>; rel="canonical"', + ]); + } } diff --git a/app/Http/Controllers/LegalController.php b/app/Http/Controllers/LegalController.php index 6ca5fb0c..30d7fc81 100644 --- a/app/Http/Controllers/LegalController.php +++ b/app/Http/Controllers/LegalController.php @@ -5,6 +5,12 @@ use App\Services\Content\SiteFacts; use App\Services\Legal\LegalDocuments; use App\Support\Content\ContentRepository; +use App\Support\Content\MarkdownExport; +use App\Support\Localization\Locales; +use App\Support\Seo\LegalPages; +use App\Support\Seo\SeoContext; +use Illuminate\Http\Request; +use Illuminate\Http\Response as HttpResponse; use Illuminate\Support\Facades\App; use Inertia\Inertia; use Inertia\Response; @@ -45,6 +51,34 @@ public function brand(LegalDocuments $documents, ContentRepository $content, Sit return $this->document('Brand', 'brand', '/brand', $documents, $content, $facts); } + public function markdown(Request $request, SeoContext $seo, LegalDocuments $documents, ContentRepository $content, MarkdownExport $export): HttpResponse + { + $locale = App::getLocale(); + $entry = $seo->entryFor($request); + $name = $entry === null ? null : (LegalPages::DOCUMENTS[$entry->route] ?? null); + + if ($entry === null || $name === null) { + abort(404); + } + + $source = $documents->source($name, $locale); + $chrome = $content->page('legal', $locale); + $notes = [str_replace('{date}', $source['updatedAt']->locale($locale)->isoFormat('LL'), $chrome['updated'])]; + + if ($locale !== Locales::default()) { + $english = (string) $entry->markdownUrl(Locales::default()); + $notes[] = $export->callout( + $chrome['notice']['title'], + (string) preg_replace('#(.*?)#', '[$1](' . $english . ')', $chrome['notice']['body']), + ); + } + + return $this->markdownResponse( + $export->document($source['title'], $source['description'], $notes, $export->body($source['body'], $locale, LegalDocuments::MARKERS)), + $entry->url($locale), + ); + } + private function document(string $component, string $name, string $path, LegalDocuments $documents, ContentRepository $content, SiteFacts $facts): Response { $locale = App::getLocale(); diff --git a/app/Http/Controllers/LlmsTxtController.php b/app/Http/Controllers/LlmsTxtController.php new file mode 100644 index 00000000..260e50a5 --- /dev/null +++ b/app/Http/Controllers/LlmsTxtController.php @@ -0,0 +1,124 @@ + [ + 'landing.features.index', + 'landing.features.show', + 'landing.pricing', + 'landing.download', + 'landing.ios', + 'landing.integrations.index', + 'landing.faq', + 'landing.security', + 'landing.about', + ], + 'Databases' => ['landing.databases.index', 'landing.databaseClient'], + 'Comparisons' => ['landing.compare.index', 'landing.compare'], + ]; + + public function __construct( + private readonly PageRegistry $registry, + private readonly ContentRepository $content, + private readonly BlogService $blog, + private readonly LegalDocuments $legal, + private readonly SiteFacts $facts, + ) {} + + public function __invoke(): Response + { + $locale = Locales::default(); + $entries = array_values(array_filter($this->registry->all(), fn(PageEntry $entry): bool => $entry->isIndexable($locale))); + + $sections = ['Docs' => ['- [Documentation](' . rtrim((string) $this->facts->links()['docs'], '/') . '/llms.txt)']]; + + foreach (self::SECTIONS as $section => $routes) { + foreach ($routes as $route) { + foreach ($entries as $entry) { + if ($entry->route === $route) { + $sections[$section][] = $this->contentLink($entry, $locale); + } + } + } + } + + $sections['Guides'] = []; + $sections['Optional'] = []; + $posts = []; + + foreach ($entries as $entry) { + if ($entry->route === BlogPosts::ROUTE) { + $posts[$entry->params['slug']] = $entry; + } + } + + foreach ($this->blog->all($locale) as $post) { + if (isset($posts[$post->slug])) { + $section = $post->kind() === Post::KIND_GUIDE ? 'Guides' : 'Optional'; + $sections[$section][] = self::link($post->title, (string) $posts[$post->slug]->markdownUrl($locale), $post->description); + } + } + + foreach ($entries as $entry) { + $document = LegalPages::DOCUMENTS[$entry->route] ?? null; + + if ($document !== null) { + $source = $this->legal->source($document, $locale); + $sections['Optional'][] = self::link($source['title'], (string) $entry->markdownUrl($locale), $source['description']); + } + } + + $lines = ['# ' . config('app.name'), '', '> ' . $this->content->page('home', $locale)['seo']['description']]; + + foreach (array_filter($sections) as $section => $links) { + array_push($lines, '', '## ' . $section, '', ...$links); + } + + return response(implode("\n", $lines) . "\n", 200, [ + 'Content-Type' => 'text/plain; charset=utf-8', + 'Cache-Control' => CacheHtmlAtEdge::CACHE_CONTROL, + ]); + } + + private function contentLink(PageEntry $entry, string $locale): string + { + $name = StaticPages::PAGES[$entry->route] ?? ContentCollection::contentName($entry); + $copy = $this->content->page((string) $name, $locale); + $title = (string) $copy['seo']['title']; + + // A title with a `{slot}` is filled in the browser; the card title is the same name without it. + if (str_contains($title, '{')) { + $title = (string) $copy['og']['title']; + } + + return self::link($title, $entry->url($locale), (string) $copy['seo']['description']); + } + + private static function link(string $title, string $url, string $description): string + { + return '- [' . $title . '](' . $url . ')' . ($description === '' ? '' : ': ' . $description); + } +} diff --git a/app/Services/Blog/BlogService.php b/app/Services/Blog/BlogService.php index 66bd400c..ea684570 100644 --- a/app/Services/Blog/BlogService.php +++ b/app/Services/Blog/BlogService.php @@ -148,6 +148,11 @@ public function html(Post $post): string return $this->renderer->render($body, '', MarkdownRenderer::BLOG_ID_PREFIX); } + public function source(Post $post): string + { + return $this->bodies[$post->locale . '/' . $post->slug] ?? ''; + } + private function parse(string $slug, string $locale): ?Post { $path = $this->localeDirectory($locale) . '/' . $slug . '.md'; diff --git a/app/Services/Legal/LegalDocuments.php b/app/Services/Legal/LegalDocuments.php index 96f3d4ce..5b03ce75 100644 --- a/app/Services/Legal/LegalDocuments.php +++ b/app/Services/Legal/LegalDocuments.php @@ -49,6 +49,8 @@ class LegalDocuments public const BRAND_ASSETS_MARKER = ''; + public const MARKERS = [self::COOKIE_SETTINGS_MARKER, self::BRAND_ASSETS_MARKER]; + private const TOKEN = '/\{([a-zA-Z][a-zA-Z0-9]*)\}/'; public function __construct( @@ -79,6 +81,32 @@ public function has(string $document, string $locale): bool * @throws ContentMissingException */ public function render(string $document, string $locale, ?string $permalinkLabel = null): array + { + $source = $this->source($document, $locale); + $html = $this->renderer->render($source['body'], $permalinkLabel); + + foreach (self::MARKERS as $marker) { + $html = (string) preg_replace('#

\s*' . preg_quote($marker, '#') . '\s*

#', $marker, $html); + } + + return [ + 'title' => $source['title'], + 'description' => $source['description'], + 'updatedAt' => $source['updatedAt']->toDateString(), + 'updatedAtFormatted' => $source['updatedAt']->locale($locale)->isoFormat('LL'), + 'html' => $html, + 'toc' => $this->toc($html), + ]; + } + + /** + * One document in one locale as markdown, its `{token}` slots filled. + * + * @return array{title: string, description: string, updatedAt: CarbonImmutable, body: string} + * + * @throws ContentMissingException + */ + public function source(string $document, string $locale): array { $path = $this->path($document, $locale); @@ -87,21 +115,12 @@ public function render(string $document, string $locale, ?string $permalinkLabel } $parsed = YamlFrontMatter::parse((string) File::get($path)); - $updatedAt = $this->date($parsed->matter('updatedAt'), $path); - - $html = $this->renderer->render($this->fill($parsed->body(), $path, $locale), $permalinkLabel); - - foreach ([self::COOKIE_SETTINGS_MARKER, self::BRAND_ASSETS_MARKER] as $marker) { - $html = (string) preg_replace('#

\s*' . preg_quote($marker, '#') . '\s*

#', $marker, $html); - } return [ 'title' => $this->fill((string) $parsed->matter('title'), $path, $locale), 'description' => $this->fill((string) $parsed->matter('description'), $path, $locale), - 'updatedAt' => $updatedAt->toDateString(), - 'updatedAtFormatted' => $updatedAt->locale($locale)->isoFormat('LL'), - 'html' => $html, - 'toc' => $this->toc($html), + 'updatedAt' => $this->date($parsed->matter('updatedAt'), $path), + 'body' => $this->fill($parsed->body(), $path, $locale), ]; } diff --git a/app/Support/Content/MarkdownExport.php b/app/Support/Content/MarkdownExport.php new file mode 100644 index 00000000..650be685 --- /dev/null +++ b/app/Support/Content/MarkdownExport.php @@ -0,0 +1,138 @@ +<\/asset-slot>$/'; + + public function __construct( + private readonly AssetManifest $assets, + ) {} + + /** + * @param list $notes markdown blocks the page shows between its title and its body + */ + public function document(string $title, string $description, array $notes, string $body): string + { + $blocks = ['# ' . $title]; + + if ($description !== '') { + $blocks[] = '> ' . $description; + } + + return implode("\n\n", [...$blocks, ...$notes, trim($body)]) . "\n"; + } + + // A titled note, as the page's Callout shows it. + public function callout(string $title, string $text): string + { + return '> **' . $title . "**\n>\n> " . $text; + } + + /** + * @param list $markers lines the page replaces with a control, dropped here + */ + public function body(string $markdown, string $locale, array $markers = []): string + { + $code = $this->codeLines($markdown); + $lines = []; + $dropped = false; + + foreach (explode("\n", $markdown) as $index => $line) { + if (isset($code[$index + 1])) { + $lines[] = $line; + $dropped = false; + + continue; + } + + $trimmed = trim($line); + + if (preg_match(self::ASSET_SLOT, $trimmed, $slot) === 1) { + $line = $this->image($slot[1], $locale); + } elseif (in_array($trimmed, $markers, true)) { + $line = null; + } else { + $line = (string) preg_replace(self::HEADING_ID, '$1', $line); + } + + if ($line === null) { + $dropped = true; + + continue; + } + + // A dropped line leaves its blank lines on both sides; keep one. + if ($dropped && trim($line) === '' && $lines !== [] && trim(end($lines)) === '') { + continue; + } + + $lines[] = $line; + $dropped = $dropped && trim($line) === ''; + } + + return implode("\n", $lines); + } + + // What production renders for a slot: the supplied image as the page's + // `` loads it, or nothing for a placeholder. + private function image(string $id, string $locale): ?string + { + $sources = $this->assets->has($id) ? $this->assets->themedSources($id, $locale) : null; + + if ($sources === null) { + return null; + } + + [$themed, $fileLocale] = $sources; + $light = $themed['light']; + $format = $light['formats'][array_key_last($light['formats'])]; + $width = $light['widths'] === [] ? null : max($light['widths']); + $url = LocalizedUrl::base() . $this->assets->fileUrl($id, 'light', $fileLocale, $width, $format); + + $text = $this->assets->slotTextProjection($locale)[$id] ?? []; + $alt = is_array($text['alt'] ?? null) ? (string) reset($text['alt']) : ''; + $caption = is_array($text['caption'] ?? null) ? (string) reset($text['caption']) : ''; + + $image = '![' . str_replace(['[', ']'], ['\[', '\]'], $alt) . '](' . $url . ')'; + + return $caption === '' ? $image : $image . "\n\n" . $caption; + } + + /** + * @return array + */ + private function codeLines(string $markdown): array + { + $environment = new Environment(); + $environment->addExtension(new CommonMarkCoreExtension()); + + $lines = []; + + foreach ((new MarkdownParser($environment))->parse($markdown)->iterator(NodeIterator::FLAG_BLOCKS_ONLY) as $node) { + if (! $node instanceof FencedCode && ! $node instanceof IndentedCode) { + continue; + } + + for ($line = (int) $node->getStartLine(); $line <= (int) $node->getEndLine(); $line++) { + $lines[$line] = true; + } + } + + return $lines; + } +} diff --git a/app/Support/Seo/BlogPosts.php b/app/Support/Seo/BlogPosts.php index 2c143b62..b27b704b 100644 --- a/app/Support/Seo/BlogPosts.php +++ b/app/Support/Seo/BlogPosts.php @@ -90,6 +90,7 @@ public function find(string $route, array $params): ?PageEntry sources: $sources, ogFamily: 'blog', ogSlug: $slug, + markdown: true, ); } diff --git a/app/Support/Seo/LegalPages.php b/app/Support/Seo/LegalPages.php index 41ae3199..9ad07653 100644 --- a/app/Support/Seo/LegalPages.php +++ b/app/Support/Seo/LegalPages.php @@ -86,6 +86,7 @@ public function find(string $route, array $params): ?PageEntry sources: $sources, ogFamily: 'site', ogSlug: null, + markdown: true, ); } diff --git a/app/Support/Seo/PageEntry.php b/app/Support/Seo/PageEntry.php index 617cb068..14ef5815 100644 --- a/app/Support/Seo/PageEntry.php +++ b/app/Support/Seo/PageEntry.php @@ -16,6 +16,9 @@ */ final readonly class PageEntry { + // `landing.privacy.md` serves `/privacy.md`, the markdown of `landing.privacy`. + public const MARKDOWN = '.md'; + /** * @param string $route base route name, e.g. `landing.compare` * @param array $params @@ -32,8 +35,14 @@ public function __construct( public array $sources, public string $ogFamily, public ?string $ogSlug, + public bool $markdown = false, ) {} + public function markdownUrl(string $locale, bool $absolute = true): ?string + { + return $this->markdown ? LocalizedUrl::route($this->route . self::MARKDOWN, $this->params, $locale, $absolute) : null; + } + public function renders(string $locale): bool { return in_array($locale, $this->renderLocales, true); diff --git a/app/Support/Seo/SeoContext.php b/app/Support/Seo/SeoContext.php index fc82099d..3cfa3374 100644 --- a/app/Support/Seo/SeoContext.php +++ b/app/Support/Seo/SeoContext.php @@ -39,7 +39,7 @@ public function __construct( ) {} /** - * @return array{robots: string, canonical: string|null, alternates: list, xDefault: string|null, ogLocale: string, ogLocaleAlternates: list, ogImage: array{url: string, width: int, height: int, type: string, alt: string|null}|null} + * @return array{robots: string, canonical: string|null, alternates: list, xDefault: string|null, ogLocale: string, ogLocaleAlternates: list, ogImage: array{url: string, width: int, height: int, type: string, alt: string|null}|null, markdown: string|null} */ public function forRequest(Request $request): array { @@ -77,11 +77,13 @@ public function forRequest(Request $request): array 'ogLocale' => Locales::definition($locale)['og'], 'ogLocaleAlternates' => $ogAlternates, 'ogImage' => $this->ogImages->for($entry, $locale), + 'markdown' => $entry->markdownUrl($locale), ]; } /** - * The registry entry for the route the request matched, if it has one. + * The registry entry for the route the request matched, if it has one. A + * page's markdown route resolves to the page. */ public function entryFor(Request $request): ?PageEntry { @@ -93,10 +95,17 @@ public function entryFor(Request $request): ?PageEntry $route = $request->route(); if ($route instanceof Route && $route->getName() !== null) { + $name = (string) LocalizedUrl::baseName($route->getName()); + $markdown = str_ends_with($name, PageEntry::MARKDOWN); + $entry = $this->registry->find( - (string) LocalizedUrl::baseName($route->getName()), + $markdown ? substr($name, 0, -strlen(PageEntry::MARKDOWN)) : $name, $this->parameters($route), ); + + if ($markdown && $entry?->markdown !== true) { + $entry = null; + } } $request->attributes->set(self::ENTRY_ATTRIBUTE, $entry); @@ -110,7 +119,7 @@ public function isErrorPage(Request $request): bool } /** - * @return array{robots: string, canonical: null, alternates: list, xDefault: null, ogLocale: string, ogLocaleAlternates: list, ogImage: null} + * @return array{robots: string, canonical: null, alternates: list, xDefault: null, ogLocale: string, ogLocaleAlternates: list, ogImage: null, markdown: null} */ private function unlisted(string $locale): array { @@ -122,6 +131,7 @@ private function unlisted(string $locale): array 'ogLocale' => Locales::definition(Locales::isSupported($locale) ? $locale : Locales::default())['og'], 'ogLocaleAlternates' => [], 'ogImage' => null, + 'markdown' => null, ]; } diff --git a/app/Support/Seo/WithoutRetiredPaths.php b/app/Support/Seo/WithoutRetiredPaths.php index 2f805ad0..9cce62bb 100644 --- a/app/Support/Seo/WithoutRetiredPaths.php +++ b/app/Support/Seo/WithoutRetiredPaths.php @@ -73,6 +73,7 @@ private function withoutRetired(PageEntry $entry): ?PageEntry sources: $entry->sources, ogFamily: $entry->ogFamily, ogSlug: $entry->ogSlug, + markdown: $entry->markdown, ); } } diff --git a/docs/architecture.md b/docs/architecture.md index 90890534..5b1a6aba 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -16,9 +16,12 @@ database and no credentials: every page is built from markdown in /privacy /terms /refund-policy /brand English only: resources/data/legal/en/brand.md, files in public/images/brand /blog /blog/{slug} markdown in resources/blog +/blog/{slug}.md /privacy.md /terms.md /refund-policy.md /brand.md + the same page as markdown, in every language it renders in /blog/feed.xml the English posts as Atom, linked from the head of blog pages /vi/… every page above, in Vietnamese, where it exists /robots.txt /sitemap.xml +/llms.txt llmstxt.org, English, built from the page registry /.well-known/security.txt RFC 9116; `SecurityTxtController::EXPIRES` is moved by hand ``` @@ -166,6 +169,27 @@ the query string, so `/mariadb-client/?ref=x` goes straight to path is a 404, never a redirect to the homepage. A retired URL is never also a page: the registry drops every path the map answers. +### For AI agents + +- `/llms.txt` follows llmstxt.org: the H1, the homepage description as the + summary, then link lists. It links the docs site's own `llms.txt`, every + indexed English page with its meta description, guides apart from release + posts, and release posts and legal pages under `Optional`. +- Blog posts and legal pages, whose source is markdown, also answer at + `{page}.md`: the authored markdown with the title, the description and the + notes the page shows above the body (date, correction, last updated, + translation notice). `{#id}` heading attributes and controls are dropped, and + an image slot becomes the image production shows, or nothing. The route is + named `{page route}.md`, so the registry gates it as that page. +- A `.md` response carries `Link: ; rel="canonical"`, and the page's head + links it with `rel="alternate" type="text/markdown"` (the `seo.markdown` prop). +- No `Accept: text/markdown` negotiation: Cloudflare caches HTML and ignores + `Vary`, so one URL with two bodies would hand the cached one to both. Its + Markdown for Agents feature, which converts at the edge, needs a paid plan. +- `robots.txt` allows every crawler and carries no AI preference line. + Cloudflare's `Content-Signal` is a vendor policy and the IETF aipref + `Content-Usage` is still a draft; add `Content-Usage` once it is an RFC. + ## What it does not serve Buying a license, signing in to an account and subscribing to the newsletter are diff --git a/lang/de/blog.php b/lang/de/blog.php new file mode 100644 index 00000000..aeb2e53d --- /dev/null +++ b/lang/de/blog.php @@ -0,0 +1,5 @@ + 'Korrektur, :date', +]; diff --git a/lang/en/blog.php b/lang/en/blog.php new file mode 100644 index 00000000..d54cda7b --- /dev/null +++ b/lang/en/blog.php @@ -0,0 +1,7 @@ + 'Correction, :date', +]; diff --git a/lang/es/blog.php b/lang/es/blog.php new file mode 100644 index 00000000..56119ecd --- /dev/null +++ b/lang/es/blog.php @@ -0,0 +1,5 @@ + 'Corrección, :date', +]; diff --git a/lang/fr/blog.php b/lang/fr/blog.php new file mode 100644 index 00000000..fd6b59da --- /dev/null +++ b/lang/fr/blog.php @@ -0,0 +1,5 @@ + 'Correction, :date', +]; diff --git a/lang/id/blog.php b/lang/id/blog.php new file mode 100644 index 00000000..80289d9d --- /dev/null +++ b/lang/id/blog.php @@ -0,0 +1,5 @@ + 'Koreksi, :date', +]; diff --git a/lang/it/blog.php b/lang/it/blog.php new file mode 100644 index 00000000..2b1d058b --- /dev/null +++ b/lang/it/blog.php @@ -0,0 +1,5 @@ + 'Correzione, :date', +]; diff --git a/lang/ja/blog.php b/lang/ja/blog.php new file mode 100644 index 00000000..e9fe185b --- /dev/null +++ b/lang/ja/blog.php @@ -0,0 +1,5 @@ + '訂正：:date', +]; diff --git a/lang/ko/blog.php b/lang/ko/blog.php new file mode 100644 index 00000000..5631c364 --- /dev/null +++ b/lang/ko/blog.php @@ -0,0 +1,5 @@ + '정정, :date', +]; diff --git a/lang/pt-BR/blog.php b/lang/pt-BR/blog.php new file mode 100644 index 00000000..7615ac61 --- /dev/null +++ b/lang/pt-BR/blog.php @@ -0,0 +1,5 @@ + 'Correção, :date', +]; diff --git a/lang/vi/blog.php b/lang/vi/blog.php new file mode 100644 index 00000000..53eee98c --- /dev/null +++ b/lang/vi/blog.php @@ -0,0 +1,5 @@ + 'Đính chính ngày :date', +]; diff --git a/lang/zh-Hans/blog.php b/lang/zh-Hans/blog.php new file mode 100644 index 00000000..7570522c --- /dev/null +++ b/lang/zh-Hans/blog.php @@ -0,0 +1,5 @@ + '更正，:date', +]; diff --git a/lang/zh-Hant/blog.php b/lang/zh-Hant/blog.php new file mode 100644 index 00000000..7570522c --- /dev/null +++ b/lang/zh-Hant/blog.php @@ -0,0 +1,5 @@ + '更正，:date', +]; diff --git a/resources/js/components/seo/seo-head.tsx b/resources/js/components/seo/seo-head.tsx index b996ca03..b2af5e04 100644 --- a/resources/js/components/seo/seo-head.tsx +++ b/resources/js/components/seo/seo-head.tsx @@ -30,7 +30,8 @@ function escapeJsonLd(payload: object | object[]): string { * * In order: title and description; exactly one robots meta; the canonical when * the page is indexed in this locale; one alternate per real translation plus - * `x-default`; Open Graph and Twitter tags; JSON-LD with `} + {seo.markdown && } {publishedTime && ( diff --git a/resources/js/types/shared-props.ts b/resources/js/types/shared-props.ts index 2ebeb16c..5b43350d 100644 --- a/resources/js/types/shared-props.ts +++ b/resources/js/types/shared-props.ts @@ -18,6 +18,8 @@ export interface SeoProp { ogLocaleAlternates: string[]; /** `alt` says what the card shows, in the page's language. */ ogImage: { url: string; width: number; height: number; type: string; alt: string | null } | null; + /** The page as markdown (llmstxt.org), on blog posts and legal pages. */ + markdown: string | null; } /** One option of the language switcher. */ diff --git a/routes/localized.php b/routes/localized.php index 0e5baf82..866b9023 100644 --- a/routes/localized.php +++ b/routes/localized.php @@ -46,6 +46,9 @@ | `/` and `/vi` both render a section with id="pricing", because shipped Mac | builds open `/?ref=…#pricing`. | +| A route named `{page}.md` serves that page's markdown, and the registry +| gates it as that page (`PageEntry::MARKDOWN`). +| */ Route::get('/', HomeController::class)->name('landing.home'); @@ -61,10 +64,18 @@ Route::get('/refund-policy', [LegalController::class, 'refundPolicy'])->name('landing.refundPolicy'); Route::get('/brand', [LegalController::class, 'brand'])->name('landing.brand'); +Route::get('/privacy.md', [LegalController::class, 'markdown'])->name('landing.privacy.md'); +Route::get('/terms.md', [LegalController::class, 'markdown'])->name('landing.terms.md'); +Route::get('/refund-policy.md', [LegalController::class, 'markdown'])->name('landing.refundPolicy.md'); +Route::get('/brand.md', [LegalController::class, 'markdown'])->name('landing.brand.md'); + Route::get('/blog', [BlogController::class, 'index'])->name('landing.blog.index'); Route::get('/blog/{slug}', [BlogController::class, 'show']) ->where('slug', '[a-z0-9-]+') ->name('landing.blog.show'); +Route::get('/blog/{slug}.md', [BlogController::class, 'markdown']) + ->where('slug', '[a-z0-9-]+') + ->name('landing.blog.show.md'); Route::get('/features', [FeatureController::class, 'index'])->name('landing.features.index'); Route::get('/features/{slug}', [FeatureController::class, 'show']) diff --git a/routes/web.php b/routes/web.php index e8c99daf..403f7e5d 100644 --- a/routes/web.php +++ b/routes/web.php @@ -1,5 +1,6 @@ name('web.robots'); Route::get(SecurityTxtController::PATH, SecurityTxtController::class)->name('web.security'); + +Route::get(LlmsTxtController::PATH, LlmsTxtController::class)->name('web.llms'); diff --git a/tests/Feature/Seo/HreflangReciprocityTest.php b/tests/Feature/Seo/HreflangReciprocityTest.php index c12ae23a..396e471a 100644 --- a/tests/Feature/Seo/HreflangReciprocityTest.php +++ b/tests/Feature/Seo/HreflangReciprocityTest.php @@ -16,7 +16,7 @@ }); /** - * @return array{robots: string, canonical: string|null, alternates: list, xDefault: string|null, ogLocale: string, ogLocaleAlternates: list} + * @return array{robots: string, canonical: string|null, alternates: list, xDefault: string|null, ogLocale: string, ogLocaleAlternates: list, markdown: string|null} */ function hreflangExpected(PageEntry $entry, string $locale): array { @@ -42,6 +42,7 @@ function hreflangExpected(PageEntry $entry, string $locale): array 'xDefault' => $paired ? $entry->url(Locales::default()) : null, 'ogLocale' => Locales::definition($locale)['og'], 'ogLocaleAlternates' => $ogAlternates, + 'markdown' => $entry->markdownUrl($locale), ]; } @@ -161,6 +162,7 @@ function hreflangScratchSite(array $dirs): void 'xDefault' => null, 'ogLocale' => 'vi_VN', 'ogLocaleAlternates' => [], + 'markdown' => null, ]); $switcher = app(LocaleSwitcher::class)->forRequest(seoMatchedRequest('/vi/blog', 'vi')); diff --git a/tests/Feature/Seo/LlmsTxtTest.php b/tests/Feature/Seo/LlmsTxtTest.php new file mode 100644 index 00000000..7a475999 --- /dev/null +++ b/tests/Feature/Seo/LlmsTxtTest.php @@ -0,0 +1,115 @@ + 'tablepro.app']); +}); + +/** + * @return array> + */ +function llmsTxtSections(string $body): array +{ + $sections = []; + $current = null; + + foreach (explode("\n", $body) as $line) { + if (str_starts_with($line, '## ')) { + $current = substr($line, 3); + $sections[$current] = []; + } elseif ($current !== null && $line !== '') { + $sections[$current][] = $line; + } + } + + return $sections; +} + +function llmsTxtUrl(string $line): string +{ + preg_match('/^- \[[^\]]+\]\(([^)]+)\)/', $line, $match); + + return $match[1] ?? ''; +} + +it('follows the llmstxt.org layout', function (): void { + $response = $this->get('/llms.txt')->assertOk(); + $body = (string) $response->getContent(); + $summary = app(ContentRepository::class)->page('home', 'en')['seo']['description']; + + expect($response->headers->get('Content-Type'))->toBe('text/plain; charset=utf-8') + ->and($response->headers->get('Cache-Control'))->toContain('s-maxage=600'); + + expect($body)->toStartWith("# TablePro\n\n> {$summary}\n\n## ") + ->toEndWith("\n") + ->not->toContain('{'); + + expect(array_keys(llmsTxtSections($body)))->toBe(['Docs', 'Product', 'Databases', 'Comparisons', 'Guides', 'Optional']); + + foreach (llmsTxtSections($body) as $section => $lines) { + foreach ($lines as $line) { + expect($line)->toMatch('/^- \[[^\]\n]+\]\(https:\/\/[^)\s]+\)(: \S.*)?$/', "{$section}: {$line}"); + } + } +}); + +it('points at the documentation index', function (): void { + $docs = rtrim((string) app(SiteFacts::class)->links()['docs'], '/'); + + expect(llmsTxtSections((string) $this->get('/llms.txt')->getContent())['Docs'])->toBe(["- [Documentation]({$docs}/llms.txt)"]); +}); + +it('lists every indexed English page once, with posts and legal pages as markdown', function (): void { + $sections = llmsTxtSections((string) $this->get('/llms.txt')->getContent()); + unset($sections['Docs']); + + $listed = array_map('llmsTxtUrl', array_merge(...array_values($sections))); + $skipped = ['landing.home', 'landing.blog.index', 'landing.integrations.show']; + $expected = []; + + foreach (app(PageRegistry::class)->all() as $entry) { + if ($entry->isIndexable('en') && ! in_array($entry->route, $skipped, true)) { + $expected[] = $entry->markdownUrl('en') ?? $entry->url('en'); + } + } + + sort($expected); + $sorted = $listed; + sort($sorted); + + expect($listed)->toBe(array_values(array_unique($listed))) + ->and($sorted)->toBe($expected); + +}); + +it('lists guides apart from release posts, newest first', function (): void { + $sections = llmsTxtSections((string) $this->get('/llms.txt')->getContent()); + $posts = app(BlogService::class)->all('en'); + + $guides = array_values(array_filter($posts, fn(Post $post): bool => $post->kind() === Post::KIND_GUIDE)); + $releases = array_values(array_filter($posts, fn(Post $post): bool => $post->kind() === Post::KIND_RELEASE)); + + expect($guides)->not->toBeEmpty() + ->and($releases)->not->toBeEmpty(); + + expect(array_map('llmsTxtUrl', $sections['Guides'])) + ->toBe(array_map(fn(Post $post): string => "https://tablepro.app/blog/{$post->slug}.md", $guides)); + + expect(array_slice(array_map('llmsTxtUrl', $sections['Optional']), 0, count($releases))) + ->toBe(array_map(fn(Post $post): string => "https://tablepro.app/blog/{$post->slug}.md", $releases)); +}); + +it('names a page by its title, and by its card title where the title has a slot', function (): void { + $body = (string) $this->get('/llms.txt')->getContent(); + $postgres = app(ContentRepository::class)->page('databases/postgresql-client', 'en'); + $pricing = app(ContentRepository::class)->page('pricing', 'en'); + + expect($postgres['seo']['title'])->toContain('{') + ->and($body)->toContain("- [{$postgres['og']['title']}](https://tablepro.app/postgresql-client): {$postgres['seo']['description']}\n") + ->and($body)->toContain("- [{$pricing['seo']['title']}](https://tablepro.app/pricing): {$pricing['seo']['description']}\n"); +}); diff --git a/tests/Feature/Seo/MarkdownVersionsTest.php b/tests/Feature/Seo/MarkdownVersionsTest.php new file mode 100644 index 00000000..113afdde --- /dev/null +++ b/tests/Feature/Seo/MarkdownVersionsTest.php @@ -0,0 +1,133 @@ + 'tablepro.app']); + Http::fake(['api.github.com/*' => Http::response([], 200)]); +}); + +it('serves a post as markdown, pointing back at its page', function (): void { + $response = $this->get('/blog/open-sqlite-file-mac.md')->assertOk(); + $body = (string) $response->getContent(); + + expect($response->headers->get('Content-Type'))->toBe('text/markdown; charset=utf-8') + ->and($response->headers->get('Link'))->toBe('; rel="canonical"') + ->and($response->headers->get('Cache-Control'))->toContain('s-maxage=600') + ->and($response->headers->getCookies())->toBe([]); + + expect($body)->toStartWith("# Open and query a SQLite file on a Mac\n\n> Open a SQLite file with sqlite3 or TablePro.") + ->toContain("\n\nOctober 8, 2026 · TablePro\n\n") + ->toContain("\n## Check that the file is SQLite\n") + ->toContain("```bash\nfile ~/Downloads/app.db\n```") + ->not->toContain('{#') + ->not->toContain('page('blog', 'en')['corrections']['tablepro-0-74']; + $body = (string) $this->get('/blog/tablepro-0-74.md')->assertOk()->getContent(); + + expect($body)->toContain("> **Correction, October 2, 2026**\n>\n> " . $correction['text']); +}); + +it('serves a translated legal page with its notice, its facts filled and its controls dropped', function (): void { + $response = $this->get('/vi/privacy.md')->assertOk(); + $body = (string) $response->getContent(); + + expect($response->headers->get('Link'))->toBe('; rel="canonical"'); + + expect($body)->toStartWith('# Chính sách quyền riêng tư') + ->toContain('Cập nhật lần cuối: ') + ->toContain("> **Bản dịch**\n>\n> ") + ->toContain('(https://tablepro.app/privacy.md)') + ->not->toContain('') + ->not->toContain('') + ->not->toMatch('/\{[a-zA-Z]+\}/'); + + expect((string) $this->get('/privacy.md')->assertOk()->getContent())->not->toContain('> **Translation**'); +}); + +it('answers 404 where the page does not render', function (string $path): void { + $this->get($path)->assertNotFound(); +})->with([ + 'an English-only post in Vietnamese' => ['/vi/blog/tablepro-0-74.md'], + 'the English-only brand page in Vietnamese' => ['/vi/brand.md'], + 'a post that does not exist' => ['/blog/no-such-post.md'], + 'a page with no markdown version' => ['/pricing.md'], +]); + +it('serves every markdown version in every locale its page renders in', function (): void { + $checked = 0; + + foreach (app(PageRegistry::class)->all() as $entry) { + foreach ($entry->markdown ? $entry->renderLocales : [] as $locale) { + $path = (string) $entry->markdownUrl($locale, false); + $response = $this->get($path); + + Assert::assertSame(200, $response->getStatusCode(), "{$path} does not answer"); + Assert::assertSame('<' . $entry->url($locale) . '>; rel="canonical"', $response->headers->get('Link'), "{$path} names the wrong page"); + Assert::assertStringStartsWith('# ', (string) $response->getContent(), "{$path} has no title"); + $checked++; + } + } + + expect($checked)->toBeGreaterThan(40); +}); + +it('links the markdown version from the page head, and only where there is one', function (): void { + $this->get('/blog/open-sqlite-file-mac') + ->assertInertia(fn(AssertableInertia $page) => $page->where('seo.markdown', 'https://tablepro.app/blog/open-sqlite-file-mac.md')); + + $this->get('/vi/terms') + ->assertInertia(fn(AssertableInertia $page) => $page->where('seo.markdown', 'https://tablepro.app/vi/terms.md')); + + $this->get('/pricing') + ->assertInertia(fn(AssertableInertia $page) => $page->where('seo.markdown', null)); +}); + +it('gives only blog posts and legal pages a markdown version', function (): void { + $routes = collect(app(PageRegistry::class)->all()) + ->filter(fn(PageEntry $entry): bool => $entry->markdown) + ->map(fn(PageEntry $entry): string => $entry->route) + ->unique() + ->sort() + ->values() + ->all(); + + expect($routes)->toBe(['landing.blog.show', 'landing.brand', 'landing.privacy', 'landing.refundPolicy', 'landing.terms']); +}); + +it('leaves code blocks as written', function (): void { + $markdown = "## Setup {#setup}\n\n```markdown\n## Kept {#kept}\n\n```\n\n \n"; + + expect(app(MarkdownExport::class)->body($markdown, 'en', [''])) + ->toBe("## Setup\n\n```markdown\n## Kept {#kept}\n\n```\n\n \n"); +}); + +it('renders a supplied image as the page loads it and drops a placeholder', function (): void { + $manifest = app(AssetManifest::class); + $placeholder = collect(array_keys($manifest->assets()))->first(fn(string $id): bool => $manifest->entry($id)['slot'] && ! $manifest->isSupplied($id)); + + expect($manifest->isSupplied('blog-tablepro-0-74-1'))->toBeTrue() + ->and($placeholder)->toBeString(); + + $body = app(MarkdownExport::class)->body( + "Before.\n\n\n\nMiddle.\n\n\n\nAfter.", + 'en', + ); + + expect($body)->toMatch('#^Before\.\n\n!\[[^\n]+\]\(https://tablepro\.app/images/blog/blog-tablepro-0-74-1-light-\d+\.webp\)\n\n(?:[^\n]+\n\n)?Middle\.\n\nAfter\.$#'); +}); + +it('advertises the markdown version in the head', function (): void { + expect(ssrHtml('/blog/open-sqlite-file-mac'))->toContain('and(ssrHtml('/pricing'))->not->toContain('text/markdown'); +})->group('ssr');