feat(seo): add /llms.txt and markdown versions of blog posts and legal pages

app/Http/Controllers/BlogController.php

@@ -9,6 +9,7 @@
 use App\Services\Blog\PostRelease;
 use App\Services\Blog\PostTopics;
 use App\Support\Content\ContentRepository;
+use App\Support\Content\MarkdownExport;
 use App\Support\Localization\Locales;
 use App\Support\Seo\BlogPosts;
 use App\Support\Seo\PageRegistry;
@@ -95,6 +96,30 @@ public function show(ContentRepository $content, PostTopics $topics, PostRelease
         ]);
     }
 
+    public function markdown(MarkdownExport $export, ContentRepository $content, string $slug): HttpResponse
+    {
+        $locale = App::getLocale();
+        $post = $this->blog->find($slug, $locale);
+        $entry = $this->registry->find(BlogPosts::ROUTE, ['slug' => $slug]);
+
+        if ($post === null || $entry === null || ! $this->isPublished($post)) {
+            abort(404);
+        }
+
+        $copy = $content->has('blog', $locale) ? $content->page('blog', $locale) : [];
+        $correction = $this->correction($copy, $post, $locale);
+        $notes = [implode(' · ', array_filter([$post->dateFormatted($locale), $post->author]))];
+
+        if ($correction !== null) {
+            $notes[] = $export->callout(trans('blog.correction', ['date' => $correction['dateFormatted']], $locale), $correction['text']);
+        }
+
+        return $this->markdownResponse(
+            $export->document($post->title, $post->description, $notes, $export->body($this->blog->source($post), $locale)),
+            $entry->url($locale),
+        );
+    }
+
     /**
      * The default language's posts as Atom, cached at the edge like a page.
      */

app/Http/Controllers/Controller.php

@@ -2,7 +2,18 @@
 
 namespace App\Http\Controllers;
 
+use App\Http\Middleware\CacheHtmlAtEdge;
+use Illuminate\Http\Response;
+
 abstract class Controller
 {
-    //
+    // Cached at the edge like its page, which the canonical names as the one to index.
+    protected function markdownResponse(string $markdown, string $canonical): Response
+    {
+        return response($markdown, 200, [
+            'Content-Type' => 'text/markdown; charset=utf-8',
+            'Cache-Control' => CacheHtmlAtEdge::CACHE_CONTROL,
+            'Link' => '<' . $canonical . '>; rel="canonical"',
+        ]);
+    }
 }

app/Http/Controllers/LegalController.php

@@ -5,6 +5,12 @@
 use App\Services\Content\SiteFacts;
 use App\Services\Legal\LegalDocuments;
 use App\Support\Content\ContentRepository;
+use App\Support\Content\MarkdownExport;
+use App\Support\Localization\Locales;
+use App\Support\Seo\LegalPages;
+use App\Support\Seo\SeoContext;
+use Illuminate\Http\Request;
+use Illuminate\Http\Response as HttpResponse;
 use Illuminate\Support\Facades\App;
 use Inertia\Inertia;
 use Inertia\Response;
@@ -45,6 +51,34 @@ public function brand(LegalDocuments $documents, ContentRepository $content, Sit
         return $this->document('Brand', 'brand', '/brand', $documents, $content, $facts);
     }
 
+    public function markdown(Request $request, SeoContext $seo, LegalDocuments $documents, ContentRepository $content, MarkdownExport $export): HttpResponse
+    {
+        $locale = App::getLocale();
+        $entry = $seo->entryFor($request);
+        $name = $entry === null ? null : (LegalPages::DOCUMENTS[$entry->route] ?? null);
+
+        if ($entry === null || $name === null) {
+            abort(404);
+        }
+
+        $source = $documents->source($name, $locale);
+        $chrome = $content->page('legal', $locale);
+        $notes = [str_replace('{date}', $source['updatedAt']->locale($locale)->isoFormat('LL'), $chrome['updated'])];
+
+        if ($locale !== Locales::default()) {
+            $english = (string) $entry->markdownUrl(Locales::default());
+            $notes[] = $export->callout(
+                $chrome['notice']['title'],
+                (string) preg_replace('#<english>(.*?)</english>#', '[$1](' . $english . ')', $chrome['notice']['body']),
+            );
+        }
+
+        return $this->markdownResponse(
+            $export->document($source['title'], $source['description'], $notes, $export->body($source['body'], $locale, LegalDocuments::MARKERS)),
+            $entry->url($locale),
+        );
+    }
+
     private function document(string $component, string $name, string $path, LegalDocuments $documents, ContentRepository $content, SiteFacts $facts): Response
     {
         $locale = App::getLocale();

app/Http/Controllers/LlmsTxtController.php

@@ -0,0 +1,124 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use App\Http\Middleware\CacheHtmlAtEdge;
+use App\Services\Blog\BlogService;
+use App\Services\Blog\Post;
+use App\Services\Content\SiteFacts;
+use App\Services\Legal\LegalDocuments;
+use App\Support\Content\ContentRepository;
+use App\Support\Localization\Locales;
+use App\Support\Seo\BlogPosts;
+use App\Support\Seo\ContentCollection;
+use App\Support\Seo\LegalPages;
+use App\Support\Seo\PageEntry;
+use App\Support\Seo\PageRegistry;
+use App\Support\Seo\StaticPages;
+use Illuminate\Http\Response;
+
+// `/llms.txt` (llmstxt.org), in English, from the page registry.
+class LlmsTxtController extends Controller
+{
+    public const PATH = '/llms.txt';
+
+    // Pages left out: the homepage is this file's summary, the blog index is
+    // the posts listed below, and an integration page is the integrations hub's.
+    public const SECTIONS = [
+        'Product' => [
+            'landing.features.index',
+            'landing.features.show',
+            'landing.pricing',
+            'landing.download',
+            'landing.ios',
+            'landing.integrations.index',
+            'landing.faq',
+            'landing.security',
+            'landing.about',
+        ],
+        'Databases' => ['landing.databases.index', 'landing.databaseClient'],
+        'Comparisons' => ['landing.compare.index', 'landing.compare'],
+    ];
+
+    public function __construct(
+        private readonly PageRegistry $registry,
+        private readonly ContentRepository $content,
+        private readonly BlogService $blog,
+        private readonly LegalDocuments $legal,
+        private readonly SiteFacts $facts,
+    ) {}
+
+    public function __invoke(): Response
+    {
+        $locale = Locales::default();
+        $entries = array_values(array_filter($this->registry->all(), fn(PageEntry $entry): bool => $entry->isIndexable($locale)));
+
+        $sections = ['Docs' => ['- [Documentation](' . rtrim((string) $this->facts->links()['docs'], '/') . '/llms.txt)']];
+
+        foreach (self::SECTIONS as $section => $routes) {
+            foreach ($routes as $route) {
+                foreach ($entries as $entry) {
+                    if ($entry->route === $route) {
+                        $sections[$section][] = $this->contentLink($entry, $locale);
+                    }
+                }
+            }
+        }
+
+        $sections['Guides'] = [];
+        $sections['Optional'] = [];
+        $posts = [];
+
+        foreach ($entries as $entry) {
+            if ($entry->route === BlogPosts::ROUTE) {
+                $posts[$entry->params['slug']] = $entry;
+            }
+        }
+
+        foreach ($this->blog->all($locale) as $post) {
+            if (isset($posts[$post->slug])) {
+                $section = $post->kind() === Post::KIND_GUIDE ? 'Guides' : 'Optional';
+                $sections[$section][] = self::link($post->title, (string) $posts[$post->slug]->markdownUrl($locale), $post->description);
+            }
+        }
+
+        foreach ($entries as $entry) {
+            $document = LegalPages::DOCUMENTS[$entry->route] ?? null;
+
+            if ($document !== null) {
+                $source = $this->legal->source($document, $locale);
+                $sections['Optional'][] = self::link($source['title'], (string) $entry->markdownUrl($locale), $source['description']);
+            }
+        }
+
+        $lines = ['# ' . config('app.name'), '', '> ' . $this->content->page('home', $locale)['seo']['description']];
+
+        foreach (array_filter($sections) as $section => $links) {
+            array_push($lines, '', '## ' . $section, '', ...$links);
+        }
+
+        return response(implode("\n", $lines) . "\n", 200, [
+            'Content-Type' => 'text/plain; charset=utf-8',
+            'Cache-Control' => CacheHtmlAtEdge::CACHE_CONTROL,
+        ]);
+    }
+
+    private function contentLink(PageEntry $entry, string $locale): string
+    {
+        $name = StaticPages::PAGES[$entry->route] ?? ContentCollection::contentName($entry);
+        $copy = $this->content->page((string) $name, $locale);
+        $title = (string) $copy['seo']['title'];
+
+        // A title with a `{slot}` is filled in the browser; the card title is the same name without it.
+        if (str_contains($title, '{')) {
+            $title = (string) $copy['og']['title'];
+        }
+
+        return self::link($title, $entry->url($locale), (string) $copy['seo']['description']);
+    }
+
+    private static function link(string $title, string $url, string $description): string
+    {
+        return '- [' . $title . '](' . $url . ')' . ($description === '' ? '' : ': ' . $description);
+    }
+}

app/Services/Blog/BlogService.php

@@ -148,6 +148,11 @@ public function html(Post $post): string
         return $this->renderer->render($body, '', MarkdownRenderer::BLOG_ID_PREFIX);
     }
 
+    public function source(Post $post): string
+    {
+        return $this->bodies[$post->locale . '/' . $post->slug] ?? '';
+    }
+
     private function parse(string $slug, string $locale): ?Post
     {
         $path = $this->localeDirectory($locale) . '/' . $slug . '.md';

app/Services/Legal/LegalDocuments.php

@@ -49,6 +49,8 @@ class LegalDocuments
 
     public const BRAND_ASSETS_MARKER = '<brand-assets></brand-assets>';
 
+    public const MARKERS = [self::COOKIE_SETTINGS_MARKER, self::BRAND_ASSETS_MARKER];
+
     private const TOKEN = '/\{([a-zA-Z][a-zA-Z0-9]*)\}/';
 
     public function __construct(
@@ -79,6 +81,32 @@ public function has(string $document, string $locale): bool
      * @throws ContentMissingException
      */
     public function render(string $document, string $locale, ?string $permalinkLabel = null): array
+    {
+        $source = $this->source($document, $locale);
+        $html = $this->renderer->render($source['body'], $permalinkLabel);
+
+        foreach (self::MARKERS as $marker) {
+            $html = (string) preg_replace('#<p>\s*' . preg_quote($marker, '#') . '\s*</p>#', $marker, $html);
+        }
+
+        return [
+            'title' => $source['title'],
+            'description' => $source['description'],
+            'updatedAt' => $source['updatedAt']->toDateString(),
+            'updatedAtFormatted' => $source['updatedAt']->locale($locale)->isoFormat('LL'),
+            'html' => $html,
+            'toc' => $this->toc($html),
+        ];
+    }
+
+    /**
+     * One document in one locale as markdown, its `{token}` slots filled.
+     *
+     * @return array{title: string, description: string, updatedAt: CarbonImmutable, body: string}
+     *
+     * @throws ContentMissingException
+     */
+    public function source(string $document, string $locale): array
     {
         $path = $this->path($document, $locale);
 
@@ -87,21 +115,12 @@ public function render(string $document, string $locale, ?string $permalinkLabel
         }
 
         $parsed = YamlFrontMatter::parse((string) File::get($path));
-        $updatedAt = $this->date($parsed->matter('updatedAt'), $path);
-
-        $html = $this->renderer->render($this->fill($parsed->body(), $path, $locale), $permalinkLabel);
-
-        foreach ([self::COOKIE_SETTINGS_MARKER, self::BRAND_ASSETS_MARKER] as $marker) {
-            $html = (string) preg_replace('#<p>\s*' . preg_quote($marker, '#') . '\s*</p>#', $marker, $html);
-        }
 
         return [
             'title' => $this->fill((string) $parsed->matter('title'), $path, $locale),
             'description' => $this->fill((string) $parsed->matter('description'), $path, $locale),
-            'updatedAt' => $updatedAt->toDateString(),
-            'updatedAtFormatted' => $updatedAt->locale($locale)->isoFormat('LL'),
-            'html' => $html,
-            'toc' => $this->toc($html),
+            'updatedAt' => $this->date($parsed->matter('updatedAt'), $path),
+            'body' => $this->fill($parsed->body(), $path, $locale),
         ];
     }