docs(seo): improve crawlability and add LLM index endpoints

src/app/[locale]/blog/[name]/page.tsx

@@ -1,6 +1,7 @@
 import { getStaticParams } from '@/locales/server';
 import { BLOGS_DIR, getMarkDownMetaData } from '@/utils/markdown';
 import fs from 'fs';
+import type { Metadata } from 'next';
 import { notFound } from 'next/navigation';
 import path from 'path';
 
@@ -28,15 +29,40 @@ export async function generateMetadata({
   params,
 }: {
   params: Promise<ParamsProps>;
-}) {
+}): Promise<Metadata> {
   const { locale, name } = await params;
   const mdxPath = path.join(BLOGS_DIR, locale, `${name}.mdx`);
   const defaultMdxEnPath = path.join(BLOGS_DIR, 'en', `${name}.mdx`);
+  const canonicalPath = `/blog/${name}`;
 
   if (fs.existsSync(mdxPath)) {
-    return await getMarkDownMetaData(mdxPath);
+    const metadata = (await getMarkDownMetaData(mdxPath)) as Metadata;
+    return {
+      ...metadata,
+      alternates: {
+        ...metadata.alternates,
+        canonical: canonicalPath,
+      },
+      openGraph: {
+        ...metadata.openGraph,
+        url: canonicalPath,
+        type: 'article',
+      },
+    };
   } else {
-    return await getMarkDownMetaData(defaultMdxEnPath);
+    const metadata = (await getMarkDownMetaData(defaultMdxEnPath)) as Metadata;
+    return {
+      ...metadata,
+      alternates: {
+        ...metadata.alternates,
+        canonical: canonicalPath,
+      },
+      openGraph: {
+        ...metadata.openGraph,
+        url: canonicalPath,
+        type: 'article',
+      },
+    };
   }
 }
 

src/app/[locale]/blog/page.tsx

@@ -10,15 +10,21 @@ import {
   Typography,
 } from '@mui/material';
 import _ from 'lodash';
+import type { Metadata } from 'next';
 import Image from 'next/image';
 import { BlogList } from './BlogList';
 export async function generateStaticParams() {
   return getStaticParams();
 }
 
-export async function generateMetadata() {
+export async function generateMetadata(): Promise<Metadata> {
   return {
     title: 'Kubeblocks blogs',
+    description:
+      'Technical blogs, release highlights, and engineering guides for running databases on Kubernetes with KubeBlocks.',
+    alternates: {
+      canonical: '/blog',
+    },
   };
 }
 

src/app/[locale]/docs/[version]/[category]/[[...paths]]/page.tsx

@@ -11,6 +11,7 @@ import {
 import fs from 'fs';
 import _ from 'lodash';
 import matter from 'gray-matter';
+import type { Metadata } from 'next';
 import { redirect } from 'next/navigation';
 import path from 'path';
 
@@ -133,17 +134,43 @@ export async function generateMetadata({
   params,
 }: {
   params: Promise<MarkdownPageParams>;
-}) {
+}): Promise<Metadata> {
   const { locale, version, category, paths = [] } = await params;
   const mdxPath =
     path.join(DOCS_DIR, locale, version, category, ...paths) + '.mdx';
 
   const defaultDdxEnPath =
     path.join(DOCS_DIR, 'en', version, category, ...paths) + '.mdx';
 
+  const canonicalPath = `/docs/${version}/${category}/${paths.join('/')}`;
+
   if (fs.existsSync(mdxPath)) {
-    return await getMarkDownMetaData(mdxPath);
+    const metadata = (await getMarkDownMetaData(mdxPath)) as Metadata;
+    return {
+      ...metadata,
+      alternates: {
+        ...metadata.alternates,
+        canonical: canonicalPath,
+      },
+      openGraph: {
+        ...metadata.openGraph,
+        url: canonicalPath,
+        type: 'article',
+      },
+    };
   } else {
-    return await getMarkDownMetaData(defaultDdxEnPath);
+    const metadata = (await getMarkDownMetaData(defaultDdxEnPath)) as Metadata;
+    return {
+      ...metadata,
+      alternates: {
+        ...metadata.alternates,
+        canonical: canonicalPath,
+      },
+      openGraph: {
+        ...metadata.openGraph,
+        url: canonicalPath,
+        type: 'article',
+      },
+    };
   }
 }

src/app/[locale]/layout.tsx

@@ -9,6 +9,7 @@ import { GoogleAnalytics } from '@next/third-parties/google';
 import type { Metadata } from 'next';
 import { setStaticParamsLocale } from 'next-international/server';
 import { Geist } from 'next/font/google';
+import { getSiteUrl } from '@/utils/site';
 import { ElevationScrollAppBar } from './ElevationScrollAppBar';
 
 import 'highlight.js/styles/github-dark.css';
@@ -20,9 +21,17 @@ const geist = Geist({
 });
 
 export const metadata: Metadata = {
-  title: 'KubeBlocks',
+  metadataBase: new URL(getSiteUrl()),
+  title: {
+    default: 'KubeBlocks',
+    template: '%s | KubeBlocks',
+  },
   description:
     'Meet KubeBlocks, the open-source, unified database operator for Kubernetes. Simplify cloud-native data management with a single API for MySQL, PostgreSQL, MongoDB, Kafka, and more. Tame operator sprawl and streamline Day-2 operations.',
+  robots: {
+    index: true,
+    follow: true,
+  },
 };
 
 export default async function RootLayout({

src/app/[locale]/page.tsx

@@ -2,6 +2,7 @@ import Footer from '@/components/Footer';
 import { getStaticParams } from '@/locales/server';
 import { getBlogs } from '@/utils/markdown';
 import { Box, Divider } from '@mui/material';
+import type { Metadata } from 'next';
 import Banner from './banner';
 import BlogsPreview from './blogs-preview';
 import Contact from './contact';
@@ -14,6 +15,14 @@ export async function generateStaticParams() {
   return getStaticParams();
 }
 
+export async function generateMetadata(): Promise<Metadata> {
+  return {
+    alternates: {
+      canonical: '/',
+    },
+  };
+}
+
 export default async function HomePage({
   params,
 }: {

src/app/llms-full.txt/route.ts

@@ -0,0 +1,29 @@
+import { normalizeRoutePath, readLlmDocEntries } from '@/utils/llms';
+import { toAbsoluteUrl } from '@/utils/site';
+
+export async function GET() {
+  const entries = readLlmDocEntries().sort((a, b) =>
+    a.path.localeCompare(b.path),
+  );
+
+  const lines: string[] = [
+    '# KubeBlocks LLM Full Index',
+    '',
+    `- Generated from: ${toAbsoluteUrl('/docs-index.json')}`,
+    `- Site: ${toAbsoluteUrl('/')}`,
+    '',
+    '## URLs',
+    ...entries.map((entry) => {
+      const title = entry.title || entry.path;
+      const absoluteUrl = toAbsoluteUrl(normalizeRoutePath(entry.path));
+      return `- ${title}: ${absoluteUrl}`;
+    }),
+  ];
+
+  return new Response(lines.join('\n'), {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600',
+    },
+  });
+}

src/app/llms.txt/route.ts

@@ -0,0 +1,65 @@
+import { normalizeRoutePath, readLlmDocEntries } from '@/utils/llms';
+import { toAbsoluteUrl } from '@/utils/site';
+
+export async function GET() {
+  const entries = readLlmDocEntries();
+  const docsEntries = entries.filter((entry) =>
+    entry.path.startsWith('docs/preview/'),
+  );
+  const blogEntries = entries.filter((entry) => entry.path.startsWith('blog/'));
+
+  const quickstarts = docsEntries
+    .filter((entry) => /\/02-quickstart$/.test(entry.path))
+    .sort((a, b) => a.path.localeCompare(b.path))
+    .slice(0, 30);
+
+  const overviews = docsEntries
+    .filter((entry) => /\/01-overview$/.test(entry.path))
+    .sort((a, b) => a.path.localeCompare(b.path))
+    .slice(0, 30);
+
+  const latestBlogs = [...blogEntries]
+    .sort((a, b) =>
+      (b.lastModified || '').localeCompare(a.lastModified || ''),
+    )
+    .slice(0, 20);
+
+  const lines: string[] = [
+    '# KubeBlocks LLM Index',
+    '',
+    `- Site: ${toAbsoluteUrl('/')}`,
+    `- Full index: ${toAbsoluteUrl('/llms-full.txt')}`,
+    `- XML sitemap: ${toAbsoluteUrl('/sitemap.xml')}`,
+    '',
+    '## Product quickstarts',
+    ...quickstarts.map(
+      (entry) =>
+        `- ${entry.title || entry.path}: ${toAbsoluteUrl(
+          normalizeRoutePath(entry.path),
+        )}`,
+    ),
+    '',
+    '## Product overviews',
+    ...overviews.map(
+      (entry) =>
+        `- ${entry.title || entry.path}: ${toAbsoluteUrl(
+          normalizeRoutePath(entry.path),
+        )}`,
+    ),
+    '',
+    '## Recent blogs',
+    ...latestBlogs.map(
+      (entry) =>
+        `- ${entry.title || entry.path}: ${toAbsoluteUrl(
+          normalizeRoutePath(entry.path),
+        )}`,
+    ),
+  ];
+
+  return new Response(lines.join('\n'), {
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': 'public, max-age=3600',
+    },
+  });
+}

src/app/robots.ts

@@ -0,0 +1,18 @@
+import { getSiteUrl } from '@/utils/site';
+import type { MetadataRoute } from 'next';
+
+export default function robots(): MetadataRoute.Robots {
+  const siteUrl = getSiteUrl();
+
+  return {
+    rules: [
+      {
+        userAgent: '*',
+        allow: '/',
+        disallow: ['/api/', '/_next/', '/static/'],
+      },
+    ],
+    sitemap: `${siteUrl}/sitemap.xml`,
+    host: siteUrl,
+  };
+}