diff --git a/src/app/[locale]/blog/[name]/page.tsx b/src/app/[locale]/blog/[name]/page.tsx index 7e4dedf6..a9cc40f5 100644 --- a/src/app/[locale]/blog/[name]/page.tsx +++ b/src/app/[locale]/blog/[name]/page.tsx @@ -1,6 +1,7 @@ import { getStaticParams } from '@/locales/server'; import { BLOGS_DIR, getMarkDownMetaData } from '@/utils/markdown'; import fs from 'fs'; +import type { Metadata } from 'next'; import { notFound } from 'next/navigation'; import path from 'path'; @@ -28,15 +29,40 @@ export async function generateMetadata({ params, }: { params: Promise; -}) { +}): Promise { const { locale, name } = await params; const mdxPath = path.join(BLOGS_DIR, locale, `${name}.mdx`); const defaultMdxEnPath = path.join(BLOGS_DIR, 'en', `${name}.mdx`); + const canonicalPath = `/blog/${name}`; if (fs.existsSync(mdxPath)) { - return await getMarkDownMetaData(mdxPath); + const metadata = (await getMarkDownMetaData(mdxPath)) as Metadata; + return { + ...metadata, + alternates: { + ...metadata.alternates, + canonical: canonicalPath, + }, + openGraph: { + ...metadata.openGraph, + url: canonicalPath, + type: 'article', + }, + }; } else { - return await getMarkDownMetaData(defaultMdxEnPath); + const metadata = (await getMarkDownMetaData(defaultMdxEnPath)) as Metadata; + return { + ...metadata, + alternates: { + ...metadata.alternates, + canonical: canonicalPath, + }, + openGraph: { + ...metadata.openGraph, + url: canonicalPath, + type: 'article', + }, + }; } } diff --git a/src/app/[locale]/blog/page.tsx b/src/app/[locale]/blog/page.tsx index b39a4a82..0ac84821 100644 --- a/src/app/[locale]/blog/page.tsx +++ b/src/app/[locale]/blog/page.tsx @@ -10,15 +10,21 @@ import { Typography, } from '@mui/material'; import _ from 'lodash'; +import type { Metadata } from 'next'; import Image from 'next/image'; import { BlogList } from './BlogList'; export async function generateStaticParams() { return getStaticParams(); } -export async function generateMetadata() { +export async function generateMetadata(): Promise { return { title: 'Kubeblocks blogs', + description: + 'Technical blogs, release highlights, and engineering guides for running databases on Kubernetes with KubeBlocks.', + alternates: { + canonical: '/blog', + }, }; } diff --git a/src/app/[locale]/docs/[version]/[category]/[[...paths]]/page.tsx b/src/app/[locale]/docs/[version]/[category]/[[...paths]]/page.tsx index 05799da3..af05fdd5 100644 --- a/src/app/[locale]/docs/[version]/[category]/[[...paths]]/page.tsx +++ b/src/app/[locale]/docs/[version]/[category]/[[...paths]]/page.tsx @@ -11,6 +11,7 @@ import { import fs from 'fs'; import _ from 'lodash'; import matter from 'gray-matter'; +import type { Metadata } from 'next'; import { redirect } from 'next/navigation'; import path from 'path'; @@ -133,7 +134,7 @@ export async function generateMetadata({ params, }: { params: Promise; -}) { +}): Promise { const { locale, version, category, paths = [] } = await params; const mdxPath = path.join(DOCS_DIR, locale, version, category, ...paths) + '.mdx'; @@ -141,9 +142,35 @@ export async function generateMetadata({ const defaultDdxEnPath = path.join(DOCS_DIR, 'en', version, category, ...paths) + '.mdx'; + const canonicalPath = `/docs/${version}/${category}/${paths.join('/')}`; + if (fs.existsSync(mdxPath)) { - return await getMarkDownMetaData(mdxPath); + const metadata = (await getMarkDownMetaData(mdxPath)) as Metadata; + return { + ...metadata, + alternates: { + ...metadata.alternates, + canonical: canonicalPath, + }, + openGraph: { + ...metadata.openGraph, + url: canonicalPath, + type: 'article', + }, + }; } else { - return await getMarkDownMetaData(defaultDdxEnPath); + const metadata = (await getMarkDownMetaData(defaultDdxEnPath)) as Metadata; + return { + ...metadata, + alternates: { + ...metadata.alternates, + canonical: canonicalPath, + }, + openGraph: { + ...metadata.openGraph, + url: canonicalPath, + type: 'article', + }, + }; } } diff --git a/src/app/[locale]/layout.tsx b/src/app/[locale]/layout.tsx index 9fe2fbbb..8879ae0b 100644 --- a/src/app/[locale]/layout.tsx +++ b/src/app/[locale]/layout.tsx @@ -9,6 +9,7 @@ import { GoogleAnalytics } from '@next/third-parties/google'; import type { Metadata } from 'next'; import { setStaticParamsLocale } from 'next-international/server'; import { Geist } from 'next/font/google'; +import { getSiteUrl } from '@/utils/site'; import { ElevationScrollAppBar } from './ElevationScrollAppBar'; import 'highlight.js/styles/github-dark.css'; @@ -20,9 +21,17 @@ const geist = Geist({ }); export const metadata: Metadata = { - title: 'KubeBlocks', + metadataBase: new URL(getSiteUrl()), + title: { + default: 'KubeBlocks', + template: '%s | KubeBlocks', + }, description: 'Meet KubeBlocks, the open-source, unified database operator for Kubernetes. Simplify cloud-native data management with a single API for MySQL, PostgreSQL, MongoDB, Kafka, and more. Tame operator sprawl and streamline Day-2 operations.', + robots: { + index: true, + follow: true, + }, }; export default async function RootLayout({ diff --git a/src/app/[locale]/page.tsx b/src/app/[locale]/page.tsx index 0ba22837..fe0ce1c3 100644 --- a/src/app/[locale]/page.tsx +++ b/src/app/[locale]/page.tsx @@ -2,6 +2,7 @@ import Footer from '@/components/Footer'; import { getStaticParams } from '@/locales/server'; import { getBlogs } from '@/utils/markdown'; import { Box, Divider } from '@mui/material'; +import type { Metadata } from 'next'; import Banner from './banner'; import BlogsPreview from './blogs-preview'; import Contact from './contact'; @@ -14,6 +15,14 @@ export async function generateStaticParams() { return getStaticParams(); } +export async function generateMetadata(): Promise { + return { + alternates: { + canonical: '/', + }, + }; +} + export default async function HomePage({ params, }: { diff --git a/src/app/llms-full.txt/route.ts b/src/app/llms-full.txt/route.ts new file mode 100644 index 00000000..2c3203ea --- /dev/null +++ b/src/app/llms-full.txt/route.ts @@ -0,0 +1,29 @@ +import { normalizeRoutePath, readLlmDocEntries } from '@/utils/llms'; +import { toAbsoluteUrl } from '@/utils/site'; + +export async function GET() { + const entries = readLlmDocEntries().sort((a, b) => + a.path.localeCompare(b.path), + ); + + const lines: string[] = [ + '# KubeBlocks LLM Full Index', + '', + `- Generated from: ${toAbsoluteUrl('/docs-index.json')}`, + `- Site: ${toAbsoluteUrl('/')}`, + '', + '## URLs', + ...entries.map((entry) => { + const title = entry.title || entry.path; + const absoluteUrl = toAbsoluteUrl(normalizeRoutePath(entry.path)); + return `- ${title}: ${absoluteUrl}`; + }), + ]; + + return new Response(lines.join('\n'), { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600', + }, + }); +} diff --git a/src/app/llms.txt/route.ts b/src/app/llms.txt/route.ts new file mode 100644 index 00000000..3514b167 --- /dev/null +++ b/src/app/llms.txt/route.ts @@ -0,0 +1,65 @@ +import { normalizeRoutePath, readLlmDocEntries } from '@/utils/llms'; +import { toAbsoluteUrl } from '@/utils/site'; + +export async function GET() { + const entries = readLlmDocEntries(); + const docsEntries = entries.filter((entry) => + entry.path.startsWith('docs/preview/'), + ); + const blogEntries = entries.filter((entry) => entry.path.startsWith('blog/')); + + const quickstarts = docsEntries + .filter((entry) => /\/02-quickstart$/.test(entry.path)) + .sort((a, b) => a.path.localeCompare(b.path)) + .slice(0, 30); + + const overviews = docsEntries + .filter((entry) => /\/01-overview$/.test(entry.path)) + .sort((a, b) => a.path.localeCompare(b.path)) + .slice(0, 30); + + const latestBlogs = [...blogEntries] + .sort((a, b) => + (b.lastModified || '').localeCompare(a.lastModified || ''), + ) + .slice(0, 20); + + const lines: string[] = [ + '# KubeBlocks LLM Index', + '', + `- Site: ${toAbsoluteUrl('/')}`, + `- Full index: ${toAbsoluteUrl('/llms-full.txt')}`, + `- XML sitemap: ${toAbsoluteUrl('/sitemap.xml')}`, + '', + '## Product quickstarts', + ...quickstarts.map( + (entry) => + `- ${entry.title || entry.path}: ${toAbsoluteUrl( + normalizeRoutePath(entry.path), + )}`, + ), + '', + '## Product overviews', + ...overviews.map( + (entry) => + `- ${entry.title || entry.path}: ${toAbsoluteUrl( + normalizeRoutePath(entry.path), + )}`, + ), + '', + '## Recent blogs', + ...latestBlogs.map( + (entry) => + `- ${entry.title || entry.path}: ${toAbsoluteUrl( + normalizeRoutePath(entry.path), + )}`, + ), + ]; + + return new Response(lines.join('\n'), { + headers: { + 'Content-Type': 'text/plain; charset=utf-8', + 'Cache-Control': 'public, max-age=3600', + }, + }); +} diff --git a/src/app/robots.ts b/src/app/robots.ts new file mode 100644 index 00000000..f33f2ab0 --- /dev/null +++ b/src/app/robots.ts @@ -0,0 +1,18 @@ +import { getSiteUrl } from '@/utils/site'; +import type { MetadataRoute } from 'next'; + +export default function robots(): MetadataRoute.Robots { + const siteUrl = getSiteUrl(); + + return { + rules: [ + { + userAgent: '*', + allow: '/', + disallow: ['/api/', '/_next/', '/static/'], + }, + ], + sitemap: `${siteUrl}/sitemap.xml`, + host: siteUrl, + }; +} diff --git a/src/app/robots.txt b/src/app/robots.txt deleted file mode 100644 index e82771f4..00000000 --- a/src/app/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -User-Agent: * -Allow: / -Sitemap: https://kubeblocks.io/sitemap.xml \ No newline at end of file diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts index 0c1d7d4c..b1656a08 100644 --- a/src/app/sitemap.ts +++ b/src/app/sitemap.ts @@ -1,57 +1,110 @@ -import { BLOGS_DIR } from '@/utils/markdown'; +import { BLOGS_DIR, DOCS_DIR } from '@/utils/markdown'; +import { getSiteUrl } from '@/utils/site'; import fs from 'fs'; import type { MetadataRoute } from 'next'; - -const lastModified = new Date(); +import path from 'path'; import { getStaticParams } from '@/locales/server'; -import path from 'path'; export default function sitemap(): MetadataRoute.Sitemap { + const siteUrl = getSiteUrl(); const sitemap: MetadataRoute.Sitemap = []; - const docsDir = path.join(process.cwd(), 'docs'); - const getPaths = (dir: string, initData: string[] = []): string[] => { - fs.readdirSync(dir).forEach((f) => { - const d = path.join(dir, f); - const stat = fs.statSync(d); + const withLocalePath = (locale: string, routePath: string) => { + if (locale === 'en') { + return routePath; + } + return `/${locale}${routePath}`; + }; + + const addStaticRoute = ( + routePath: string, + locale = 'en', + priority = 0.7, + changeFrequency: MetadataRoute.Sitemap[number]['changeFrequency'] = 'weekly', + ) => { + sitemap.push({ + url: `${siteUrl}${withLocalePath(locale, routePath)}`, + lastModified: new Date(), + changeFrequency, + priority, + }); + }; + + const getMdxFiles = (dir: string, files: string[] = []): string[] => { + if (!fs.existsSync(dir)) { + return files; + } + + fs.readdirSync(dir).forEach((entryName) => { + if (entryName.startsWith('_')) { + return; + } + const entryPath = path.join(dir, entryName); + const stat = fs.statSync(entryPath); if (stat.isDirectory()) { - getPaths(d, initData); + getMdxFiles(entryPath, files); + return; } - if (stat.isFile() && f.endsWith('.mdx')) { - initData.push(d); + if (stat.isFile() && entryName.endsWith('.mdx')) { + files.push(entryPath); } }); - return initData; + + return files; }; + addStaticRoute('/', 'en', 1, 'daily'); + addStaticRoute('/blog', 'en', 0.8, 'daily'); + getStaticParams().forEach((item) => { - // locals - const localeDir = path.join(docsDir, item.locale); + const localeDir = path.join(DOCS_DIR, item.locale); + + if (!fs.existsSync(localeDir)) { + return; + } + + if (item.locale !== 'en') { + addStaticRoute('/', item.locale, 0.9, 'daily'); + addStaticRoute('/blog', item.locale, 0.7, 'weekly'); + } - fs.readdirSync(localeDir).forEach((version) => { - // versions + const versions = fs.readdirSync(localeDir); + + versions.forEach((version) => { const versionDir = path.join(localeDir, version); - fs.readdirSync(versionDir).forEach((category) => { - // categories - const cateDir = path.join(versionDir, category); - const paths: string[] = getPaths(cateDir).map((item) => - item.replace(cateDir + '/', '').replace('.mdx', ''), - ); - - // ignore some category - if (['release_notes'].includes(category)) { + if (!fs.statSync(versionDir).isDirectory()) { + return; + } + const categories = fs.readdirSync(versionDir); + + categories.forEach((category) => { + if (category.startsWith('_') || category === 'release_notes') { return; } - paths.forEach((p) => { - const items = p.split('/'); + const categoryDir = path.join(versionDir, category); + if (!fs.statSync(categoryDir).isDirectory()) { + return; + } + + const files = getMdxFiles(categoryDir); + files.forEach((filePath) => { + const relative = filePath + .replace(`${categoryDir}/`, '') + .replace(/\.mdx$/, ''); + + if (relative.split('/').some((segment) => segment.startsWith('_'))) { + return; + } + + const routePath = `/docs/${version}/${category}/${relative}`; + const fileLastModified = fs.statSync(filePath).mtime; + sitemap.push({ - url: `https://kubeblocks.io/docs/${version}/${category}/${items?.join( - '/', - )}`, - lastModified, + url: `${siteUrl}${withLocalePath(item.locale, routePath)}`, + lastModified: fileLastModified, changeFrequency: 'weekly', - priority: 0.5, + priority: 0.6, }); }); }); @@ -60,19 +113,29 @@ export default function sitemap(): MetadataRoute.Sitemap { getStaticParams().forEach((item) => { const dir = path.join(BLOGS_DIR, item.locale); - if (fs.existsSync(dir)) { - fs.readdirSync(dir) - .filter((f) => f.endsWith('.mdx')) - .forEach((f) => { - sitemap.push({ - url: `https://kubeblocks.io/blog/${f.replace(/\.mdx/, '')}`, - lastModified, - changeFrequency: 'weekly', - priority: 0.5, - }); - }); + if (!fs.existsSync(dir)) { + return; } + + fs.readdirSync(dir) + .filter((f) => f.endsWith('.mdx')) + .forEach((f) => { + const filePath = path.join(dir, f); + const fileLastModified = fs.statSync(filePath).mtime; + const slug = f.replace(/\.mdx/, ''); + const routePath = `/blog/${slug}`; + + sitemap.push({ + url: `${siteUrl}${withLocalePath(item.locale, routePath)}`, + lastModified: fileLastModified, + changeFrequency: 'weekly', + priority: 0.5, + }); + }); }); + addStaticRoute('/llms.txt', 'en', 0.5, 'weekly'); + addStaticRoute('/llms-full.txt', 'en', 0.4, 'weekly'); + return sitemap; } diff --git a/src/utils/llms.ts b/src/utils/llms.ts new file mode 100644 index 00000000..12f8aac0 --- /dev/null +++ b/src/utils/llms.ts @@ -0,0 +1,39 @@ +import fs from 'fs'; +import path from 'path'; + +export type LlmDocEntry = { + path: string; + title?: string; + description?: string; + docType?: string; + lastModified?: string; +}; + +const DOCS_INDEX_PATH = path.join(process.cwd(), 'public', 'docs-index.json'); +let cachedEntries: LlmDocEntry[] | null = null; + +export const readLlmDocEntries = (): LlmDocEntry[] => { + if (cachedEntries) { + return cachedEntries; + } + + if (!fs.existsSync(DOCS_INDEX_PATH)) { + return []; + } + + try { + const raw = fs.readFileSync(DOCS_INDEX_PATH, 'utf8'); + const parsed = JSON.parse(raw) as LlmDocEntry[]; + cachedEntries = Array.isArray(parsed) ? parsed : []; + return cachedEntries; + } catch { + return []; + } +}; + +export const normalizeRoutePath = (rawPath: string): string => { + if (rawPath.startsWith('/')) { + return rawPath; + } + return `/${rawPath}`; +}; diff --git a/src/utils/site.ts b/src/utils/site.ts new file mode 100644 index 00000000..857c48e0 --- /dev/null +++ b/src/utils/site.ts @@ -0,0 +1,19 @@ +const DEFAULT_SITE_URL = 'https://kubeblocks.io'; + +export const getSiteUrl = (): string => { + const raw = + process.env.NEXT_PUBLIC_SITE_URL || + process.env.SITE_URL || + DEFAULT_SITE_URL; + + return raw.replace(/\/+$/, ''); +}; + +export const toAbsoluteUrl = (path: string): string => { + if (/^https?:\/\//.test(path)) { + return path; + } + + const normalizedPath = path.startsWith('/') ? path : `/${path}`; + return `${getSiteUrl()}${normalizedPath}`; +};