diff --git a/.env-dist b/.env-dist new file mode 100644 index 0000000000..87c80bfbe8 --- /dev/null +++ b/.env-dist @@ -0,0 +1 @@ +METABASE_REPO_PATH=../metabase diff --git a/README.md b/README.md new file mode 100644 index 0000000000..2cf83a48b2 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Metabase docs site + +An Astro site that renders the Metabase docs. + +## Quick start + +```sh +bun i +bun dev +``` + +The dev server runs at http://localhost:4321/docs/latest/. + +## Serving docs from a local Metabase repo + +``` +cp .env-dist .env +``` + +Point `METABASE_REPO_PATH` at your local Metabase repo (defaults to `../metabase`, i.e. it assumes the repo is a sibling of this one). + +With `METABASE_REPO_PATH` set, `/docs/latest` serves and hot-reloads files from your local Metabase repo, and only `/latest` routes are available — earlier versions 404. Comment it out to serve all versions from `./_docs` instead. + +Restart the dev server after changing `.env`. diff --git a/astro.config.mjs b/astro.config.mjs index 089fa73a51..4111e4d6b6 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,9 +1,20 @@ // @ts-check +import path from "node:path"; import { defineConfig } from "astro/config"; import { viteStaticCopy } from "vite-plugin-static-copy"; +import { DOCS_DEST, DOCS_SRC_ROOT } from "./src/constants"; import { collectRedirects } from "./src/lib/docs/collectRedirects"; import { noopMarkdownProcessor } from "./src/lib/markdown/noopMarkdownProcessor"; +// The number of leading path segments to strip from each copied file's directory. +// Computes the directory relative to the project root and strips any leading `../`. +// e.g. `_docs` -> 1, `../metabase/docs` -> 2 +const docsSrcStripBase = path + .relative(process.cwd(), path.resolve(DOCS_SRC_ROOT)) + .replace(/^(?:\.\.\/)+/, "") + .split("/") + .filter(Boolean).length; + // https://astro.build/config export default defineConfig({ site: "https://www.metabase.com", @@ -24,16 +35,16 @@ export default defineConfig({ viteStaticCopy({ targets: [ { - src: "_docs/**/*.{jpg,png,gif,json}", - dest: "docs", - rename: { stripBase: 1 }, // strips `_docs/` + src: `${DOCS_SRC_ROOT}/**/*.{jpg,png,gif,json}`, + dest: DOCS_DEST, + rename: { stripBase: docsSrcStripBase }, }, { // TypeDoc-generated CSS/JS/icons the SDK API reference .html // docs load via relative `assets/...` URLs. - src: "_docs/**/embedding/sdk/api/assets/*.{css,js,svg,ico}", - dest: "docs", - rename: { stripBase: 1 }, + src: `${DOCS_SRC_ROOT}/**/embedding/sdk/api/assets/*.{css,js,svg,ico}`, + dest: DOCS_DEST, + rename: { stripBase: docsSrcStripBase }, }, ], }), diff --git a/lib/fetch-docs.js b/lib/fetch-docs.js index 1ad14242aa..05add42423 100644 --- a/lib/fetch-docs.js +++ b/lib/fetch-docs.js @@ -14,10 +14,10 @@ const tar = require("tar-fs"); const gunzip = require("gunzip-maybe"); // String functions -const toTitleCase = require("titlecase"); const matter = require("gray-matter"); const { canBeProcessedByFrontmatter, squashVersion } = require("./utils.js"); +const { constructDocMetadata } = require('../src/lib/docs/constructDocMetadata.ts'); const glob = require("glob"); /** @@ -204,27 +204,7 @@ function buildDocsMetadata(docPath, fileContent, settings) { return fileContent; } - const metadata = constructDocMetadata( - docPath, - settings.majorVersion, - ); - metadata.source_url = constructSourceUrl(docPath); - - if ( - parseInt(settings.majorVersion.split(".").pop()) > 43 || - settings.version === "master" - ) { - metadata.layout = "new-docs"; - } - - if (path.basename(docPath, ".md") === "README") { - metadata.permalink = - "/" + - path - .join("docs", settings.majorVersion, "index.html") - .split(path.sep) - .join("/"); - } + const metadata = constructDocMetadata(docPath, settings.majorVersion); return frontMatterify(fileContent, metadata); } @@ -240,79 +220,6 @@ function frontMatterify(fileContent, metadata) { return frontmatterBlock + content; } -function constructSourceUrl(path) { - const baseUrl = "https://github.com/metabase/metabase/blob/master/"; - const source = path.split("/"); - source.splice(0, 1); - return baseUrl + source.join("/"); -} - -function constructDocMetadata(path, version) { - const metadata = {}; - metadata.version = version; - metadata.has_magic_breadcrumbs = true; - // We default to showing both category and title breadcrumbs, then toggle either a category and/or title breadcrumb in certain scenarios - // For documentation TOC pages, we _only_ use the title in the breadcrumb - // For _category_ TOC pages (one level below the root), we only use the category in the breadbrumb - metadata.show_category_breadcrumb = true; - metadata.show_title_breadcrumb = true; - - // Remove prefixes to docs directory, making all paths below relative - const pathArray = path.split("/"); - pathArray.splice(0, 2); - - // #breadcrumb and title/category logic - if (pathArray.length === 1) { - metadata.show_category_breadcrumb = false; - metadata.category = "Table of Contents"; - metadata.title = formatDocTitle(pathArray[0]); - } else { - if (pathArray[0] === "faq") metadata.category = "FAQ"; - else metadata.category = toTitleCase(pathArray[0]).replace(/-/g, " "); - metadata.title = formatDocTitle(pathArray[pathArray.length - 1]); - } - - // MOST categories use start.md, except for the troubleshooting guide :) - if (pathArray.length > 1 && pathArray[1].match(/^(index|start)\.md$/)) - metadata.show_title_breadcrumb = false; - - return metadata; -} - -const ACRONYMS = [ - "API", - "AWS", - "DB", - "GTAP", - "JMX", - "JWT", - "LDAP", - "RDS", - "SAML", - "SQL", - "SSL", - "SSO", -]; - -function formatDocTitle(filename) { - filename = filename.replace(".md", ""); - filename = filename.replace(/-/g, " "); - filename = toTitleCase(filename); - return filename - .split(" ") - .map((word) => { - const wordIndex = ACRONYMS.findIndex( - (acronym) => acronym.toUpperCase() === word.toUpperCase(), - ); - if (wordIndex > -1) { - return ACRONYMS[wordIndex]; - } - - return word; - }) - .join(" "); -} - // Take newly extracted 'docs' directory, name it according // to appropriate Electron version, copy it to electron.atom.io // '_docs' directory and delete temp directory diff --git a/lib/test-utils.js b/lib/test-utils.js index 9482d32af4..21f36f72dd 100644 --- a/lib/test-utils.js +++ b/lib/test-utils.js @@ -1,4 +1,4 @@ -const {reformatMarkdownUrls} = require('./utils.js') +const { reformatMarkdownUrls } = require('../src/lib/docs/reformatMarkdownUrls') // Test URL reformatting @@ -67,10 +67,8 @@ Anchors work [in footers][footer-anchor] too. ] function testUrlReformatting () { - const filePath = '' - for (const test of tests) { - const actual = reformatMarkdownUrls(filePath, test.fixture) + const actual = reformatMarkdownUrls(test.fixture) const expectedLines = test.expected.split('\n') const actualLines = actual.split('\n') expectedLines.forEach((eL, i) => { diff --git a/lib/utils.js b/lib/utils.js index 37f29e5e9f..54e756fed0 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,149 +1,19 @@ -const MARKDOWN_LINK_REGEX = /\[(.+?)\]\((.+?)\)/gim; -const FOOTER_LINK_REGEX = /^\[(.+?)\]:\s+(.+?)\n/gim; const fs = require("fs"); const path = require("path"); const glob = require("glob"); const matter = require("gray-matter"); const yaml = require("yamljs"); +const { reformatMarkdownUrls } = require('../src/lib/docs/reformatMarkdownUrls'); function canBeProcessedByFrontmatter(filePath) { return path.extname(filePath) === ".md" || path.extname(filePath) === ".html" } -function reformatMarkdownUrls(filePath, body) { - let formattedBody = `${body}\n`; - - // "[text](url.md?query=something#hash=else)" => "[text](url?query=something#hash=else)" - const bodyMatchesReplacements = getReplacements( - filePath, - formattedBody.match(MARKDOWN_LINK_REGEX), - ); - if (bodyMatchesReplacements) { - bodyMatchesReplacements.forEach( - (replaceObj) => - (formattedBody = formattedBody.replace( - replaceObj.match, - replaceObj.updatedMatch, - )), - ); - } - - // "[text]: url.md?query=something#hash=else" => "[text]: url?query=something#hash=else" - const footerMatchesReplacements = getReplacements( - filePath, - formattedBody.match(FOOTER_LINK_REGEX), - ); - if (footerMatchesReplacements) { - footerMatchesReplacements.forEach( - (replaceObj) => - (formattedBody = formattedBody.replace( - replaceObj.match, - replaceObj.updatedMatch, - )), - ); - } - - return formattedBody; -} - function replaceVersionInUrls(content, { version }) { return content .replaceAll('/latest/embedding/', `/${version}/embedding/`) } -function extractUrl(match) { - // body - let url = match.match(/(?<=\[(.+?)\]\()(.+?)(?=\))/gim); - if (url) { - return url[0].trim(); - } - - // footer - url = match.match(/(?<=]: )(.+?)+/gim); - if (url) { - return url[0].trim(); - } - - return null; -} - -function isRelativeUrl(url) { - return url.indexOf("http://") === -1 && url.indexOf("https://") === -1; -} - -function isMetabaseUrl(url) { - // - metabase.com/ - // - ://www.metabase.com/ - // - ://metabase.com/ - return !!( - url.indexOf("metabase.") === 0 || - url.indexOf("://www.metabase.") === 4 || - url.indexOf("://www.metabase.") === 5 || - url.indexOf("://metabase.") === 4 || - url.indexOf("://metabase.") === 5 - ); -} - -function formatUrl(url) { - // .{md,markdown,html,htm} - url = url - // Remove extensions - .replace(".md", "") - .replace(".markdown", "") - .replace(".html", "") - .replace(".htm", "") - // Remove metabase.com - .replace("http://metabase.com", "") - .replace("https://metabase.com", "") - .replace("http://www.metabase.com", "") - .replace("https://www.metabase.com", ""); - - return url; -} - -function getReplacements(filePath, matches) { - if (matches) { - return matches - .map((match) => { - const url = extractUrl(match); - if (url) { - // Relative - if (isRelativeUrl(url)) { - return { - match, - updatedMatch: match.replace(url, formatUrl(url)), - }; - } - // Absolute + Metabase - else if (isMetabaseUrl(url)) { - // Has an extension? - const urlPaths = url.match(/(?<=.com)(.*)/gim); - if (urlPaths && urlPaths.length > 0) { - // Error if there's an extension - const [urlPath] = urlPaths; - if (urlPath.indexOf(".") > -1) { - console.error( - `Error:\n\t${filePath}\n\tMetabase url do not need extension: ${url}`, - ); - } - } - return { - match, - updatedMatch: match.replace(url, formatUrl(url)), - }; - } - } else { - console.warn(`Warning:\n\t${filePath}\n\turl not found: ${match}`); - } - - return null; - }) - .filter((match) => !!match); - } - - return null; -} - /** * Lops off the last point so that point releases will overwrite (i.e., update) the * existing docs for that major release. So docs for 40.3 will update the existing docs for 40.0 @@ -187,7 +57,7 @@ function updateRedirectsAndLinks(dir) { const { content, data } = matter(fileContent); // Trim space buffering frontmatter; // we'll add it back later in stringifyDataAndContent - let contentUpdatedLinks = reformatMarkdownUrls(filePath, content.trim()); + let contentUpdatedLinks = reformatMarkdownUrls(content.trim()); if (dir !== "_docs/latest") { contentUpdatedLinks = replaceVersionInUrls(contentUpdatedLinks, { @@ -209,7 +79,6 @@ function updateRedirectsAndLinks(dir) { module.exports = { canBeProcessedByFrontmatter, - reformatMarkdownUrls, squashVersion, stringifyDataAndContent, updateRedirectsAndLinks, diff --git a/script/docs b/script/docs index da125353c9..7590f6c0ed 100755 --- a/script/docs +++ b/script/docs @@ -1,4 +1,4 @@ -#!/usr/bin/env node +#!/usr/bin/env bun const program = require("commander"); const yaml = require("yamljs"); const fs = require("fs"); diff --git a/src/constants.ts b/src/constants.ts index 98b3b3597b..8674302ace 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,23 @@ -export const DOCS_SRC_ROOT = "_docs"; +import { loadEnv } from "vite"; + +// loadEnv() (rather than import.meta.env) so this resolves the same way here +// and in astro.config.mjs, which runs outside Vite's SSR pipeline and can't +// see import.meta.env values. +const env = loadEnv(process.env.NODE_ENV ?? "development", process.cwd(), ""); + +export const METABASE_REPO_PATH = env.METABASE_REPO_PATH; + +export const DOCS_SRC_ROOT = METABASE_REPO_PATH + ? `${METABASE_REPO_PATH}/docs` + : "_docs"; + +// Only build/watch/serve/etc /docs/latest when pointing at a local metabase repo. +// When previewing those changes, there's very unlikely ever a need to preview old versions at the same time. +// And it's much simpler and faster if the source of truth is EITHER a local metabase repo OR the committed _docs files, +// not an odd mix of both. +export const DOCS_VERSION = METABASE_REPO_PATH ? "latest" : null; + +export const DOCS_DEST = DOCS_VERSION ? `docs/${DOCS_VERSION}` : "docs"; export const UNIFY_ENABLED_PAGES = [ "/docs/latest/embedding/modular-embedding", diff --git a/src/content.config.ts b/src/content.config.ts index 49c5563892..9407d370af 100644 --- a/src/content.config.ts +++ b/src/content.config.ts @@ -1,22 +1,27 @@ import { defineCollection } from "astro:content"; import { glob } from "astro/loaders"; +import { DOCS_SRC_ROOT, DOCS_VERSION } from "./constants"; import { docsHtmlLoader } from "./lib/docs/docsHtmlLoader"; -import { DOCS_SRC_ROOT } from "./constants"; + +const PREFIX = DOCS_VERSION ? `${DOCS_VERSION}/` : ""; const docs = defineCollection({ loader: glob({ - pattern: ["**/*.md", "!**/embedding/sdk/api/snippets/**"], + pattern: ["**/*.md", "!**/embedding/sdk/api/snippets/**", "!**/util/**"], base: DOCS_SRC_ROOT, // Preserves dots (.) in pathnames - generateId: ({ entry }) => entry.replace(/\.md$/, ""), + generateId: ({ entry }) => `${PREFIX}${entry.replace(/\.md$/, "")}`, }), }); // Raw, standalone HTML docs (TypeDoc-generated SDK API reference pages, // per-version api.html ToC pages) that the glob() loader can't parse. const docsHtml = defineCollection({ - loader: docsHtmlLoader(), + loader: docsHtmlLoader({ + base: DOCS_SRC_ROOT, + generateId: (entry) => `${PREFIX}${entry}`, + }), }); export const collections = { docs, docsHtml }; diff --git a/src/lib/docs/collectRedirects.ts b/src/lib/docs/collectRedirects.ts index a2a9f1385b..bc59e41d94 100644 --- a/src/lib/docs/collectRedirects.ts +++ b/src/lib/docs/collectRedirects.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import glob from "glob"; import matter from "gray-matter"; -import { DOCS_SRC_ROOT } from "../../constants"; +import { DOCS_SRC_ROOT, DOCS_VERSION } from "../../constants"; import { resolveDocUrl } from "./resolveDoc"; const EXCLUDE = ["**/embedding/sdk/api/snippets/**"]; @@ -18,7 +18,9 @@ type DocEntry = { relPath: string; url: string; redirectFrom: string[] }; // Scans every doc under DOCS_SRC_ROOT once, returning each doc's canonical // URL (for detecting collisions with real pages) alongside the subset that -// declare `redirect_from`. +// declare `redirect_from`. When METABASE_REPO_PATH is set, scans that local +// checkout's docs/ instead, treating it as the `latest` version (matching +// the `docs` and `docsHtml` collections in content.config.ts). const scanDocEntries = (): { validUrls: Set; docsWithRedirects: DocEntry[]; @@ -37,8 +39,13 @@ const scanDocEntries = (): { const absPath = path.join(base, relPath); const { data } = matter(fs.readFileSync(absPath, "utf8")); - const id = stripExtension ? relPath.replace(/\.md$/, "") : relPath; - const { url } = resolveDocUrl({ id, permalink: data.permalink }); + const strippedPath = stripExtension + ? relPath.replace(/\.md$/, "") + : relPath; + const id = DOCS_VERSION + ? `${DOCS_VERSION}/${strippedPath}` + : strippedPath; + const { url } = resolveDocUrl({ id }); validUrls.add(url); const redirectFrom: string[] | undefined = data.redirect_from; diff --git a/src/lib/docs/constructDocMetadata.ts b/src/lib/docs/constructDocMetadata.ts new file mode 100644 index 0000000000..688afbfdd4 --- /dev/null +++ b/src/lib/docs/constructDocMetadata.ts @@ -0,0 +1,110 @@ +// Extracted from lib/fetch-docs.js so it can be used in script/docs (for cross-repo ingestion) and [...slug].astro for JIT processing +import path from "node:path"; +import toTitleCase from "titlecase"; + +const ACRONYMS = [ + "API", + "AWS", + "DB", + "GTAP", + "JMX", + "JWT", + "LDAP", + "RDS", + "SAML", + "SQL", + "SSL", + "SSO", +]; + +function formatDocTitle(filename: string) { + filename = filename.replace(".md", ""); + filename = filename.replace(/-/g, " "); + filename = toTitleCase(filename); + return filename + .split(" ") + .map((word) => { + const wordIndex = ACRONYMS.findIndex( + (acronym) => acronym.toUpperCase() === word.toUpperCase(), + ); + if (wordIndex > -1) { + return ACRONYMS[wordIndex]; + } + + return word; + }) + .join(" "); +} + +function constructSourceUrl(path: string) { + const baseUrl = "https://github.com/metabase/metabase/blob/master/"; + const source = path.split("/"); + source.splice(0, 1); + return baseUrl + source.join("/"); +} + +export type DocMetadata = { + version: string; + has_magic_breadcrumbs: true; + show_category_breadcrumb: boolean; + show_title_breadcrumb: boolean; + category: string; + title: string; + source_url: string; + layout: "docs" | "new-docs"; + permalink?: string; + latest?: boolean; +}; + +// The `page` shape passed to doc layouts: metadata plus the resolved page URL. +export type DocPage = DocMetadata & { url: string }; + +export function constructDocMetadata( + docPath: string, + version: string, + isLatest = false, +): DocMetadata { + const versionNumber = parseInt(version.split(".").pop() || "", 10); + const metadata: Partial = { + version, + has_magic_breadcrumbs: true, + // We default to showing both category and title breadcrumbs, then toggle either a category and/or title breadcrumb in certain scenarios + // For documentation TOC pages, we _only_ use the title in the breadcrumb + // For _category_ TOC pages (one level below the root), we only use the category in the breadcrumb + show_category_breadcrumb: true, + show_title_breadcrumb: true, + }; + + // Remove prefixes to docs directory, making all paths below relative + const pathArray = docPath.split("/"); + pathArray.splice(0, 2); + + // #breadcrumb and title/category logic + if (pathArray.length === 1) { + metadata.show_category_breadcrumb = false; + metadata.category = "Table of Contents"; + metadata.title = formatDocTitle(pathArray[0]); + } else { + if (pathArray[0] === "faq") metadata.category = "FAQ"; + else metadata.category = toTitleCase(pathArray[0]).replace(/-/g, " "); + metadata.title = formatDocTitle(pathArray[pathArray.length - 1]); + } + + // MOST categories use start.md, except for the troubleshooting guide :) + if (pathArray.length > 1 && pathArray[1].match(/^(index|start)\.md$/)) + metadata.show_title_breadcrumb = false; + + metadata.source_url = constructSourceUrl(docPath); + metadata.layout = versionNumber > 43 ? "new-docs" : "docs"; + + if (path.basename(docPath, ".md") === "README") { + metadata.permalink = + "/" + path.join("docs", version, "index.html").split(path.sep).join("/"); + } + + if (isLatest) { + metadata.latest = true; + } + + return metadata as DocMetadata; +} diff --git a/src/lib/docs/docsHtmlLoader.ts b/src/lib/docs/docsHtmlLoader.ts index b31fd154fe..f207f5787b 100644 --- a/src/lib/docs/docsHtmlLoader.ts +++ b/src/lib/docs/docsHtmlLoader.ts @@ -11,35 +11,47 @@ import matter from "gray-matter"; // SDK API reference pages and per-version api.html pages) that use the same // frontmatter-plus-Liquid conventions as the Markdown docs. This custom loader // lets us process both file types consistently. -export const docsHtmlLoader = (): Loader => ({ - name: "docs-html-loader", - load: async ({ config, store, parseData, generateDigest, logger }) => { - store.clear(); +// +// `base` and `generateId` mirror the options of Astro's glob() loader, so +// callers can point this at a local metabase checkout (METABASE_REPO_PATH) +// the same way the `docs` collection does. +export const docsHtmlLoader = (options?: { + base?: string; + generateId?: (entry: string) => string; +}): Loader => { + const baseDirName = options?.base ?? DOCS_SRC_ROOT; + const generateId = options?.generateId ?? ((entry: string) => entry); - const base = new URL(`${DOCS_SRC_ROOT}/`, config.root); - const baseDir = fileURLToPath(base); - const rootDir = fileURLToPath(config.root); - const entries: string[] = glob.sync("**/*.html", { - cwd: baseDir, - ignore: ["**/embedding/sdk/api/snippets/**"], - }); + return { + name: "docs-html-loader", + load: async ({ config, store, parseData, generateDigest, logger }) => { + store.clear(); - for (const entry of entries) { - const absPath = path.join(baseDir, entry); - const contents = await fs.readFile(absPath, "utf-8"); - const { data, content: body } = matter(contents); - const id = entry; - - const parsedData = await parseData({ id, data, filePath: absPath }); - store.set({ - id, - data: parsedData, - body, - filePath: path.relative(rootDir, absPath).split(path.sep).join("/"), - digest: generateDigest(contents), + const base = new URL(`${baseDirName}/`, config.root); + const baseDir = fileURLToPath(base); + const rootDir = fileURLToPath(config.root); + const entries: string[] = glob.sync("**/*.html", { + cwd: baseDir, + ignore: ["**/embedding/sdk/api/snippets/**"], }); - } - logger.info(`Loaded ${entries.length} html docs`); - }, -}); + for (const entry of entries) { + const absPath = path.join(baseDir, entry); + const contents = await fs.readFile(absPath, "utf-8"); + const { data, content: body } = matter(contents); + const id = generateId(entry); + + const parsedData = await parseData({ id, data, filePath: absPath }); + store.set({ + id, + data: parsedData, + body, + filePath: path.relative(rootDir, absPath).split(path.sep).join("/"), + digest: generateDigest(contents), + }); + } + + logger.info(`Loaded ${entries.length} html docs`); + }, + }; +}; diff --git a/src/lib/docs/reformatMarkdownUrls.ts b/src/lib/docs/reformatMarkdownUrls.ts new file mode 100644 index 0000000000..74ddb3fdec --- /dev/null +++ b/src/lib/docs/reformatMarkdownUrls.ts @@ -0,0 +1,78 @@ +// Extracted from lib/utils.js so it can be used in script/docs (for cross-repo ingestion) and [...slug].astro for JIT processing +const MARKDOWN_LINK_REGEX = /\[(.+?)\]\((.+?)\)/gim; +const FOOTER_LINK_REGEX = /^\[(.+?)\]:\s+(.+?)\n/gim; + +const extractUrl = (match: string): string | null => { + const bodyMatch = match.match(/(?<=\[(.+?)\]\()(.+?)(?=\))/gim); + if (bodyMatch) { + return bodyMatch[0].trim(); + } + const footerMatch = match.match(/(?<=]: )(.+?)+/gim); + if (footerMatch) { + return footerMatch[0].trim(); + } + return null; +}; + +const isRelativeUrl = (url: string): boolean => + !url.includes("http://") && !url.includes("https://"); + +const isMetabaseUrl = (url: string): boolean => + url.indexOf("metabase.") === 0 || + url.indexOf("://www.metabase.") === 4 || + url.indexOf("://www.metabase.") === 5 || + url.indexOf("://metabase.") === 4 || + url.indexOf("://metabase.") === 5; + +const formatUrl = (url: string): string => + url + // Remove extensions + .replace(".md", "") + .replace(".markdown", "") + .replace(".html", "") + .replace(".htm", "") + // Remove metabase.com + .replace("http://metabase.com", "") + .replace("https://metabase.com", "") + .replace("http://www.metabase.com", "") + .replace("https://www.metabase.com", ""); + +const getReplacements = ( + matches: RegExpMatchArray | null, +): { match: string; updatedMatch: string }[] | null => { + if (!matches) { + return null; + } + return matches + .map((match) => { + const url = extractUrl(match); + if (!url) { + return null; + } + if (isRelativeUrl(url) || isMetabaseUrl(url)) { + return { match, updatedMatch: match.replace(url, formatUrl(url)) }; + } + return null; + }) + .filter((replacement) => replacement !== null); +}; + +export const reformatMarkdownUrls = (body: string): string => { + let formattedBody = `${body}\n`; + + const bodyReplacements = getReplacements( + formattedBody.match(MARKDOWN_LINK_REGEX), + ); + bodyReplacements?.forEach(({ match, updatedMatch }) => { + formattedBody = formattedBody.replace(match, updatedMatch); + }); + + const footerReplacements = getReplacements( + formattedBody.match(FOOTER_LINK_REGEX), + ); + footerReplacements?.forEach(({ match, updatedMatch }) => { + formattedBody = formattedBody.replace(match, updatedMatch); + }); + + return formattedBody; +}; diff --git a/src/lib/docs/resolveDoc.ts b/src/lib/docs/resolveDoc.ts index 38de3749f7..fea8b5b5f1 100644 --- a/src/lib/docs/resolveDoc.ts +++ b/src/lib/docs/resolveDoc.ts @@ -1,20 +1,14 @@ -// Derives a doc's version/slug/URL from its content collection id (or -// frontmatter `permalink` override), since docs are stored as -// `/.md` but need a canonical `/docs//` URL for -// routing, sitemaps, and redirects. +// Derives a doc's version/slug/URL from its content collection id, since +// docs are stored as `/.md` but need a canonical +// `/docs//` URL for routing, sitemaps, and redirects. export const resolveDocUrl = ({ id, - permalink, includeTrailingIndex, }: { id: string; - permalink?: string; includeTrailingIndex?: boolean; }): { version: string; slug: string; url: string } => { - let resolvedId = (permalink?.replace(/^\/docs\//, "") ?? id).replace( - /\.html$/, - "", - ); + let resolvedId = id.replace(/\.html$/, "").replace(/\/README$/, "/index"); if (!includeTrailingIndex) { resolvedId = resolvedId.replace(/index$/, ""); } diff --git a/src/lib/markdown/plugins/relativeImagePlugin.ts b/src/lib/markdown/plugins/relativeImagePlugin.ts index 3a42f3ba94..6ff8b47646 100644 --- a/src/lib/markdown/plugins/relativeImagePlugin.ts +++ b/src/lib/markdown/plugins/relativeImagePlugin.ts @@ -1,11 +1,17 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; -import { DOCS_SRC_ROOT } from "@/constants"; +import { DOCS_DEST, DOCS_SRC_ROOT } from "@/constants"; import { defineHastPlugin } from "satteri"; // Resolves relative images from docs md files. // The images themselves are copied via viteStaticCopy in astro.config.mjs. +const ROOT_ABS = path.resolve(DOCS_SRC_ROOT); +const URL_PREFIX = `/${DOCS_DEST}/`; + +const toSitePath = (relPath: string) => + relPath.split(path.sep).map(encodeURIComponent).join("/"); + export const relativeImagePlugin = defineHastPlugin({ name: "relative-image-resolver", element: { @@ -18,11 +24,10 @@ export const relativeImagePlugin = defineHastPlugin({ if (!ctx.fileURL) return; const absPath = fileURLToPath(new URL(decodeURI(rawSrc), ctx.fileURL)); - const relPath = path.relative(DOCS_SRC_ROOT, absPath); + const relPath = path.relative(ROOT_ABS, absPath); if (relPath.startsWith("..") || path.isAbsolute(relPath)) return; - const newSrc = `/docs/${relPath.split(path.sep).map(encodeURIComponent).join("/")}`; - ctx.setProperty(node, "src", newSrc); + ctx.setProperty(node, "src", `${URL_PREFIX}${toSitePath(relPath)}`); }, }, }); diff --git a/src/pages/docs/[version]/[...slug].astro b/src/pages/docs/[version]/[...slug].astro index a3ce511184..beb8d2a1ca 100644 --- a/src/pages/docs/[version]/[...slug].astro +++ b/src/pages/docs/[version]/[...slug].astro @@ -1,10 +1,13 @@ --- import path from "node:path"; import { pathToFileURL } from "node:url"; +import { DOCS_SRC_ROOT, METABASE_REPO_PATH } from "@/constants"; import NewDocsLayout from "@/layouts/NewDocsLayout.astro"; import OldDocsLayout from "@/layouts/OldDocsLayout.astro"; +import { constructDocMetadata } from "@/lib/docs/constructDocMetadata"; +import { reformatMarkdownUrls } from "@/lib/docs/reformatMarkdownUrls"; import { resolveDocUrl } from "@/lib/docs/resolveDoc"; -import { getLiquidRenderer } from "@/lib/liquid/liquidRenderer"; +import { baseCtx, getLiquidRenderer } from "@/lib/liquid/liquidRenderer"; import { getMarkdownRenderer } from "@/lib/markdown/markdownRenderer"; import { getCollection, type DataEntryMap } from "astro:content"; @@ -20,10 +23,9 @@ export const getStaticPaths = async () => { const toPath = (kind: Kind) => - (doc: { id: string; data: { permalink?: string } }) => { + (doc: { id: string; data: Record; body?: string }) => { const { version, slug } = resolveDocUrl({ id: doc.id, - permalink: doc.data.permalink, // For prod builds, we want to output like folder/index.html, but for the dev server, the route should exclude /index includeTrailingIndex: import.meta.env.MODE !== "development", }); @@ -34,7 +36,29 @@ export const getStaticPaths = async () => { }; const { version, slug = "" } = Astro.params; -const { kind, doc } = Astro.props; +const { kind, doc: propsDoc } = Astro.props; +const url = `/docs/${version}/${slug}`; +// TODO: GRO-688 Remove processing from /script/docs so src files are raw whether reading from _docs or METABASE_REPO_PATH +// The only known gap is updateRedirectsAndLinks for non-latest versions, which is two separate fixes: +// 1. replaceVersionInUrls belongs alongside reformatMarkdownUrls +// 2. redirect_from rewriting belongs in collectRedirects +const isRaw = !!METABASE_REPO_PATH; +const doc = !isRaw + ? propsDoc + : { + ...propsDoc, + data: { + ...constructDocMetadata( + METABASE_REPO_PATH + ? `/${path.relative(METABASE_REPO_PATH, propsDoc.filePath!)}` + : `/docs/${path.relative(`${DOCS_SRC_ROOT}/${version}`, propsDoc.filePath!)}`, + version === "latest" ? baseCtx.site.docs_version : version, + version === "latest", + ), + ...propsDoc.data, + }, + body: reformatMarkdownUrls(propsDoc.body ?? ""), + }; const dirname = path.dirname(doc.filePath!); // Process liquid first (e.g. control flow, includes, variables, etc) @@ -72,10 +96,7 @@ const Layout = doc.data.layout === "docs" ? OldDocsLayout : NewDocsLayout; kind === "html" ? ( ) : ( - + ) diff --git a/src/pages/docs/sitemap.xml.ts b/src/pages/docs/sitemap.xml.ts index 695305541c..7250d32c59 100644 --- a/src/pages/docs/sitemap.xml.ts +++ b/src/pages/docs/sitemap.xml.ts @@ -14,9 +14,7 @@ export const GET: APIRoute = async () => { ]); const paths = [...docs, ...docsHtml] - .map( - (doc) => resolveDocUrl({ id: doc.id, permalink: doc.data.permalink }).url, - ) + .map((doc) => resolveDocUrl({ id: doc.id }).url) .sort(); const body = ` diff --git a/src/types/titlecase.d.ts b/src/types/titlecase.d.ts new file mode 100644 index 0000000000..e4424feae3 --- /dev/null +++ b/src/types/titlecase.d.ts @@ -0,0 +1,3 @@ +declare module "titlecase" { + export default function toTitleCase(input: string): string; +}