diff --git a/antora-playbook.yml b/antora-playbook.yml index 8fc6cefdc..a6981418b 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -29,6 +29,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/lib/hub-supported-versions.js b/lib/hub-supported-versions.js new file mode 100644 index 000000000..71f2bcafc --- /dev/null +++ b/lib/hub-supported-versions.js @@ -0,0 +1,194 @@ +// Keeps the "Supported versions" lists in sync with the Stackable Hub. +// +// At build time the supported-versions.adoc partial of every product operator +// module is generated from https://hub.stackable.tech/api/v1/components/. +// +// * Released docs versions map to that SDP release +// * Nightly maps to the _next_ upcoming release if there is one +// * If there is no upcoming release yet on the Hub it will say so ("not decided yet") +// +// We usually have a gap after a release was made before we decide on the versions for the next one. +// +// Because we need the partials there is a cache of Hub data which is used while/if it is unavailable. +// +// Useful links: +// Extensions: https://docs.antora.org/antora/latest/extend/extensions/ +// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/ +'use strict' + +const fs = require('fs') +const ospath = require('path') + +const HUB_API = 'https://hub.stackable.tech/api/v1/components' +const PARTIAL = 'supported-versions.adoc' +const HUB_TIMEOUT_MS = 10_000 + +// docs module name -> Hub component slug +const MODULE_TO_SLUG = { + airflow: 'airflow', + druid: 'druid', + hbase: 'hbase', + hdfs: 'hdfs', + hive: 'hive', + kafka: 'kafka', + nifi: 'nifi', + opa: 'opa', + opensearch: 'opensearch', + 'spark-k8s': 'spark', + superset: 'superset', + trino: 'trino', + zookeeper: 'zookeeper', +} + +const STATUS_SUFFIX = { + lts: ' (LTS)', + deprecated: ' (deprecated)', + experimental: ' (experimental)', + preview: ' (preview)', +} + +module.exports.register = function () { + const logger = this.getLogger('hub-supported-versions') + + // contentClassified rather than contentAggregated: the content catalog is what + // can add a file, and partials are resolved later, when pages are converted. + this.once('contentClassified', async ({ playbook, contentCatalog }) => { + const cacheDir = ospath.join(playbook.dir || '.', playbook.runtime.cacheDir || './cache', 'hub') + const components = await fetchComponents(cacheDir, logger) + + const component = contentCatalog.getComponent('home') + if (!component) return logger.info('no home component, nothing to do') + + let written = 0 + for (const { version } of component.versions) { + for (const [moduleName, slug] of Object.entries(MODULE_TO_SLUG)) { + const existing = contentCatalog.getById({ + component: 'home', version, module: moduleName, family: 'partial', relative: PARTIAL, + }) + // A module we do not carry in this docs version at all: nothing includes + // the partial, so do not invent one. + if (!existing && !contentCatalog.getById({ + component: 'home', version, module: moduleName, family: 'page', relative: 'index.adoc', + })) continue + + const body = renderPartial({ + components, slug, version, logger, hasRepoCopy: Boolean(existing), + }) + if (!body) continue // the repo ships a copy and the Hub has nothing better + + if (existing) { + existing.contents = Buffer.from(body, 'utf8') + } else { + contentCatalog.addFile({ + contents: Buffer.from(body, 'utf8'), + src: { component: 'home', version, module: moduleName, family: 'partial', relative: PARTIAL }, + }) + } + written++ + } + } + logger.info(`wrote ${written} supported-versions partial(s) from the Hub`) + }) +} + +// Returns the AsciiDoc body, or undefined to mean "leave whatever is there". +function renderPartial ({ components, slug, version, logger, hasRepoCopy }) { + const header = `// Generated at build time from ${HUB_API}/${slug}.\n` + + '// Do not edit: the Portal is the source of truth. See lib/hub-supported-versions.js.\n' + + if (!components) { + // No Hub data at all. An existing partial is better than anything we can say, + // but a missing one still has to resolve or the include fails the build. + return `${header}// The Stackable Hub was unreachable during this build.\n` + + 'NOTE: The supported version list is temporarily unavailable, so there is nothing to list here.\n' + } + + const found = resolveRelease(components[slug], version) + + if (!found || !found.entry.versions || !found.entry.versions.length) { + if (version === 'nightly') { + logger.info(`no public upcoming release for ${slug}, rendering the undecided note on nightly`) + return `${header}// No upcoming SDP release is public yet, so there is nothing to list.\n` + + 'NOTE: The product versions for the next Stackable Data Platform release have not been\n' + + 'decided yet, so there is nothing to list here.\n' + } + // A docs version the Hub knows nothing about. Prefer the repo's own copy, + // but if there is none we still have to emit something: an include with no + // target fails the build, and generating these is what lets the operator + // repos delete theirs in the first place. + if (hasRepoCopy) { + logger.info(`no Hub data for ${slug} in SDP ${version}, keeping the partial from the repo`) + return undefined + } + logger.info(`no Hub data for ${slug} in SDP ${version} and no copy in the repo`) + return `${header}// The Hub has no data for SDP ${version}.\n` + + `NOTE: The supported version list for SDP ${version} is unavailable, so there is nothing\n` + + 'to list here.\n' + } + + const lines = found.entry.versions.map((v) => `- ${v.version}${STATUS_SUFFIX[v.status] || ''}`) + const label = found.entry.release || 'next' + const provisional = found.provisional + ? `// Provisional: SDP ${label} has not been released yet.\n` + + `NOTE: These are the planned product versions for SDP ${label}. They may still change.\n\n` + : '' + return header + provisional + lines.join('\n') + '\n' +} + +// Resolves a docs version to a Hub release entry, shipped releases first and +// then the public upcoming ones. Nightly is not a release identifier, so it maps +// to whichever upcoming release is next. +// +// Whether the entry came from upcomingReleases is what makes a list provisional +// - not whether the docs version is nightly. A release branch is often cut and +// built before its SDP release ships, so those docs need the upcoming data too. +function resolveRelease (component, version) { + if (version === 'nightly') { + const next = nextUpcoming(component) + return next ? { entry: next, provisional: true } : undefined + } + const shipped = (component?.releases || []).find((r) => r.release === version) + if (shipped) return { entry: shipped, provisional: false } + const upcoming = (component?.upcomingReleases || []).find((r) => r.release === version) + return upcoming ? { entry: upcoming, provisional: true } : undefined +} + +// The next public upcoming release: earliest planned date, falling back to the order the Hub returned. +// The Hub only lists upcoming releases it considers public, so anything here is safe to show. +function nextUpcoming (component) { + const upcoming = component?.upcomingReleases || [] + if (upcoming.length < 2) return upcoming[0] + return [...upcoming].sort((a, b) => + String(a.plannedReleaseDate || '9999').localeCompare(String(b.plannedReleaseDate || '9999')))[0] +} + +async function fetchComponents (cacheDir, logger) { + const cacheFile = ospath.join(cacheDir, 'components.json') + try { + const { components: list } = await getJson(`${HUB_API}`) + const components = {} + for (const { slug } of list) { + components[slug] = await getJson(`${HUB_API}/${slug}`) + } + fs.mkdirSync(cacheDir, { recursive: true }) + fs.writeFileSync(cacheFile, JSON.stringify(components)) + return components + } catch (err) { + logger.info(`could not fetch ${HUB_API} (${err.message}), trying cache`) + try { + return JSON.parse(fs.readFileSync(cacheFile, 'utf8')) + } catch { + logger.info('no cached Hub data available') + return undefined + } + } +} + +// A Hub that accepts the connection and never answers is not an error, so it +// would otherwise stall the docs build indefinitely rather than falling back to +// the cache. An abort surfaces as a rejection, which the caller already handles. +async function getJson (url) { + const response = await fetch(url, { signal: AbortSignal.timeout(HUB_TIMEOUT_MS) }) + if (!response.ok) throw new Error(`${url} returned ${response.status}`) + return await response.json() +} diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 1469b3fd9..c8550b6fe 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/only-dev-antora-playbook.yml b/only-dev-antora-playbook.yml index 14f5e6813..06fccadea 100644 --- a/only-dev-antora-playbook.yml +++ b/only-dev-antora-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/truly-local-playbook.yml b/truly-local-playbook.yml index 220fd3512..e7e74b1ce 100644 --- a/truly-local-playbook.yml +++ b/truly-local-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: