diff --git a/package.json b/package.json index 1f26bab5..0f7e2c49 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,8 @@ "test:negative-cache": "node tests/negative-cache/test-runner.js", "test:head-meta": "node --test tests/head-meta/*.test.js", "test:property-tooltips": "node --test tests/property-tooltips/*.test.js", - "test:all": "npm run test:playground && npm run test:interactive && npm run test:negative-cache && npm run test:head-meta && npm run test:property-tooltips", + "test:kapa-source-groups": "node --test tests/kapa-source-groups/*.test.js", + "test:all": "npm run test:playground && npm run test:interactive && npm run test:negative-cache && npm run test:head-meta && npm run test:property-tooltips && npm run test:kapa-source-groups", "build:wasm": "cd blobl-editor/wasm && GOOS=js GOARCH=wasm go build -o ../../src/static/blobl.wasm .", "copy:wasm-exec": "cp \"$(go env GOROOT)/lib/wasm/wasm_exec.js\" src/js/vendor/", "serve:playground": "npx serve ." diff --git a/src/helpers/get-kapa-source-groups.js b/src/helpers/get-kapa-source-groups.js new file mode 100644 index 00000000..0b56f5f7 --- /dev/null +++ b/src/helpers/get-kapa-source-groups.js @@ -0,0 +1,229 @@ +'use strict' + +/** + * Resolves the Kapa source group that scopes Ask AI retrieval to the docs + * version the reader is actually on. + * + * WHY THIS EXISTS + * --------------- + * Kapa indexes one separately crawled source per published docs version. With no + * scoping, a question is answered from any of them. Measured against the live + * retrieval API for "hardware requirements for enterprise redpanda self-hosted", + * the exact question in DOC-2450: 11 results spread across 24.2, 24.3, 25.1, + * 25.2, 25.3 and current, with only ONE from current. Scoped to the current + * group, 14 of 14 came from current. + * + * HOW THE SEGMENT IS DERIVED + * -------------------------- + * From `page.url`, not from page.version. The two disagree for the latest + * release: `latest_version_segment: 'current'` publishes 26.2 at + * /streaming/current/ while page.version reads 26.2. The Kapa mapping is keyed on + * the URL segment because that is what Kapa's own source_url values use, so + * reading the URL avoids having to know which version is currently latest. + * + * WHAT IT RETURNS + * --------------- + * An array, because that is the shape both Kapa providers want + * (sourceGroupIdsInclude on AgentProvider, sourceGroupIDsInclude on + * KapaProvider). An EMPTY array means "send no filter", which is the pre-DOC-2450 + * behaviour: Kapa searches everything. That is the deliberate degradation for any + * case where scoping cannot be resolved, because a wrong group is worse than no + * group -- scoping to a group that does not hold the reader's version returns + * only Kapa's global sources, so the reader gets no version-specific content at + * all and no error either. + * + * Unversioned pages (Cloud, Connect, Agentic Data Plane, labs, home, search, the + * 404 page) resolve to the mapping's default_segment rather than to nothing. + * Sending no filter there is what produced DOC-2450 in the first place: the + * reporter was on a page with no version of its own. + * + * Note that scoping to a version group does NOT hide Cloud, Connect or Agentic + * Data Plane content. Those sources are deliberately left unassigned in Kapa, so + * they are "global" and come through alongside whichever group is selected. + * Verified live: scoped to the 25.2 group, an Agentic Data Plane question + * returned 10 of 10 results from /agentic-data-plane/. + * + * Usage in templates: + * window.KAPA_SOURCE_GROUP_IDS = [ + * {{#each (get-kapa-source-groups)}}"{{{this}}}"{{#unless @last}},{{/unless}}{{/each}} + * ]; + * + * @param {object} options - Handlebars options with data.root.page and data.root.site + * @returns {string[]} Zero or one Kapa source group id + */ +/** + * Two outputs from one resolution, selected by an optional mode argument: + * + * {{#each (get-kapa-source-groups)}} -> array of zero or one group id + * {{get-kapa-source-groups 'segment'}} -> the segment that group represents + * + * One helper rather than two, because Antora compiles every UI helper in + * isolation and a helper CANNOT require a sibling helper: doing so fails at + * page-composition time with a fatal "Cannot find module" and takes the whole + * build down. Copying the resolution into a second file instead would + * reintroduce exactly the disagreement this is here to prevent. + * + * Handlebars passes params before the options object, so with no argument the + * first parameter IS the options object. + */ +module.exports = function (mode, options) { + const opts = options === undefined ? mode : options + const { segment, groupId } = resolve(opts) + // Only name a segment when a group is genuinely being sent, so the agent + // prompt cannot claim a restriction that is not in force. + const segmentOut = groupId && segment ? segment : '' + const idsOut = groupId ? [groupId] : [] + switch (mode) { + case 'segment': return segmentOut + // The two JSON modes are what the template uses. They exist so that no raw + // value is ever interpolated into an executable in one would end the + // string or the script element. Handlebars' triple-stash does no escaping + // at all inside ` inside a value would close the script element early and + * dump the rest into the document as markup. Same treatment the docs-site edge + * function gives the same value for the /api/ pages. + * + * @param {*} value + * @returns {string} A JavaScript expression safe to place inside in one would end the string or the script element. --}} + window.KAPA_SOURCE_GROUP_IDS = window.KAPA_SOURCE_GROUP_IDS || {{{get-kapa-source-groups 'json'}}}; + {{!-- The segment that the group above represents, so the agent prompt can + name the version its answers came from without deriving it again from + the URL. A second derivation disagrees: /streaming/26.2/ reads as + "26.2" while the group sent is `current`. Empty string means nothing + was scoped, so the prompt must not claim a restriction. --}} + window.KAPA_SOURCE_GROUP_SEGMENT = window.KAPA_SOURCE_GROUP_SEGMENT || {{{get-kapa-source-groups 'segment-json'}}}; {{!-- Signed-in agent tier: per-component prompts that showcase the agent tools --}} window.AGENT_SUGGESTIONS = window.AGENT_SUGGESTIONS || [ {{#each (get-agent-suggestions)}} diff --git a/tests/kapa-source-groups/kapa-scope-fallback.test.js b/tests/kapa-source-groups/kapa-scope-fallback.test.js new file mode 100644 index 00000000..c90463d4 --- /dev/null +++ b/tests/kapa-source-groups/kapa-scope-fallback.test.js @@ -0,0 +1,142 @@ +'use strict' + +// Dropping the version scope when Kapa rejects the source group (DOC-1807, +// DOC-2450 review round 2). +// +// The design assumed a stale group id degrades silently to Kapa's global +// sources. Measured live it does not: the Chat SDK's query endpoint answers +// HTTP 400 {"source_group_ids_include":["Invalid pk ... object does not +// exist."]} and the Agent SDK throws "Agent request failed: 400 ..." with that +// body. Without this module every question on the affected pages fails until +// the regenerated mapping ships through three repos. + +const test = require('node:test') +const assert = require('node:assert/strict') +const path = require('node:path') +const fs = require('node:fs') + +const ROOT = path.join(__dirname, '..', '..') +const scope = require(path.join(ROOT, 'src/js/react/kapaScope.js')) + +// A minimal window for the module's globals and events. +function fakeWindow (ids) { + const listeners = {} + return { + KAPA_SOURCE_GROUP_IDS: ids, + KAPA_SOURCE_GROUP_SEGMENT: '25.2', + addEventListener: (n, fn) => { (listeners[n] = listeners[n] || []).push(fn) }, + dispatchEvent: (ev) => { (listeners[ev.type] || []).forEach((fn) => fn(ev)); return true }, + _listeners: listeners, + CustomEvent: class CustomEvent { constructor (type, init) { this.type = type; this.detail = init && init.detail } }, + } +} + +test.beforeEach(() => { + global.window = fakeWindow(['grp-252']) +}) +test.afterEach(() => { delete global.window }) + +test('isScopeRejection matches the Agent SDK message for a stale group, and nothing else', () => { + assert.equal(scope.isScopeRejection('Agent request failed: 400 {"source_group_ids_include":["Invalid pk \\"0000\\" - object does not exist."]}'), true) + assert.equal(scope.isScopeRejection('Agent request failed: 500 Internal Server Error'), false) + assert.equal(scope.isScopeRejection('Network error while fetching answer.'), false) + for (const v of [undefined, null, 42, {}]) assert.equal(scope.isScopeRejection(v), false) +}) + +test('chatSdkErrorMayBeScopeRejection needs a sent scope, no streamed bytes, and the generic error', () => { + const generic = 'Something went wrong. If the issue persists reach out to support.' + assert.equal(scope.chatSdkErrorMayBeScopeRejection(generic, true, false), true) + // No scope was sent, so the group cannot be the cause. + assert.equal(scope.chatSdkErrorMayBeScopeRejection(generic, false, false), false) + // Bytes streamed, so the request was accepted; a mid-stream failure is not a rejection. + assert.equal(scope.chatSdkErrorMayBeScopeRejection(generic, true, true), false) + // The SDK names captcha, rate-limit and network failures distinctly. + for (const other of [ + 'We noticed unusual activity. Please try asking your question again.', + 'There have been too many requests, please try again in a minute.', + 'Network error while fetching answer.', + ]) assert.equal(scope.chatSdkErrorMayBeScopeRejection(other, true, false), false, other) +}) + +test('dropScope clears both globals, announces once, and reports whether anything was dropped', () => { + const seen = [] + window.addEventListener(scope.SCOPE_DROPPED_EVENT, (ev) => seen.push(ev.detail.reason)) + const warn = console.warn + console.warn = () => {} + try { + assert.equal(scope.dropScope('400 from Kapa'), true) + assert.deepEqual(window.KAPA_SOURCE_GROUP_IDS, []) + assert.equal(window.KAPA_SOURCE_GROUP_SEGMENT, '') + assert.deepEqual(seen, ['400 from Kapa']) + // Already dropped: nothing to announce, no second event. + assert.equal(scope.dropScope('again'), false) + assert.deepEqual(seen, ['400 from Kapa']) + } finally { console.warn = warn } +}) + +test('readScopeIds keeps only non-empty strings', () => { + window.KAPA_SOURCE_GROUP_IDS = ['grp', '', null, 7, 'grp2'] + assert.deepEqual(scope.readScopeIds(), ['grp', 'grp2']) + window.KAPA_SOURCE_GROUP_IDS = 'nope' + assert.deepEqual(scope.readScopeIds(), []) +}) + +test('wrapScopeFallback leaves callbacks alone when no scope is sent', () => { + const cbs = { onError: () => {} } + assert.equal(scope.wrapScopeFallback({ query: 'q' }, cbs), cbs) + assert.equal(scope.wrapScopeFallback({ query: 'q', sourceGroupIDsInclude: [] }, cbs), cbs) +}) + +test('wrapScopeFallback turns a pre-stream generic failure into a dropped scope and the drop message', () => { + const warn = console.warn + console.warn = () => {} + try { + const errors = [] + const wrapped = scope.wrapScopeFallback({ sourceGroupIDsInclude: ['grp-252'] }, { onError: (m) => errors.push(m) }) + wrapped.onError('Something went wrong. If the issue persists reach out to support.') + assert.deepEqual(errors, [scope.SCOPE_DROPPED_MESSAGE]) + assert.deepEqual(window.KAPA_SOURCE_GROUP_IDS, []) + } finally { console.warn = warn } +}) + +test('wrapScopeFallback passes through a failure after bytes streamed, and every other error verbatim', () => { + const errors = [] + const started = [] + const wrapped = scope.wrapScopeFallback( + { sourceGroupIDsInclude: ['grp-252'] }, + { onError: (m) => errors.push(m), onStreamStart: () => started.push('start'), onFirstToken: () => started.push('token') } + ) + wrapped.onStreamStart() + wrapped.onFirstToken() + wrapped.onError('Something went wrong. If the issue persists reach out to support.') + wrapped.onError('Network error while fetching answer.') + assert.deepEqual(started, ['start', 'token']) + assert.deepEqual(errors, [ + 'Something went wrong. If the issue persists reach out to support.', + 'Network error while fetching answer.', + ]) + assert.deepEqual(window.KAPA_SOURCE_GROUP_IDS, ['grp-252'], 'scope must survive a mid-stream failure') +}) + +test('wrapScopeFallback tolerates callbacks the SDK did not supply', () => { + const wrapped = scope.wrapScopeFallback({ sourceGroupIDsInclude: ['grp-252'] }, {}) + assert.doesNotThrow(() => { wrapped.onStreamStart(); wrapped.onFirstToken(); wrapped.onError('x') }) +}) + +test('the app re-renders the providers without the prop after a drop', () => { + // Structural: App must hold the scope in state fed by SCOPE_DROPPED_EVENT and + // pass it to sourceGroupProps, or the providers keep sending the dead group. + const askai = fs.readFileSync(path.join(ROOT, 'src/js/react/AskAI.jsx'), 'utf8') + assert.match(askai, /const scopeIds = useKapaScopeIds\(\)/) + assert.match(askai, /sourceGroupProps\('agent', scopeIds\)/) + assert.match(askai, /sourceGroupProps\('chat', scopeIds\)/) + assert.match(askai, /addEventListener\(SCOPE_DROPPED_EVENT/) + // The agent tier detects the rejection from its error event. + assert.match(askai, /case 'response_error':[\s\S]*?isScopeRejection\(event\.data\.error\)\) dropScope/) + // The chat tier shows the drop message and retries once instead of blaming the captcha. + const chat = fs.readFileSync(path.join(ROOT, 'src/js/react/components/ChatSdkInterface.jsx'), 'utf8') + assert.match(chat, /error === SCOPE_DROPPED_MESSAGE/) + assert.match(chat, /handleRetry\(latestQA\.question\)/) + const service = fs.readFileSync(path.join(ROOT, 'src/js/react/persistentApiService.js'), 'utf8') + assert.match(service, /wrapScopeFallback\(enhancedArgs, callbacks\)/) +}) diff --git a/tests/kapa-source-groups/kapa-source-groups.test.js b/tests/kapa-source-groups/kapa-source-groups.test.js new file mode 100644 index 00000000..523dc921 --- /dev/null +++ b/tests/kapa-source-groups/kapa-source-groups.test.js @@ -0,0 +1,411 @@ +'use strict' + +// Verifies version-scoped Ask AI retrieval (DOC-1807, DOC-2450) end to end +// through the REAL helper and the REAL chat-panel.hbs partial, not stand-ins. +// +// The bug being fixed, measured against Kapa's live retrieval API for the exact +// question in DOC-2450 ("hardware requirements for enterprise redpanda +// self-hosted"): unscoped, 11 results spread across 24.2, 24.3, 25.1, 25.2, 25.3 +// and current, with only ONE from current. Scoped to the current group, 14 of 14 +// came from current. +// +// Two things here are easy to get wrong and fail silently, so both are pinned: +// +// 1. The segment must come from page.url, not page.version. +// latest_version_segment: 'current' publishes 26.2 at /streaming/current/, +// so page.version says 26.2 while the Kapa mapping is keyed on 'current'. +// Deriving from page.version would look right and miss every latest-version +// reader, which is precisely the DOC-2450 population. +// +// 2. The two SDKs spell the option differently on purpose +// (sourceGroupIdsInclude vs sourceGroupIDsInclude). An unknown React prop is +// ignored with no error, so a typo means no filter is sent and answers come +// from every version again. + +const test = require('node:test') +const assert = require('node:assert/strict') +const path = require('node:path') +const fs = require('node:fs') +const Handlebars = require('handlebars') + +const ROOT = path.join(__dirname, '..', '..') +const helper = require(path.join(ROOT, 'src/helpers/get-kapa-source-groups.js')) +const { versionSegmentFromUrl } = helper + +// Antora compiles each helper in isolation, so the segment is a MODE of the one +// helper rather than a second file that requires it. A sibling require fails at +// page-composition time with a fatal error and takes the whole build down. +const segmentHelper = (opts) => helper('segment', opts) + +// A trimmed copy of the shape doc-tools' generate kapa-source-groups emits. +const MAPPING = { + project_id: '97f44223-f930-4fb9-ae1e-ecd436a4d85c', + parent_group: { id: '238b3c08', name: 'Streaming', type: 'product' }, + default_segment: 'current', + segments: { + '24.2': { group_id: 'grp-242', group_name: '24.2', source_ids: ['s1'], source_names: ['Documentation (24.2)'] }, + '25.2': { group_id: 'grp-252', group_name: '25.2', source_ids: ['s2'], source_names: ['Documentation (25.2)'] }, + current: { group_id: 'grp-cur', group_name: 'current', source_ids: ['s3'], source_names: ['Documentation (current)'] }, + }, + global_sources: ['Agentic Data Plane', 'Documentation (Cloud)'], +} + +const call = (page, { mapping = MAPPING, where = 'componentVersion' } = {}) => { + const root = { page: { ...page } } + const raw = mapping === null ? undefined : JSON.stringify(mapping) + if (where === 'componentVersion') root.page.componentVersion = { asciidoc: { attributes: { 'kapa-source-groups': raw } } } + if (where === 'component') root.page.component = { asciidoc: { attributes: { 'kapa-source-groups': raw } } } + if (where === 'site') root.site = { asciidoc: { attributes: { 'kapa-source-groups': raw } } } + if (where === 'none') { /* no attribute anywhere */ } + return helper({ data: { root } }) +} + +const SEGS = MAPPING.segments +const seg = (url) => versionSegmentFromUrl(url, SEGS) + +test('versionSegmentFromUrl reads the URL segment, which is what the mapping is keyed on', () => { + assert.equal(seg('/streaming/25.2/get-started/intro/'), '25.2') + // The latest release publishes at /streaming/current/ even though page.version + // is 26.2. Getting this from the URL is the whole point. + assert.equal(seg('/streaming/current/get-started/intro/'), 'current') + // Unversioned components have no segment. + assert.equal(seg('/cloud-data-platform/get-started/'), null) + assert.equal(seg('/agentic-data-plane/reference/'), null) + assert.equal(seg('/connect/components/'), null) + assert.equal(seg('/home/'), null) + // Junk must not throw. + for (const v of [undefined, null, '', 42, {}, '/streaming/']) assert.equal(seg(v), null) + // No segments map: degrade, do not throw. + assert.equal(versionSegmentFromUrl('/streaming/25.2/x/', null), null) +}) + +test('versionSegmentFromUrl also reads the pre-rename layout', () => { + // The docs component was renamed ROOT -> streaming, which moved every + // versioned page from // to /streaming//. A build over + // pre-rename branches emitted 451 pages of 24.3 content scoped to current, + // silently, because the old function matched a hardcoded /streaming/ prefix. + assert.equal(seg('/24.2/manage/monitoring/'), '24.2') + assert.equal(seg('/current/manage/monitoring/'), 'current') +}) + +test('versionSegmentFromUrl recognises only real segments, never a path word', () => { + // Driven off the mapping's keys, so it cannot mistake an ordinary path + // component for a version. + assert.equal(seg('/cloud-data-platform/manage/cluster/'), null) + assert.equal(seg('/streaming/beta/get-started/'), null, 'beta is not in this mapping') + // Only the first two positions are considered, so a version-shaped word deep + // in a path cannot hijack the scope. + assert.equal(seg('/connect/components/outputs/25.2/'), null) + // A file named after a version is not a segment. + assert.equal(seg('/streaming/25.2.json'), null) + assert.equal(seg('/25.2.html'), null) +}) + +test('a versioned page resolves to its own version group', () => { + assert.deepEqual(call({ url: '/streaming/25.2/manage/monitoring/' }), ['grp-252']) + assert.deepEqual(call({ url: '/streaming/24.2/manage/monitoring/' }), ['grp-242']) +}) + +test('the latest version resolves via its URL segment, not page.version', () => { + // page.version deliberately disagrees with the URL, as it does in production. + const got = call({ url: '/streaming/current/get-started/intro/', version: '26.2' }) + assert.deepEqual(got, ['grp-cur']) +}) + +test('unversioned pages resolve to the default segment, which is the DOC-2450 fix', () => { + // The reporter was on a page with no version. Sending no filter there is what + // let 25.2 content answer a latest-version question. + for (const url of ['/cloud-data-platform/get-started/', '/agentic-data-plane/reference/', '/connect/components/', '/home/', '/search/']) { + assert.deepEqual(call({ url }), ['grp-cur'], `expected default group for ${url}`) + } +}) + +test('a published version with no group falls back to the default rather than sending nothing', () => { + // This is the drift case: 26.3 published, nobody made the Kapa group yet. + // Falling back to current beats searching all nine versions at once. + assert.deepEqual(call({ url: '/streaming/26.3/get-started/intro/' }), ['grp-cur']) +}) + +test('reads the mapping from site.keys, the only channel that reaches the 404 page', () => { + // 404.hbs has no page.component and no page.componentVersion, yet it renders + // the Ask AI panel. Without site.keys it searched every docs version. Verified + // in a real Antora build: 404.html now emits the default group. + const root = { page: { url: '/nonexistent/' }, site: { keys: { 'kapa-source-groups': JSON.stringify(MAPPING) } } } + assert.deepEqual(helper({ data: { root } }), ['grp-cur']) + + // With no page object at all, which is closer to what the 404 model provides. + const bare = { site: { keys: { 'kapa-source-groups': JSON.stringify(MAPPING) } } } + assert.deepEqual(helper({ data: { root: bare } }), ['grp-cur']) +}) + +test('a component attribute still wins over site.keys', () => { + const other = { ...MAPPING, segments: { ...MAPPING.segments, '25.2': { group_id: 'grp-override' } } } + const root = { + page: { url: '/streaming/25.2/x/', componentVersion: { asciidoc: { attributes: { 'kapa-source-groups': JSON.stringify(other) } } } }, + site: { keys: { 'kapa-source-groups': JSON.stringify(MAPPING) } }, + } + assert.deepEqual(helper({ data: { root } }), ['grp-override']) +}) + +test('reads the mapping from componentVersion, component or site attributes', () => { + for (const where of ['componentVersion', 'component', 'site']) { + assert.deepEqual(call({ url: '/streaming/25.2/x/' }, { where }), ['grp-252'], `from ${where}`) + } +}) + +test('accepts an already-parsed object, not only a JSON string', () => { + const root = { page: { url: '/streaming/25.2/x/', componentVersion: { asciidoc: { attributes: { 'kapa-source-groups': MAPPING } } } } } + assert.deepEqual(helper({ data: { root } }), ['grp-252']) +}) + +test('degrades to no filter rather than throwing, in every unresolvable case', () => { + // A wrong group is worse than no group: scoping to a group that does not hold + // the reader's version returns only Kapa's global sources, silently. + assert.deepEqual(call({ url: '/streaming/25.2/x/' }, { where: 'none' }), [], 'no attribute') + assert.deepEqual(helper({ data: { root: {} } }), [], 'no page at all') + assert.deepEqual(helper({ data: {} }), [], 'no root') + assert.deepEqual(helper({}), [], 'no data') + assert.deepEqual(helper(undefined), [], 'no options') + + // Malformed attribute: a broken helper is a broken build, so it must not throw. + const bad = { page: { url: '/streaming/25.2/x/', componentVersion: { asciidoc: { attributes: { 'kapa-source-groups': '{not json' } } } } } + assert.deepEqual(helper({ data: { root: bad } }), []) + + // Mapping present but shaped wrong. + assert.deepEqual(call({ url: '/streaming/25.2/x/' }, { mapping: {} }), []) + assert.deepEqual(call({ url: '/streaming/25.2/x/' }, { mapping: { segments: {} } }), []) + // default_segment pointing at a segment that does not exist. + assert.deepEqual(call({ url: '/nope/' }, { mapping: { default_segment: 'gone', segments: MAPPING.segments } }), []) +}) + +test('chat-panel.hbs emits the group id as a JS array literal', () => { + const source = fs.readFileSync(path.join(ROOT, 'src/partials/chat-panel.hbs'), 'utf8') + // Only the config , no unescaped quote, and it round-trips to the value. + assert.doesNotMatch(json, /<\/script>/i) + assert.doesNotMatch(json, /