diff --git a/ts/packages/agents/markdown/src/agent/documentOperations.ts b/ts/packages/agents/markdown/src/agent/documentOperations.ts new file mode 100644 index 0000000000..1f826bdde8 --- /dev/null +++ b/ts/packages/agents/markdown/src/agent/documentOperations.ts @@ -0,0 +1,567 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Applies DocumentOperation values against the raw Markdown *string*, using +// character offsets into that string. This module is the authoritative +// applier for both the headless path (no view process) and for the +// server-authoritative apply in the view process. Callers computed the +// offsets against the same raw Markdown they read via getDocumentContent +// and paired the apply with the SHA-256 of that base content, so the +// service rejects the apply when the current Markdown hashes differently. + +import type { + ContentItem, + DocumentOperation, + MarkItem, +} from "./markdownOperationSchema.js"; + +export function applyDocumentOperations( + content: string, + operations: DocumentOperation[], +): string { + const orderedOperations = orderBaseRelativeOperations( + operations, + content.length, + ); + return orderedOperations.reduce( + (updatedContent, operation) => + applyDocumentOperation(updatedContent, operation), + content, + ); +} + +type OperationSpan = { + operation: DocumentOperation; + index: number; + from: number; + to: number; +}; + +function orderBaseRelativeOperations( + operations: DocumentOperation[], + contentLength: number, +): DocumentOperation[] { + const spans = operations.map((operation, index) => { + const [from, to] = + operation.type === "insert" + ? [ + validatePosition(operation.position, contentLength), + operation.position, + ] + : validateRange(operation.from, operation.to, contentLength); + return { operation, index, from, to }; + }); + + for (let leftIndex = 0; leftIndex < spans.length; leftIndex += 1) { + for ( + let rightIndex = leftIndex + 1; + rightIndex < spans.length; + rightIndex += 1 + ) { + if (operationsOverlap(spans[leftIndex], spans[rightIndex])) { + throw new Error("Document operations must not overlap"); + } + } + } + + return spans + .sort((left, right) => { + const positionOrder = right.from - left.from; + if (positionOrder !== 0) { + return positionOrder; + } + if (left.from === left.to && right.from !== right.to) { + return 1; + } + if (right.from === right.to && left.from !== left.to) { + return -1; + } + return right.index - left.index; + }) + .map(({ operation }) => operation); +} + +function operationsOverlap(left: OperationSpan, right: OperationSpan): boolean { + if (left.from === left.to) { + return right.from < left.from && left.from < right.to; + } + if (right.from === right.to) { + return left.from < right.from && right.from < left.to; + } + return left.from < right.to && right.from < left.to; +} + +function applyDocumentOperation( + content: string, + operation: DocumentOperation, +): string { + switch (operation.type) { + case "insert": { + const position = validatePosition( + operation.position, + content.length, + ); + return ( + content.slice(0, position) + + contentItemsToText(operation.content) + + content.slice(position) + ); + } + case "replace": { + const [from, to] = validateRange( + operation.from, + operation.to, + content.length, + ); + return ( + content.slice(0, from) + + contentItemsToText(operation.content) + + content.slice(to) + ); + } + case "delete": { + const [from, to] = validateRange( + operation.from, + operation.to, + content.length, + ); + return content.slice(0, from) + content.slice(to); + } + case "format": { + const [from, to] = validateRange( + operation.from, + operation.to, + content.length, + ); + return operation.add + ? addFormatMarks(content, from, to, operation.marks) + : removeFormatMarks(content, from, to, operation.marks); + } + } +} + +function contentItemsToText(items: ContentItem[]): string { + return items.map((item) => contentItemToText(item)).join(""); +} + +function contentItemToText(item: ContentItem): string { + const text = getPlainText(item); + switch (item.type) { + case "heading": { + if (/^#{1,6}\s/.test(text)) { + return ensureBlockSeparator(text); + } + const attrs = item.attrs as { level?: number } | undefined; + const requestedLevel = attrs?.level; + const level = + requestedLevel !== undefined && + Number.isInteger(requestedLevel) && + requestedLevel >= 1 && + requestedLevel <= 6 + ? requestedLevel + : 1; + return `${"#".repeat(level)} ${text}\n\n`; + } + case "paragraph": + return ensureBlockSeparator(text); + case "bullet_list": + return serializeList(item, "-"); + case "ordered_list": + return serializeList(item, "1."); + case "code_block": + return `\`\`\`\n${text}\n\`\`\`\n\n`; + case "blockquote": + return `${text + .split("\n") + .map((line) => `> ${line}`) + .join("\n")}\n\n`; + case "horizontal_rule": + return "---\n\n"; + case "hard_break": + return " \n"; + case "text": + return applyMarks(text, item); + default: + return text; + } +} + +function getPlainText(item: ContentItem): string { + if (item.text !== undefined) { + return item.text; + } + return item.content ? item.content.map(getPlainText).join("") : ""; +} + +function ensureBlockSeparator(text: string): string { + return text.endsWith("\n\n") ? text : `${text}\n\n`; +} + +function serializeList(item: ContentItem, marker: string): string { + const lines = + item.content?.map( + (child) => `${marker} ${getPlainText(child).trim()}`, + ) ?? []; + return `${lines.join("\n")}\n\n`; +} + +function applyMarks(text: string, item: ContentItem): string { + return (item.marks ?? []).reduce( + (markedText, mark) => wrapWithMark(markedText, mark), + text, + ); +} + +// Markdown wrapper for a single MarkItem. `symmetric` marks use identical left +// and right delimiters and, for removal, may also accept a set of alternate +// GFM-valid delimiters (e.g. `_em_` and `__strong__`). `code` marks use a +// backtick run chosen at wrap time so a run inside the selected text can +// never terminate the span, plus code-span padding when the content begins +// or ends with a backtick or spaces. `link` marks emit `[text](href)`; when +// the LLM did not supply `attrs.href` the mark is dropped so we never emit +// a link with an empty target. +type SymmetricMarkWrapper = { + kind: "symmetric"; + delimiter: string; + alternates?: readonly string[]; +}; +type MarkWrapper = + | SymmetricMarkWrapper + | { kind: "code" } + | { kind: "link"; href: string }; + +function markWrapper(mark: MarkItem): MarkWrapper | undefined { + switch (mark.type) { + case "strong": + return { + kind: "symmetric", + delimiter: "**", + alternates: ["__"], + }; + case "em": + return { + kind: "symmetric", + delimiter: "*", + alternates: ["_"], + }; + case "code": + return { kind: "code" }; + case "link": { + const attrs = mark.attrs as { href?: string } | undefined; + if (!attrs?.href) { + return undefined; + } + return { kind: "link", href: attrs.href }; + } + default: + return undefined; + } +} + +// Pick the shortest backtick run strictly longer than any run already in +// `text`. That is the canonical CommonMark rule: no interior run can close +// the span, so a selection containing "`" gets wrapped in "``", "``" in +// "```", and so on. +function codeSpanDelimiter(text: string): string { + const runs = text.match(/`+/g); + let longest = 0; + if (runs) { + for (const run of runs) { + if (run.length > longest) { + longest = run.length; + } + } + } + return "`".repeat(longest + 1); +} + +// CommonMark code-span padding: add a single space on each side when the +// content begins or ends with a backtick, so the delimiter run and the +// interior can be told apart by a reader. Also pad when the content is +// entirely spaces so the span is not read as empty. We deliberately do +// NOT pad on plain leading/trailing spaces alone, because those are +// semantic content the user selected. +function shouldPadCodeSpan(text: string): boolean { + if (text.length === 0) { + return false; + } + if (text.startsWith("`") || text.endsWith("`")) { + return true; + } + if (/^ +$/.test(text)) { + return true; + } + return false; +} + +function wrapCodeSpan(text: string): string { + const delimiter = codeSpanDelimiter(text); + const padded = shouldPadCodeSpan(text) ? ` ${text} ` : text; + return `${delimiter}${padded}${delimiter}`; +} + +function wrapWithMark(text: string, mark: MarkItem): string { + const wrapper = markWrapper(mark); + if (wrapper === undefined) { + return text; + } + switch (wrapper.kind) { + case "symmetric": + return `${wrapper.delimiter}${text}${wrapper.delimiter}`; + case "code": + return wrapCodeSpan(text); + case "link": + return `[${text}](${wrapper.href})`; + } +} + +// Add the requested marks around content[from..to]. Marks are applied +// innermost-first to match applyMarks so `[strong, em]` produces +// `*text*`. An empty range or an empty marks list is a +// no-op so callers don't have to guard. +function addFormatMarks( + content: string, + from: number, + to: number, + marks: MarkItem[], +): string { + if (marks.length === 0 || from === to) { + return content; + } + const wrapped = marks.reduce( + (text, mark) => wrapWithMark(text, mark), + content.slice(from, to), + ); + return content.slice(0, from) + wrapped + content.slice(to); +} + +// Remove the requested marks by peeling matching Markdown delimiters that +// immediately surround content[from..to]. Marks are processed +// innermost-first so `[strong, em]` correctly peels `*` then `**` off +// `*text*`. A mark whose delimiter is not present at the +// current boundary is silently skipped, so remove is idempotent when the +// user asked to strip formatting that was never applied. +type Boundaries = { leftPos: number; rightPos: number }; + +function removeFormatMarks( + content: string, + from: number, + to: number, + marks: MarkItem[], +): string { + if (marks.length === 0 || from === to) { + return content; + } + let leftPos = from; + let rightPos = to; + for (const mark of marks) { + const wrapper = markWrapper(mark); + if (wrapper === undefined) { + continue; + } + const peeled = peelMarkWrapper(content, leftPos, rightPos, wrapper); + if (peeled !== undefined) { + leftPos = peeled.leftPos; + rightPos = peeled.rightPos; + } + } + return ( + content.slice(0, leftPos) + + content.slice(from, to) + + content.slice(rightPos) + ); +} + +function peelMarkWrapper( + content: string, + leftPos: number, + rightPos: number, + wrapper: MarkWrapper, +): Boundaries | undefined { + switch (wrapper.kind) { + case "symmetric": + return peelSymmetricDelimiter(content, leftPos, rightPos, wrapper); + case "code": + return peelCodeSpan(content, leftPos, rightPos); + case "link": + return peelLink(content, leftPos, rightPos, wrapper.href); + } +} + +// Peel a symmetric delimiter (or one of its alternates) that surrounds +// content[leftPos..rightPos]. Preferring the canonical delimiter keeps +// existing tests deterministic while still accepting the GFM alternates +// (`__` and `_`) the LLM may emit alongside `**` and `*`. +function peelSymmetricDelimiter( + content: string, + leftPos: number, + rightPos: number, + wrapper: SymmetricMarkWrapper, +): Boundaries | undefined { + const candidates = [wrapper.delimiter, ...(wrapper.alternates ?? [])]; + for (const delimiter of candidates) { + if (isSurroundedBy(content, leftPos, rightPos, delimiter)) { + return { + leftPos: leftPos - delimiter.length, + rightPos: rightPos + delimiter.length, + }; + } + } + return undefined; +} + +function isSurroundedBy( + content: string, + leftPos: number, + rightPos: number, + delimiter: string, +): boolean { + return ( + leftPos >= delimiter.length && + content.slice(leftPos - delimiter.length, leftPos) === delimiter && + rightPos + delimiter.length <= content.length && + content.slice(rightPos, rightPos + delimiter.length) === delimiter + ); +} + +// Peel the outer code-span delimiters and the optional CommonMark +// single-space padding. The delimiter length is discovered from the +// actual backtick run rather than hard-coded, so any pair emitted by +// wrapCodeSpan (`` ` ``, `` `` ``, `` ``` ``, ...) can be undone. +// Padding must be symmetric or absent: wrapCodeSpan only emits both +// spaces together, so an unbalanced pattern is not something we wrote +// and we leave it alone. +function peelCodeSpan( + content: string, + leftPos: number, + rightPos: number, +): Boundaries | undefined { + const left = stripCodeSpanPad(content, leftPos, -1); + const right = stripCodeSpanPad(content, rightPos, 1); + if (left.padded !== right.padded) { + return undefined; + } + const runLength = countRun(content, left.pos, "`", -1); + if (runLength === 0) { + return undefined; + } + if (countRun(content, right.pos, "`", 1) !== runLength) { + return undefined; + } + return { + leftPos: left.pos - runLength, + rightPos: right.pos + runLength, + }; +} + +// Consume a single optional space adjacent to `pos`. `direction` is -1 for +// the left boundary (checking content[pos - 1]) and 1 for the right +// boundary (checking content[pos]). +function stripCodeSpanPad( + content: string, + pos: number, + direction: -1 | 1, +): { pos: number; padded: boolean } { + if (direction === -1) { + if (pos >= 1 && content[pos - 1] === " ") { + return { pos: pos - 1, padded: true }; + } + } else { + if (pos < content.length && content[pos] === " ") { + return { pos: pos + 1, padded: true }; + } + } + return { pos, padded: false }; +} + +// Count consecutive occurrences of `char` starting from `pos`, walking +// left (direction -1) or right (direction 1). Used to discover the actual +// backtick-run length on each side of a code span. +function countRun( + content: string, + pos: number, + char: string, + direction: -1 | 1, +): number { + let count = 0; + if (direction === -1) { + while (pos - (count + 1) >= 0 && content[pos - (count + 1)] === char) { + count += 1; + } + } else { + while (pos + count < content.length && content[pos + count] === char) { + count += 1; + } + } + return count; +} + +// Peel a link wrapper `[text](href)`. Expect `[` immediately before +// leftPos and `](...)` starting at rightPos. Only peel when the full +// pattern is present; malformed links are left alone. +function peelLink( + content: string, + leftPos: number, + rightPos: number, + expectedHref: string, +): Boundaries | undefined { + if ( + leftPos < 1 || + content[leftPos - 1] !== "[" || + content[rightPos] !== "]" || + content[rightPos + 1] !== "(" + ) { + return undefined; + } + const hrefStart = rightPos + 2; + let depth = 1; + let closeParen = -1; + for (let index = hrefStart; index < content.length; index += 1) { + if (content[index] === "\\") { + index += 1; + } else if (content[index] === "(") { + depth += 1; + } else if (content[index] === ")") { + depth -= 1; + if (depth === 0) { + closeParen = index; + break; + } + } + } + if ( + closeParen === -1 || + content.slice(hrefStart, closeParen) !== expectedHref + ) { + return undefined; + } + return { leftPos: leftPos - 1, rightPos: closeParen + 1 }; +} + +function validatePosition(position: number, contentLength: number): number { + if ( + !Number.isInteger(position) || + position < 0 || + position > contentLength + ) { + throw new Error(`Invalid document position: ${position}`); + } + return position; +} + +function validateRange( + from: number, + to: number, + contentLength: number, +): [number, number] { + if ( + !Number.isInteger(from) || + !Number.isInteger(to) || + from < 0 || + to < from || + to > contentLength + ) { + throw new Error(`Invalid document range: ${from}-${to}`); + } + return [from, to]; +} diff --git a/ts/packages/agents/markdown/src/agent/markdownOperationSchema.ts b/ts/packages/agents/markdown/src/agent/markdownOperationSchema.ts index 22f4e3e468..f36039fdab 100644 --- a/ts/packages/agents/markdown/src/agent/markdownOperationSchema.ts +++ b/ts/packages/agents/markdown/src/agent/markdownOperationSchema.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. // Document operation types for incremental updates to ProseMirror documents -// Position references should be line numbers (0-based) in the document. +// Position references are character offsets (0-based) in the markdown text. export type DocumentOperation = | InsertOperation | DeleteOperation diff --git a/ts/packages/agents/markdown/test/documentOperations.spec.ts b/ts/packages/agents/markdown/test/documentOperations.spec.ts new file mode 100644 index 0000000000..7ca538b7a4 --- /dev/null +++ b/ts/packages/agents/markdown/test/documentOperations.spec.ts @@ -0,0 +1,280 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { applyDocumentOperations } from "../src/agent/documentOperations.js"; +import type { DocumentOperation } from "../src/agent/markdownOperationSchema.js"; + +describe("base-relative DocumentOperation batches", () => { + test("applies length-changing operations without shifting later offsets", () => { + const before = + "# Title\n\nAlpha paragraph.\n\nBravo paragraph.\n\nCharlie paragraph.\n"; + const charlieStart = before.indexOf("Charlie"); + const operations: DocumentOperation[] = [ + { + type: "insert", + position: before.indexOf("Alpha"), + content: [{ type: "text", text: "NEW INTRO\n\n" }], + }, + { + type: "delete", + from: charlieStart, + to: before.length, + }, + ]; + + expect(applyDocumentOperations(before, operations)).toBe( + "# Title\n\nNEW INTRO\n\nAlpha paragraph.\n\nBravo paragraph.\n\n", + ); + }); + + test("preserves operation order for inserts at the same position", () => { + const operations: DocumentOperation[] = [ + { + type: "insert", + position: 0, + content: [{ type: "text", text: "first " }], + }, + { + type: "insert", + position: 0, + content: [{ type: "text", text: "second " }], + }, + ]; + + expect(applyDocumentOperations("body", operations)).toBe( + "first second body", + ); + }); + + test("rejects overlapping operations", () => { + const operations: DocumentOperation[] = [ + { type: "delete", from: 0, to: 4 }, + { + type: "replace", + from: 2, + to: 6, + content: [{ type: "text", text: "updated" }], + }, + ]; + + expect(() => applyDocumentOperations("content", operations)).toThrow( + "Document operations must not overlap", + ); + }); +}); + +describe("format DocumentOperation", () => { + test("adds strong marks around a character range", () => { + const before = "hello world"; + const op: DocumentOperation = { + type: "format", + from: 6, + to: 11, + add: true, + marks: [{ type: "strong" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello **world**"); + }); + + test("adds nested em+strong innermost-first", () => { + const before = "abc"; + const op: DocumentOperation = { + type: "format", + from: 0, + to: 3, + add: true, + marks: [{ type: "strong" }, { type: "em" }], + }; + // strong applied first (innermost), then em wraps: *abc* + expect(applyDocumentOperations(before, [op])).toBe("***abc***"); + }); + + test("adds code marks", () => { + const before = "run cmd here"; + const op: DocumentOperation = { + type: "format", + from: 4, + to: 7, + add: true, + marks: [{ type: "code" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("run `cmd` here"); + }); + + test("adds code marks around selection containing a backtick using a longer delimiter", () => { + // CommonMark code spans: choose a backtick run STRICTLY longer + // than any run in the content, and pad with a single space when + // the content begins or ends with a backtick, so removal can + // symmetrically peel the emitted form back to the original. + const before = "prefix `x suffix"; + const add: DocumentOperation = { + type: "format", + from: 7, + to: 9, + add: true, + marks: [{ type: "code" }], + }; + const wrapped = applyDocumentOperations(before, [add]); + // Delimiter must be at least length 2 (content has a run of 1). + expect(wrapped).toBe("prefix `` `x `` suffix"); + // Removal targets the CONTENT positions in the wrapped string + // (matching how the strong/em `from/to` semantics work): the + // content "`x" now sits at positions 10..12 in the wrapped + // string. The peel walks outward through the padding and the + // discovered backtick run so both delimiters and pads are + // stripped symmetrically. + const remove: DocumentOperation = { + type: "format", + from: 10, + to: 12, + add: false, + marks: [{ type: "code" }], + }; + expect(applyDocumentOperations(wrapped, [remove])).toBe(before); + }); + + test("adds a link when href is provided", () => { + const before = "click here"; + const op: DocumentOperation = { + type: "format", + from: 6, + to: 10, + add: true, + marks: [{ type: "link", attrs: { href: "https://example.com" } }], + }; + expect(applyDocumentOperations(before, [op])).toBe( + "click [here](https://example.com)", + ); + }); + + test("drops a link mark with no href instead of emitting empty target", () => { + const before = "click here"; + const op: DocumentOperation = { + type: "format", + from: 6, + to: 10, + add: true, + marks: [{ type: "link", attrs: {} }], + }; + expect(applyDocumentOperations(before, [op])).toBe("click here"); + }); + + test("removes strong marks around a character range", () => { + const before = "hello **world**"; + const op: DocumentOperation = { + type: "format", + from: 8, + to: 13, + add: false, + marks: [{ type: "strong" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello world"); + }); + + test("removes nested marks innermost-first", () => { + const before = "***abc***"; + const op: DocumentOperation = { + type: "format", + from: 3, + to: 6, + add: false, + marks: [{ type: "strong" }, { type: "em" }], + }; + // Peel ** first (matches inner **), then * around it: "abc" + expect(applyDocumentOperations(before, [op])).toBe("abc"); + }); + + test("removes a link", () => { + const before = "click [here](https://example.com)"; + const op: DocumentOperation = { + type: "format", + from: 7, + to: 11, + add: false, + marks: [{ type: "link", attrs: { href: "https://example.com" } }], + }; + expect(applyDocumentOperations(before, [op])).toBe("click here"); + }); + + test("removes a link whose destination contains parentheses", () => { + const before = "click [here](https://example.com/a(b))"; + const op: DocumentOperation = { + type: "format", + from: 7, + to: 11, + add: false, + marks: [ + { + type: "link", + attrs: { href: "https://example.com/a(b)" }, + }, + ], + }; + expect(applyDocumentOperations(before, [op])).toBe("click here"); + }); + + test("does not remove a link with a different destination", () => { + const before = "click [here](https://example.com/one)"; + const op: DocumentOperation = { + type: "format", + from: 7, + to: 11, + add: false, + marks: [ + { + type: "link", + attrs: { href: "https://example.com/two" }, + }, + ], + }; + expect(applyDocumentOperations(before, [op])).toBe(before); + }); + + test("remove is idempotent when the delimiter is not present", () => { + const before = "hello world"; + const op: DocumentOperation = { + type: "format", + from: 6, + to: 11, + add: false, + marks: [{ type: "strong" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello world"); + }); + + test("empty range is a no-op", () => { + const before = "hello"; + const op: DocumentOperation = { + type: "format", + from: 2, + to: 2, + add: true, + marks: [{ type: "strong" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello"); + }); + + test("removes __strong__ alt-delimiter form", () => { + const before = "hello __world__"; + const op: DocumentOperation = { + type: "format", + from: 8, + to: 13, + add: false, + marks: [{ type: "strong" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello world"); + }); + + test("removes _em_ alt-delimiter form", () => { + const before = "hello _world_"; + const op: DocumentOperation = { + type: "format", + from: 7, + to: 12, + add: false, + marks: [{ type: "em" }], + }; + expect(applyDocumentOperations(before, [op])).toBe("hello world"); + }); +});