From eebf5a9796f52674222a74eacb9af8906f406285 Mon Sep 17 00:00:00 2001 From: JJ Lee Date: Sat, 12 Sep 2026 08:59:03 -0400 Subject: [PATCH] chore: sync overlay to opencode 8f0f5d9097 (Shiki highlighting) --- packages/app-bundle/manifest.json | 36 ++-- .../preview-baseline-contract.spec.ts | 29 +++ .../overlay/packages/app/src/app.tsx | 5 + .../overlay/packages/session-ui/bunfig.toml | 2 +- .../overlay/packages/session-ui/package.json | 3 + .../session-ui/preload-worker-mock.ts | 25 +++ .../src/components/markdown-shiki.worker.ts | 18 +- .../packages/session-ui/src/pierre/worker.ts | 13 +- .../v2/components/editable-diff-view-core.ts | 3 + .../src/v2/components/editable-diff-view.tsx | 1 + .../src/v2/components/editor-core.test.ts | 11 + .../src/v2/components/editor-core.ts | 58 ++++- .../src/v2/components/preview-editor.tsx | 4 +- .../shiki-highlight-decorations.test.ts | 107 ++++++++++ .../components/shiki-highlight-decorations.ts | 85 ++++++++ .../shiki-highlight-integration.test.ts | 141 ++++++++++++ .../v2/components/shiki-highlight-plugin.ts | 202 ++++++++++++++++++ .../v2/components/shiki-highlight-worker.ts | 122 +++++++++++ .../v2/components/shiki-theme-state.test.ts | 65 ++++++ .../src/v2/components/shiki-theme-state.ts | 165 ++++++++++++++ 20 files changed, 1069 insertions(+), 26 deletions(-) create mode 100644 packages/app-bundle/overlay/packages/session-ui/preload-worker-mock.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.test.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-integration.test.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-plugin.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-worker.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.test.ts create mode 100644 packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.ts diff --git a/packages/app-bundle/manifest.json b/packages/app-bundle/manifest.json index 43aef390..7ea37ae5 100644 --- a/packages/app-bundle/manifest.json +++ b/packages/app-bundle/manifest.json @@ -3,10 +3,10 @@ "scope": "complete fork-vs-base delta of packages/{app,ui,session-ui,schema,core,sdk}", "fork_ref": "local/amicode", "fork_tag": "v1.18.29-amicode.30-11-g0667bccd6d", - "fork_sha": "6fa55b660a317357ac30e360e694f65aa7410723", - "upstream_base": "v1.18.29", + "fork_sha": "8f0f5d9097550eb3b3824d9edd76411339786f06", + "upstream_base": "16747470f976aca3d362ad730bcd3fe82ecc2c9a", "upstream_base_sha": "16747470f976aca3d362ad730bcd3fe82ecc2c9a", - "promoted_at": "2026-09-11T16:58:54.648Z", + "promoted_at": "2026-09-12T12:58:48.994Z", "files": { "packages/app/AGENTS.md": "6ad84d752791908326d30b96703136918b14a1822f1dd4424a2ebc7ebfa6c78b", "packages/app/CONTEXT.md": "7dc4bf176a32ee578a54593a58a6b97907ffc5a9baf7ae7c16a8fa920d8e6e86", @@ -23,7 +23,7 @@ "packages/app/e2e/regression/new-session-panel-corner.spec.ts": "df11b094012f8206117575c63fd9ac0475d14b54ea4f9d719c6b21aac04d1c4b", "packages/app/e2e/regression/openai-browser-oauth.spec.ts": "7df4521d1d82a186e383afd9351eef64f33f2adf8a3dce86dd5915b5f4828b68", "packages/app/e2e/regression/pane-boot-watch.spec.ts": "3aa869f8b1d9881797e14dceb9854c4408f768f490d4db8722e29e511bca1833", - "packages/app/e2e/regression/preview-baseline-contract.spec.ts": "27faae9571de30d05e7f7aef4e3b3ba318deea2042fe0662fe753cad8bf512e8", + "packages/app/e2e/regression/preview-baseline-contract.spec.ts": "78c23b9e52496a779ee10aabf5bfad6900afb9b905d82ddedfa954031ce633f5", "packages/app/e2e/regression/project-picker-recent-search.spec.ts": "64f4249f29cfce29d5e755a544c5aa2f27599d9fd874d363628f696f5f0e02b9", "packages/app/e2e/regression/review-line-comment.spec.ts": "953e9c67ed4cee91968369088237cf2288558eaa55e55b61f385571bccad7003", "packages/app/e2e/regression/session-timeline-lifecycle-state.spec.ts": "00bfd4f6199c1b350d14a9eda5e3a4c546b020df32e3604609326281869d6d4f", @@ -55,7 +55,7 @@ "packages/app/src/amicode/inspector/inspector-context.tsx": "ac81b9e0e2a4035f85b6af68b7f0cefcb9f256a175b244ba13136f3a307c0c77", "packages/app/src/amicode/inspector/run-inspector.tsx": "63a6351fad12a8377550eb87f7602a712bae586fe482ff46cd847c8544fc6e8a", "packages/app/src/app-desktop-commands.test.ts": "1d2057f238dc14e4407ff5c4e53afe091784790bc0fd0dbfde0eacd91e2bd968", - "packages/app/src/app.tsx": "91ea1817db2f7e21caae642f3c2e276b4835fcecf499699cb787e7a5521ae20b", + "packages/app/src/app.tsx": "c32eae50e48492404aa2057f37fc0ce229e24640d26ce9a81c3b64066483f62f", "packages/app/src/components/amicode-default-model-value.ts": "416bb4a7cf051c678bdfc5246ddc5796a9a6efe48d1e5c2e933756800cfdae8d", "packages/app/src/components/amicode-default-model.test.ts": "941750d074706bd5f4e433c9737b19ebc0ac0f9ba9cd1faaa599a24b22126e05", "packages/app/src/components/amicode-default-model.tsx": "16a2de1feb12e07416fe5fbb25879391f912ee0909b553c868c1b531f2ed1f2d", @@ -434,9 +434,10 @@ "packages/sdk/js/src/v2/gen/types.gen.ts": "0c23b262085d1e14b89bd03292a380c9ec8b52e66c4308304a05173362b79339", "packages/sdk/js/test/session-history.test.ts": "f10913af3157694465273d54e05c03cf96a0002df0e34898d9808f92977b11fd", "packages/sdk/openapi.json": "5bbd6493a1a488ef4294889341c896e420f814ecea95822100aaa9f3f95ab2d1", - "packages/session-ui/bunfig.toml": "82585987c8258caf58d07c6aaf4e49b7338e4da16f97c785af398aace9e6e722", + "packages/session-ui/bunfig.toml": "468114e9b52c3cf926d2385c12d93385992cd70c97ea9e32ed22154617b7799d", "packages/session-ui/happydom.ts": "d81d6b14e988f058fa502c841df34b55307c57fc94a26fb4f8a5deb6aa0d4834", - "packages/session-ui/package.json": "c5838c61cc5726509e8da4d1708283590070df88601d26ae400e62e71e6fd0c6", + "packages/session-ui/package.json": "15edfe2287a76b544bb359c809da056087bf00e71684a34d22cdcc405a77868a", + "packages/session-ui/preload-worker-mock.ts": "b53280029a042575734d7bfdf27713f6c5cd6a6665599dc429d839a3e3372bd1", "packages/session-ui/src/components/basic-tool.css": "0bcf0592ce8af0d902bd69da26ecad2f00f5a7890275392abbffe6f4d80c7853", "packages/session-ui/src/components/basic-tool.stories.tsx": "059c4bdce6a9ddc93c69949f5baf7323631ef0d5e69f04030fde0f2771a9fc09", "packages/session-ui/src/components/basic-tool.tsx": "ff2678cc17510e1b39657469f6dd1a0311843664cfd5161d385256e617b4e759", @@ -449,7 +450,7 @@ "packages/session-ui/src/components/markdown-file-refs.test.ts": "e2b1bed33629d3ab53c521f33266814088ea4122b6e64d6ab294eacc4cf15070", "packages/session-ui/src/components/markdown-file-refs.ts": "b75bddeb565dc0163635fab8f84bdd26723742f9f82d06b6fd5991777b311032", "packages/session-ui/src/components/markdown-preload.test.ts": "3a2db02aa3c199375ae3a8a240d4505c2c0bd5e84e1ef3a25d19618071a0b810", - "packages/session-ui/src/components/markdown-shiki.worker.ts": "0f438e40aee871d3da81488e21f42c0fefcd9af4a0eec9c599cd969566c2610e", + "packages/session-ui/src/components/markdown-shiki.worker.ts": "2748ddbfcd625bf696b27d9b64895e64ee47a3f316c88930010e260b4ba37f4b", "packages/session-ui/src/components/markdown-stream.test.ts": "b73b1a15ffa2cfea5358b2d6313ce17d26a5a11302b0e3b7d823cde0c98cbc83", "packages/session-ui/src/components/markdown-stream.ts": "50e38abd5d4ff919564655799678be10463dec58502fc5487c1c97a9a8db5399", "packages/session-ui/src/components/markdown-worker-protocol.test.ts": "fdd7f1a35b208c7c2b57238ce89bc81255d99cde4b8ac1e7f261b7c8feafcd1a", @@ -482,23 +483,23 @@ "packages/session-ui/src/components/tool-error-card.tsx": "e27b89836b7c709120ac15fbffd2200038944058ea3506e61e5897ffb3537503", "packages/session-ui/src/components/tool-status-title.tsx": "ae4561b58f55985d2294a352802e723b18943ac9a6551d97f1c4a6cc95368c8a", "packages/session-ui/src/context/data.tsx": "7f7eb7e6d9bc3e2ad3dca7f3e873fade43a1be2ba441950aa210f3254be0f1cb", - "packages/session-ui/src/pierre/worker.ts": "b94539c599ff0628def07d2a07f857f5523241550673e207e8425a91ea78eef1", + "packages/session-ui/src/pierre/worker.ts": "7f210f931777956567eaeea15f2f5478befc7874a64a4b685504c59c66503fee", "packages/session-ui/src/util/clipboard.ts": "c37ba53914714ed04135e5a254e8ea90186f097d7470fa0873a83de82f3fef2d", "packages/session-ui/src/v2/components/attachment-card-v2.css": "3b4b9c23d69e76b35488bc3a65fb51211038c3e8aac015df3f531b4e97e96359", "packages/session-ui/src/v2/components/basic-tool-v2.css": "55d3aa179704deddda7149342a951229ff5561127f786dd551e5df9aea8a7835", "packages/session-ui/src/v2/components/basic-tool-v2.tsx": "ece46d761f7d51a7a0925b3df00a5c289d891e9a514ee721016eb723d87fb406", "packages/session-ui/src/v2/components/diff-gutter-extension.test.ts": "7c3a9d541f8cc2cdb537cc6630d8902a17647f0df063439a93e8d563cd4a97b2", "packages/session-ui/src/v2/components/diff-gutter-extension.ts": "aed07587c0c9a5f110f9ea91aa42679d6f9f66b2ec4de9043d6e2233a375acb8", - "packages/session-ui/src/v2/components/editable-diff-view-core.ts": "f465e41349b820805bb9a2395886cc6bb2612bb40cbe6f1157823a2441ae75f3", + "packages/session-ui/src/v2/components/editable-diff-view-core.ts": "f01a341c4c1cdb4a9522f9ccd3c856bd3b9b7310c01d87327b3ce777052c8c59", "packages/session-ui/src/v2/components/editable-diff-view.test.ts": "ab206fa17779e6e9015d343ae37e3067dfc1821a93b6ba566b7a0dce0fc19acb", - "packages/session-ui/src/v2/components/editable-diff-view.tsx": "4c0ee2525b94b04b1e708a5cacab6d3eed29ca30644321986e429b2f912b730b", - "packages/session-ui/src/v2/components/editor-core.test.ts": "aee8416e89ec0c38da77e57d422a0a59e4add150aa0875955f5124535c0a9585", - "packages/session-ui/src/v2/components/editor-core.ts": "bae3cebbaa1138885f28da94f32781a12c6f3e0ef2990776b15be18d8aead10e", + "packages/session-ui/src/v2/components/editable-diff-view.tsx": "f682e398ece864b5b854ca36698581c7e1cc5964eed96e7b45cabfcb64748b24", + "packages/session-ui/src/v2/components/editor-core.test.ts": "bd6552c0e7080afd554cb57ad690cc1504555712274c8b02702c64bb09460234", + "packages/session-ui/src/v2/components/editor-core.ts": "a03da4863d0dc4333772b8369a63822dd730bfd84dfa876f965cdd594292484b", "packages/session-ui/src/v2/components/file-name-picker.test.tsx": "1ce9741e315b549165d390c74b5885a43580aa70e24de987f9ba8388cd521b41", "packages/session-ui/src/v2/components/markdown-utils.test.ts": "10e1034293bf385cf50d860bf1667d8370abc61dcee0c5a93295f21f5f55876a", "packages/session-ui/src/v2/components/markdown-utils.ts": "3bc47cfed5b99ab23f857d5324be3189f3a7e800d991103d72ef1c2f970b910b", "packages/session-ui/src/v2/components/preview-editor.test.ts": "b8da9320a594aaa680ed3848612fcea66f41c0a046a673f0cf6155c2a0b697dd", - "packages/session-ui/src/v2/components/preview-editor.tsx": "d859721bf3ea4e3bdbf24985e95198697307a78f8cb1644dc9fba5ddf3a189fc", + "packages/session-ui/src/v2/components/preview-editor.tsx": "d356ff168919abc8446c5fac2836791338fb3b9124f558c43ea7d3d5a1bcc5f4", "packages/session-ui/src/v2/components/preview-file-routing.test.ts": "fca02a21261b291fd94001baa2294e7b2fa45b18b0c0403848f045763b41bf15", "packages/session-ui/src/v2/components/preview-nav-state.test.ts": "cec09a8b3c08ab16581d3d5b6ff704584b3c87d81c25e8af20f6f00272f3593f", "packages/session-ui/src/v2/components/preview-nav-state.ts": "ccaeb94d1ab39271f5dcd49c935c8fc7b54d4ffbd66e1352e4d797c9205553fc", @@ -507,6 +508,13 @@ "packages/session-ui/src/v2/components/session-review-file-preview-v2.tsx": "fc06f46ed3c92c8066744da1f6ac97db7d8b7d2aeec5ef776d743dd93b317936", "packages/session-ui/src/v2/components/session-review-v2.css": "bb306f226fda8f8ba6cf0ae706843ba9f2b48f3a3446433515e8a8dd1f9dc626", "packages/session-ui/src/v2/components/session-review-v2.tsx": "4151286efb6e0e148246ee3eebae24a2e5d4eac2603fd5ea8546d0839a694161", + "packages/session-ui/src/v2/components/shiki-highlight-decorations.test.ts": "f50e8156b708f3e0ddbc509abf71751778a74923dac9d7e65385b178f6346718", + "packages/session-ui/src/v2/components/shiki-highlight-decorations.ts": "57fee1804df78c627f7ddfd092a55d71afe041baa153d0d5ed7c7d8781e30206", + "packages/session-ui/src/v2/components/shiki-highlight-integration.test.ts": "c3f62945f0258d7462bf9936a8b78294c4295f445e2a75fca186432f241443dc", + "packages/session-ui/src/v2/components/shiki-highlight-plugin.ts": "85b01203590f6de0af75b1cb4f35d1b87dea92989a950dbe6a9cf10aac19ffaf", + "packages/session-ui/src/v2/components/shiki-highlight-worker.ts": "0f9628b9e0d3ec9ff49d53d05cb77259e1762f66daf7b0e1f2403649385e66e4", + "packages/session-ui/src/v2/components/shiki-theme-state.test.ts": "b305c3f669b2083851fb574bacccab4f311228ce842f18465443b7807226e619", + "packages/session-ui/src/v2/components/shiki-theme-state.ts": "5ab15c15f4a32afce2279068adda732846404133bbc2a44b90779a4da88d123a", "packages/session-ui/src/v2/components/tool-error-card-v2.css": "9eb0bf8b583a18159f3855465118ff6a853d0193ea783037d5c90a868fdb3ff1", "packages/ui/package.json": "b1d168d0371e9094faae1107fc6c00be197f09bc69daa2247a3890d607f4b629", "packages/ui/src/amicode/amico-presence.css": "b3f345b607e318dcfbefcdc677fe21cc7d387b9946b0f9d625b0a842fe7e31b9", diff --git a/packages/app-bundle/overlay/packages/app/e2e/regression/preview-baseline-contract.spec.ts b/packages/app-bundle/overlay/packages/app/e2e/regression/preview-baseline-contract.spec.ts index b5a3c916..6031ea53 100644 --- a/packages/app-bundle/overlay/packages/app/e2e/regression/preview-baseline-contract.spec.ts +++ b/packages/app-bundle/overlay/packages/app/e2e/regression/preview-baseline-contract.spec.ts @@ -11,6 +11,8 @@ const sessionID = "ses_preview_baseline_contract" const title = "Preview baseline contract" const markdownFile = "notes/baseline.md" const secondMarkdownFile = "notes/second.md" +const juliaFile = "scripts/preview.jl" +const juliaContent = "function evolve(state)\n return state + 1\nend\n" const imageFile = "assets/preview.png" const server = `http://${process.env.PLAYWRIGHT_SERVER_HOST ?? "127.0.0.1"}:${process.env.PLAYWRIGHT_SERVER_PORT ?? "4096"}` const markdownContent = ["# Baseline Preview", "", "The renderer must keep focus."] @@ -50,6 +52,32 @@ test("keeps Markdown typing in CodeMirror after its language support loads", asy await expect(page.locator('[data-component="prompt-input"]')).toHaveText("") }) +test("applies the bridged VS Code token color to a Julia Preview editor", async ({ page }) => { + await openPreview(page) + + // A distinctive color makes this an end-to-end check of the full chain: + // host message -> theme state -> Shiki Worker -> CM6 Decoration.mark. + await page.evaluate(() => { + window.postMessage({ + source: "amicode", + kind: "syntax-theme", + theme: { + name: "Preview E2E Theme", + tokenColors: [{ scope: "keyword", settings: { foreground: "#ff00aa" } }], + }, + }, "*") + }) + await openPreviewFile(page, juliaFile) + + const editor = page.locator(`#review-panel [data-preview-host="${juliaFile}"] .cm-content`) + await expect(editor).toBeVisible() + const functionToken = editor.locator("span").filter({ hasText: "function" }).first() + await expect(functionToken).toBeVisible() + await expect + .poll(() => functionToken.evaluate((element) => getComputedStyle(element).color)) + .toBe("rgb(255, 0, 170)") +}) + test("keeps the default cursor while a Preview tab crosses an editable file", async ({ page }) => { await openPreview(page) await openPreviewFile(page, secondMarkdownFile) @@ -1505,6 +1533,7 @@ async function openPreview(page: Parameters[0]) { fileContent: (path) => { if (path === markdownFile) return { type: "text", content: markdownContent } if (path === secondMarkdownFile) return { type: "text", content: secondMarkdownContent } + if (path === juliaFile) return { type: "text", content: juliaContent } if (path === imageFile) return { type: "binary", content: imageContent, encoding: "base64", mimeType: "image/png" } if (path === pdfFile) diff --git a/packages/app-bundle/overlay/packages/app/src/app.tsx b/packages/app-bundle/overlay/packages/app/src/app.tsx index b22536e6..387c70f6 100644 --- a/packages/app-bundle/overlay/packages/app/src/app.tsx +++ b/packages/app-bundle/overlay/packages/app/src/app.tsx @@ -7,6 +7,7 @@ import { DialogProvider } from "@opencode-ai/ui/context/dialog" import { FileComponentProvider } from "@opencode-ai/ui/context/file" import { MarkedProvider } from "@opencode-ai/ui/context/marked" import { File } from "@opencode-ai/session-ui/file" +import { handleSyntaxThemeMessage } from "@opencode-ai/session-ui/v2/shiki-theme-state" import { Font } from "@opencode-ai/ui/font" import { Splash } from "@opencode-ai/ui/logo" import { ThemeProvider, useTheme } from "@opencode-ai/ui/theme/context" @@ -451,6 +452,10 @@ function AmicodeThemeBridge() { adoptExplorerIconTheme(d.theme) return } + if (d.kind === "syntax-theme") { + handleSyntaxThemeMessage((d as { theme?: string | object }).theme ?? "") + return + } if (d.kind !== "theme") return if (d.colorScheme === "light" || d.colorScheme === "dark") theme.setColorScheme(d.colorScheme) } diff --git a/packages/app-bundle/overlay/packages/session-ui/bunfig.toml b/packages/app-bundle/overlay/packages/session-ui/bunfig.toml index f1caabbc..8a253683 100644 --- a/packages/app-bundle/overlay/packages/session-ui/bunfig.toml +++ b/packages/app-bundle/overlay/packages/session-ui/bunfig.toml @@ -1,3 +1,3 @@ [test] root = "./src" -preload = ["./happydom.ts"] +preload = ["./happydom.ts", "./preload-worker-mock.ts"] diff --git a/packages/app-bundle/overlay/packages/session-ui/package.json b/packages/app-bundle/overlay/packages/session-ui/package.json index d9ede8aa..7610595b 100644 --- a/packages/app-bundle/overlay/packages/session-ui/package.json +++ b/packages/app-bundle/overlay/packages/session-ui/package.json @@ -23,6 +23,8 @@ "./v2/*": "./src/v2/components/*.tsx", "./v2/editor-core": "./src/v2/components/editor-core.ts", "./v2/markdown-utils": "./src/v2/components/markdown-utils.ts", + "./v2/shiki-theme-state": "./src/v2/components/shiki-theme-state.ts", + "./v2/shiki-highlight-plugin": "./src/v2/components/shiki-highlight-plugin.ts", "./v2/preview-editor": "./src/v2/components/preview-editor.tsx", "./v2/preview-nav-state": "./src/v2/components/preview-nav-state.ts", "./v2/prompt-input": "./src/v2/components/prompt-input/index.tsx", @@ -53,6 +55,7 @@ "@codemirror/lang-markdown": "6.5.2", "@codemirror/lang-python": "6.2.1", "@codemirror/language": "6.12.4", + "@codemirror/language-data": "6.5.1", "@codemirror/merge": "6.12.2", "@codemirror/state": "6.7.1", "@codemirror/view": "6.43.9", diff --git a/packages/app-bundle/overlay/packages/session-ui/preload-worker-mock.ts b/packages/app-bundle/overlay/packages/session-ui/preload-worker-mock.ts new file mode 100644 index 00000000..ceb1055f --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/preload-worker-mock.ts @@ -0,0 +1,25 @@ +/** + * Bun preload plugin: mock Vite-specific ?worker&url imports. + * + * Vite transforms `import Url from "./foo.ts?worker&url"` into a URL string + * at build time. Bun's test runner doesn't understand this suffix. This plugin + * intercepts any import ending in `?worker&url` and returns a dummy string, + * so modules that import worker URLs can load in tests without crashing. + * + * The Worker constructor will receive the dummy string and fail, which is + * expected — getWorker() catches the error and returns null, falling back + * to lezer-only highlighting in tests. + */ +import { plugin } from "bun" + +plugin({ + name: "vite-worker-url-mock", + setup(build) { + build.onResolve({ filter: /\?worker&url$/ }, (args) => { + return { path: args.path, namespace: "vite-worker-url-mock" } + }) + build.onLoad({ filter: /.*/, namespace: "vite-worker-url-mock" }, () => { + return { contents: 'export default "mock-worker-url"', loader: "js" } + }) + }, +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/components/markdown-shiki.worker.ts b/packages/app-bundle/overlay/packages/session-ui/src/components/markdown-shiki.worker.ts index d7acabee..6e874148 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/components/markdown-shiki.worker.ts +++ b/packages/app-bundle/overlay/packages/session-ui/src/components/markdown-shiki.worker.ts @@ -20,6 +20,7 @@ type Stream = { const streams = new Map() let highlighter: ReturnType | undefined +let currentThemeName = "OpenCode" const queue = createLatestWorkerQueue>({ run: highlight, supersede: (request) => post({ type: "superseded", id: request.id, key: request.key }), @@ -35,6 +36,19 @@ self.onmessage = (event: MessageEvent) => { queue.dispose(event.data.key) return } + // Theme update from the extension bridge (via shiki-theme-state) + if ((event.data as any).type === "theme-update") { + const msg = event.data as any + if (typeof msg.theme === "object" && highlighter) { + highlighter.then((instance) => { + if (instance) instance.loadTheme(msg.theme) + }) + } + if (typeof msg.name === "string") currentThemeName = msg.name + // Clear all streams so they pick up the new theme on next highlight + streams.clear() + return + } queue.highlight(event.data) } @@ -48,7 +62,7 @@ async function highlight(request: Extract { + if (unified) { unified.terminate(); unified = undefined } + if (split) { split.terminate(); split = undefined } +}) + export function getWorkerPool(style: WorkerPoolStyle | undefined): WorkerPoolManager | undefined { if (typeof window === "undefined") return diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view-core.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view-core.ts index 76609ef5..3c5de093 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view-core.ts +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view-core.ts @@ -84,6 +84,8 @@ export function createDiffEditor(opts: { readOnly: boolean theme: Extension language?: LanguageSupport | null + /** File extension for Shiki highlighting (e.g. "ts", "py", "jl"). */ + lang?: string onChange?: (content: string) => void }): DiffEditorHandle { let mergeView: MergeView | null = null @@ -97,6 +99,7 @@ export function createDiffEditor(opts: { const base = baseExtensions({ theme: opts.theme, language: opts.language, + lang: opts.lang, }) if (opts.diffStyle === "split") { diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view.tsx b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view.tsx index 6c1c64dd..9b338f31 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view.tsx +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editable-diff-view.tsx @@ -116,6 +116,7 @@ export function EditableDiffView(props: EditableDiffViewProps): JSX.Element { readOnly, theme, language: lang, + lang: props.language, onChange: readOnly ? undefined : onChange, }) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.test.ts index 910cee0e..b102e6d4 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.test.ts +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.test.ts @@ -61,6 +61,17 @@ describe("loadLanguage (via editor-core)", () => { expect(await loadLanguage("json")).not.toBeNull() }) + test("resolves extensions via @codemirror/language-data fallback", async () => { + // These are not in the manual EXTENSION_MAP but should be found + // by @codemirror/language-data's auto-detection + expect(await loadLanguage("rs")).not.toBeNull() // Rust + expect(await loadLanguage("go")).not.toBeNull() // Go + expect(await loadLanguage("yaml")).not.toBeNull() // YAML + expect(await loadLanguage("sql")).not.toBeNull() // SQL + expect(await loadLanguage("sh")).not.toBeNull() // Shell + expect(await loadLanguage("toml")).not.toBeNull() // TOML + }) + test("returns null for unknown extensions", async () => { expect(await loadLanguage("xyz-unknown")).toBeNull() }) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.ts index 482114d9..d2f344b6 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.ts +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/editor-core.ts @@ -27,6 +27,8 @@ import { syntaxHighlighting, } from "@codemirror/language" import { tags } from "@lezer/highlight" +import { shikiHighlightExtension, updateWorkerTheme } from "./shiki-highlight-plugin" +import { onThemeChange } from "./shiki-theme-state" // --------------------------------------------------------------------------- // External-update annotation — marks programmatic content dispatches so the @@ -43,7 +45,12 @@ import { tags } from "@lezer/highlight" export const externalUpdate = Annotation.define() // --------------------------------------------------------------------------- -// Language loader — dynamic imports so unused grammars stay out of the bundle. +// Language loader — uses a manual map for common extensions (with specific +// options like jsx/typescript flags) and falls back to @codemirror/language-data +// for broader coverage (~150 languages including Rust, Go, YAML, SQL, etc.). +// Shiki handles visual highlighting for 200+ languages via TextMate grammars; +// this loader provides lezer grammars for structural features (bracket +// matching, code folding, auto-indent). // --------------------------------------------------------------------------- const EXTENSION_MAP: Record Promise> = { @@ -69,19 +76,47 @@ const EXTENSION_MAP: Record Promise> = { /** * Dynamically load a CodeMirror language support by file extension. - * Returns null for unknown extensions. + * + * Uses the manual EXTENSION_MAP first (covers common extensions with specific + * options), then falls back to @codemirror/language-data's LanguageDescription + * auto-detection for broader coverage. + * + * Returns null for extensions with no known lezer grammar. Note: Shiki still + * provides visual highlighting for these files via TextMate grammars — only + * structural features (bracket matching, folding) degrade to CM6's generic + * behavior. */ export async function loadLanguage( ext: string, ): Promise { const normalized = ext.replace(/^\./, "").toLowerCase() + + // Fast path: manual map with specific options const loader = EXTENSION_MAP[normalized] - if (!loader) return null + if (loader) { + try { + return await loader() + } catch { + return null + } + } + + // Slow path: @codemirror/language-data auto-detection try { - return await loader() + const { languages } = await import("@codemirror/language-data") + const filename = `file.${normalized}` + const desc = languages.find((lang) => + lang.extensions.some((e) => filename.endsWith(e)) || + lang.filename?.test(filename), + ) + if (desc) { + return await desc.load() + } } catch { - return null + // language-data not available or failed to load — fall through } + + return null } // --------------------------------------------------------------------------- @@ -193,7 +228,7 @@ export function buildSyntaxHighlightStyle(): Extension { { tag: [tags.variableName, tags.definition(tags.variableName)], color: "var(--v2-text-text-base, var(--text-strong))" }, { tag: [tags.function(tags.variableName), tags.function(tags.propertyName)], - color: "var(--syntax-function, var(--syntax-property))" }, + color: "var(--syntax-property)" }, { tag: [tags.constant(tags.variableName), tags.atom], color: "var(--syntax-constant)" }, { tag: [tags.operator, tags.punctuation, tags.separator], @@ -252,6 +287,8 @@ export function editableExtensions(opts: { export function baseExtensions(opts: { theme: Extension language?: LanguageSupport | null + /** File extension for Shiki highlighting (e.g. "ts", "py", "jl"). */ + lang?: string }): Extension[] { return [ lineNumbers(), @@ -261,9 +298,10 @@ export function baseExtensions(opts: { bracketMatching(), keymap.of([...defaultKeymap, ...historyKeymap]), EditorView.lineWrapping, - buildSyntaxHighlightStyle(), + buildSyntaxHighlightStyle(), // first-paint bridge: lezer colors until Shiki responds opts.theme, ...(opts.language ? [opts.language] : []), + ...(opts.lang ? shikiHighlightExtension(opts.lang) : []), ] } @@ -292,3 +330,9 @@ export function detectMode(): "light" | "dark" { return "dark" return "light" } + +// --------------------------------------------------------------------------- +// Wire theme changes to the Shiki highlight worker +// --------------------------------------------------------------------------- + +onThemeChange(() => updateWorkerTheme()) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/preview-editor.tsx b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/preview-editor.tsx index edab61f9..45032b21 100644 --- a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/preview-editor.tsx +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/preview-editor.tsx @@ -34,6 +34,7 @@ export function PreviewEditor(props: { let containerRef!: HTMLDivElement let editorView: EditorView | null = null const [langSupport, setLangSupport] = createSignal(null) + const [fileExt, setFileExt] = createSignal("txt") const editableCompartment = new Compartment() const zoomCompartment = new Compartment() @@ -51,6 +52,7 @@ export function PreviewEditor(props: { // Extract extension from filepath const parts = props.filePath.split(".") const ext = parts.length > 1 ? parts[parts.length - 1] : "txt" + setFileExt(ext) const lang = await loadLanguage(ext) setLangSupport(lang) }) @@ -86,7 +88,7 @@ export function PreviewEditor(props: { state: EditorState.create({ doc: content, extensions: [ - ...baseExtensions({ theme, language: lang }), + ...baseExtensions({ theme, language: lang, lang: fileExt() }), editableCompartment.of( editableExtensions({ readOnly: false, diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.test.ts new file mode 100644 index 00000000..38eacf27 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from "bun:test" +import { Decoration } from "@codemirror/view" +import { + tokensToDecorations, + shikiDecorationField, + type ShikiTokenizeResult, +} from "./shiki-highlight-decorations" + +describe("shiki-highlight-decorations", () => { + describe("tokensToDecorations", () => { + test("converts Shiki tokens to CM6 decorations with inline color styles", () => { + const result: ShikiTokenizeResult = { + type: "tokenize-result", + id: 1, + lines: [ + { + tokens: [ + { offset: 0, length: 5, color: "#569CD6" }, // "const" + { offset: 5, length: 1, color: "" }, // " " (no color) + { offset: 6, length: 1, color: "#9CDCFE" }, // "x" + ], + }, + ], + } + const docText = "const x = 1" + const decos = tokensToDecorations(result, docText) + expect(decos).not.toBe(Decoration.none) + + const marks: Array<{ from: number; to: number }> = [] + const cursor = decos.iter() + while (cursor.value) { + marks.push({ from: cursor.from, to: cursor.to }) + cursor.next() + } + // "const" (0-5) and "x" (6-7) — empty color is skipped + expect(marks).toEqual([ + { from: 0, to: 5 }, + { from: 6, to: 7 }, + ]) + }) + + test("handles multi-line tokens", () => { + const result: ShikiTokenizeResult = { + type: "tokenize-result", + id: 1, + lines: [ + { tokens: [{ offset: 0, length: 3, color: "#FF0000" }] }, + { tokens: [{ offset: 0, length: 3, color: "#00FF00" }] }, + ], + } + const docText = "foo\nbar" + const decos = tokensToDecorations(result, docText) + + const marks: Array<{ from: number; to: number }> = [] + const cursor = decos.iter() + while (cursor.value) { + marks.push({ from: cursor.from, to: cursor.to }) + cursor.next() + } + expect(marks).toEqual([ + { from: 0, to: 3 }, + { from: 4, to: 7 }, + ]) + }) + + test("returns Decoration.none for empty token results", () => { + const result: ShikiTokenizeResult = { + type: "tokenize-result", + id: 1, + lines: [], + } + expect(tokensToDecorations(result, "hello")).toBe(Decoration.none) + }) + + test("applies fontStyle attributes", () => { + const result: ShikiTokenizeResult = { + type: "tokenize-result", + id: 1, + lines: [ + { + tokens: [ + { offset: 0, length: 7, color: "#888", fontStyle: 1 }, // italic + { offset: 8, length: 4, color: "#999", fontStyle: 2 }, // bold + { offset: 13, length: 4, color: "#AAA", fontStyle: 4 }, // underline + ], + }, + ], + } + const docText = "comment bold_var underline" + const decos = tokensToDecorations(result, docText) + + const marks: Array<{ from: number; to: number }> = [] + const cursor = decos.iter() + while (cursor.value) { + marks.push({ from: cursor.from, to: cursor.to }) + cursor.next() + } + expect(marks.length).toBe(3) + }) + }) + + describe("shikiDecorationField", () => { + test("is exported and defined", () => { + expect(shikiDecorationField).toBeDefined() + }) + }) +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.ts new file mode 100644 index 00000000..5cb6e3b4 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-decorations.ts @@ -0,0 +1,85 @@ +/** + * shiki-highlight-decorations — Pure token-to-decoration conversion logic. + * + * Separated from the plugin (which imports a Vite ?worker&url) so this + * module is testable in bun without Worker URL resolution issues. + * + * @module + */ + +import { type Range, StateEffect, StateField } from "@codemirror/state" +import { Decoration, type DecorationSet, EditorView } from "@codemirror/view" + +// --------------------------------------------------------------------------- +// Types — shared between the worker and the plugin +// --------------------------------------------------------------------------- + +export interface ShikiTokenLine { + tokens: Array<{ offset: number; length: number; color: string; fontStyle?: number }> +} + +export interface ShikiTokenizeResult { + type: "tokenize-result" + id: number + lines: ShikiTokenLine[] +} + +// --------------------------------------------------------------------------- +// Decoration effect + field +// --------------------------------------------------------------------------- + +export const setShikiDecorations = StateEffect.define() + +export const shikiDecorationField = StateField.define({ + create: () => Decoration.none, + update(value, tr) { + for (const e of tr.effects) { + if (e.is(setShikiDecorations)) return e.value + } + // Map decorations through document changes (shift offsets) + return tr.docChanged ? value.map(tr.changes) : value + }, + provide: (f) => EditorView.decorations.from(f), +}) + +// --------------------------------------------------------------------------- +// Token → Decoration conversion +// --------------------------------------------------------------------------- + +/** + * Convert a Shiki tokenization result into a CM6 DecorationSet. + * Each token becomes a Decoration.mark with an inline `style` attribute + * carrying the foreground color (and optional fontStyle). + */ +export function tokensToDecorations(result: ShikiTokenizeResult, docText: string): DecorationSet { + if (result.lines.length === 0) return Decoration.none + + const ranges: Range[] = [] + let lineStart = 0 + + for (let i = 0; i < result.lines.length && lineStart <= docText.length; i++) { + const line = result.lines[i] + for (const token of line.tokens) { + const from = lineStart + token.offset + const to = from + token.length + if (from >= to || from < 0 || to > docText.length) continue + if (!token.color) continue + + let style = `color: ${token.color}` + if (token.fontStyle) { + if (token.fontStyle & 1) style += "; font-style: italic" + if (token.fontStyle & 2) style += "; font-weight: bold" + if (token.fontStyle & 4) style += "; text-decoration: underline" + } + ranges.push(Decoration.mark({ attributes: { style } }).range(from, to)) + } + + // Move past this line's content + the newline + const newlineIdx = docText.indexOf("\n", lineStart) + lineStart = newlineIdx >= 0 ? newlineIdx + 1 : docText.length + 1 + } + + // Decorations must be sorted by from position + ranges.sort((a, b) => a.from - b.from || a.to - b.to) + return Decoration.set(ranges) +} diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-integration.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-integration.test.ts new file mode 100644 index 00000000..30388659 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-integration.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from "bun:test" +import { createHighlighter, bundledLanguages, type BundledLanguage } from "shiki" +import { OpenCodeTheme } from "@opencode-ai/ui/context/marked" +import { tokensToDecorations, type ShikiTokenizeResult } from "./shiki-highlight-decorations" +import { Decoration } from "@codemirror/view" + +/** + * Integration test: verifies that Shiki produces non-empty, colored tokens + * for multiple file types using the actual OpenCodeTheme, and that those + * tokens convert to valid CM6 decorations. + * + * This is the test that catches the empty-theme bug: if the theme has no + * tokenColors, all tokens come back with empty/default colors and the + * decoration set is empty. + */ + +const SNIPPETS: Record = { + julia: { + lang: "jl", + code: `function solve(prob)\n x = prob.initial\n for i in 1:100\n x = step(x)\n end\n return x\nend`, + expectKeyword: "function", + }, + typescript: { + lang: "ts", + code: `const x: number = 42;\nfunction greet(name: string): void {\n console.log(name);\n}`, + expectKeyword: "const", + }, + python: { + lang: "py", + code: `def train(model, data):\n for epoch in range(100):\n loss = model.step(data)\n return loss`, + expectKeyword: "def", + }, + rust: { + lang: "rs", + code: `fn main() {\n let x = 42;\n println!("{}", x);\n}`, + expectKeyword: "fn", + }, + go: { + lang: "go", + code: `package main\n\nimport "fmt"\n\nfunc main() {\n\tfmt.Println("hello")\n}`, + expectKeyword: "package", + }, + yaml: { + lang: "yaml", + code: `name: test\nversion: 1.0\ndependencies:\n - foo: "^2.0"`, + expectKeyword: "name", + }, + shell: { + lang: "sh", + code: `#!/bin/bash\nfor f in *.jl; do\n echo "$f"\ndone`, + expectKeyword: "for", + }, +} + +describe("Shiki integration — real tokenization with OpenCodeTheme", () => { + // Create one highlighter for all tests (expensive to create) + let highlighter: Awaited> + + test("OpenCodeTheme has tokenColors", () => { + // Verify the theme isn't empty — this is what the bug was + expect(OpenCodeTheme).toBeDefined() + expect((OpenCodeTheme as any).tokenColors?.length).toBeGreaterThan(0) + }) + + test("creates a highlighter with OpenCodeTheme", async () => { + highlighter = await createHighlighter({ + themes: [OpenCodeTheme as any], + langs: [], + }) + expect(highlighter).toBeDefined() + }) + + for (const [name, { lang, code, expectKeyword }] of Object.entries(SNIPPETS)) { + test(`${name} (.${lang}): produces colored tokens`, async () => { + if (!highlighter) throw new Error("highlighter not initialized") + + const language = lang in bundledLanguages ? lang : "text" + if (!highlighter.getLoadedLanguages().includes(language)) { + await highlighter.loadLanguage(bundledLanguages[language as BundledLanguage]) + } + + const result = highlighter.codeToTokens(code, { + lang: language as BundledLanguage, + theme: "OpenCode", + }) + + // Verify we got non-empty results + expect(result.tokens.length).toBeGreaterThan(0) + + // Verify at least some tokens have a non-empty color + const allTokens = result.tokens.flatMap((line) => line) + const coloredTokens = allTokens.filter((t) => t.color && t.color.length > 0) + expect(coloredTokens.length).toBeGreaterThan(0) + + // Verify the expected keyword is among the tokens + const keywordToken = allTokens.find((t) => t.content.trim() === expectKeyword) + expect(keywordToken).toBeDefined() + // The keyword should have a color assigned (not empty, not just the default fg) + expect(keywordToken!.color).toBeDefined() + expect(keywordToken!.color!.length).toBeGreaterThan(0) + }) + + test(`${name} (.${lang}): tokens convert to non-empty CM6 decorations`, async () => { + if (!highlighter) throw new Error("highlighter not initialized") + + const language = lang in bundledLanguages ? lang : "text" + if (!highlighter.getLoadedLanguages().includes(language)) { + await highlighter.loadLanguage(bundledLanguages[language as BundledLanguage]) + } + + const result = highlighter.codeToTokens(code, { + lang: language as BundledLanguage, + theme: "OpenCode", + }) + + // Convert to the wire format the worker would send + const wireResult: ShikiTokenizeResult = { + type: "tokenize-result", + id: 1, + lines: result.tokens.map((lineTokens) => ({ + tokens: lineTokens.map((token) => ({ + offset: token.offset, + length: token.content.length, + color: token.color ?? "", + ...(token.fontStyle ? { fontStyle: token.fontStyle } : {}), + })), + })), + } + + const decos = tokensToDecorations(wireResult, code) + expect(decos).not.toBe(Decoration.none) + + // Count decorations + let count = 0 + const cursor = decos.iter() + while (cursor.value) { count++; cursor.next() } + // Each language should produce at least a few decorations + expect(count).toBeGreaterThan(2) + }) + } +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-plugin.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-plugin.ts new file mode 100644 index 00000000..4eab77a6 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-plugin.ts @@ -0,0 +1,202 @@ +/** + * shiki-highlight-plugin — CM6 ViewPlugin that applies Shiki-tokenized + * syntax highlighting as decorations, replacing the lezer-based + * syntaxHighlighting for visual coloring while keeping lezer for + * structural features (bracket matching, folding, auto-indent). + * + * Runs Shiki on the main thread with the pure-JS regex engine — no Worker, + * no WASM, no URL resolution issues in webview contexts. + * + * @module + */ + +import { type Extension } from "@codemirror/state" +import { + EditorView, + type PluginValue, + ViewPlugin, + type ViewUpdate, +} from "@codemirror/view" +import { + type ShikiTokenizeResult, + setShikiDecorations, + shikiDecorationField, + tokensToDecorations, +} from "./shiki-highlight-decorations" +import { getActiveShikiTheme, getActiveThemeObject, onThemeChange } from "./shiki-theme-state" +import { OpenCodeTheme } from "@opencode-ai/ui/context/marked" + +// --------------------------------------------------------------------------- +// Main-thread Shiki highlighter — lazy singleton +// --------------------------------------------------------------------------- + +import { createHighlighterCore, type HighlighterCore } from "shiki/core" +import { createJavaScriptRegexEngine } from "shiki/engine/javascript" +import { bundledLanguages, type BundledLanguage } from "shiki/langs" +import type { ThemeRegistrationRaw } from "shiki/types" + +const jsEngine = createJavaScriptRegexEngine() + +let highlighterPromise: Promise | null = null +let currentThemeName: string | undefined + +function getHighlighter(): Promise { + if (highlighterPromise) return highlighterPromise + + const themeObj = getActiveThemeObject() + const themeName = getActiveShikiTheme() + + if (themeObj) { + currentThemeName = "vscode-active" + highlighterPromise = createHighlighterCore({ + themes: [{ ...(themeObj as ThemeRegistrationRaw), name: "vscode-active" }], + langs: [], + engine: jsEngine, + }) + } else { + currentThemeName = themeName + highlighterPromise = createHighlighterCore({ + themes: [{ ...(OpenCodeTheme as ThemeRegistrationRaw), name: themeName }], + langs: [], + engine: jsEngine, + }) + } + + return highlighterPromise +} + +async function handleThemeUpdate(): Promise { + if (!highlighterPromise) return + const instance = await highlighterPromise + const themeObj = getActiveThemeObject() + const name = getActiveShikiTheme() + + if (themeObj) { + const themed = { ...(themeObj as ThemeRegistrationRaw), name: "vscode-active" } + await instance.loadTheme(themed) + currentThemeName = "vscode-active" + } else { + currentThemeName = name + } +} + +async function tokenize( + text: string, + lang: string, +): Promise { + try { + const instance = await getHighlighter() + if (!currentThemeName) { + return { type: "tokenize-result", id: 0, lines: [] } + } + + const langId = lang as BundledLanguage + const language = langId in bundledLanguages ? langId : null + if (language && !instance.getLoadedLanguages().includes(language)) { + await instance.loadLanguage(bundledLanguages[language]) + } + + const result = instance.codeToTokens(text, { + lang: language ?? "text", + theme: currentThemeName, + }) + + return { + type: "tokenize-result", + id: 0, + lines: result.tokens.map((lineTokens) => ({ + tokens: lineTokens.map((token) => ({ + offset: token.offset, + length: token.content.length, + color: token.color ?? "", + ...(token.fontStyle ? { fontStyle: token.fontStyle } : {}), + })), + })), + } + } catch (err) { + console.error("[shiki] tokenize failed:", err) + return { type: "tokenize-result", id: 0, lines: [] } + } +} + +// --------------------------------------------------------------------------- +// CM6 ViewPlugin +// --------------------------------------------------------------------------- + +const DEBOUNCE_MS = 150 + +class ShikiHighlightPluginValue implements PluginValue { + private view: EditorView + private lang: string + private debounceTimer: ReturnType | null = null + private currentTokenizeId = 0 + private unsubTheme: (() => void) | null = null + + constructor(view: EditorView, lang: string) { + this.view = view + this.lang = lang + + this.unsubTheme = onThemeChange(() => { + void handleThemeUpdate() + this.scheduleTokenize() + }) + + // Initial tokenization + this.scheduleTokenize() + } + + update(update: ViewUpdate): void { + if (update.docChanged || update.viewportChanged) { + this.scheduleTokenize() + } + } + + destroy(): void { + if (this.debounceTimer) clearTimeout(this.debounceTimer) + if (this.unsubTheme) this.unsubTheme() + } + + private scheduleTokenize(): void { + if (this.debounceTimer) clearTimeout(this.debounceTimer) + this.debounceTimer = setTimeout(() => this.doTokenize(), DEBOUNCE_MS) + } + + private async doTokenize(): Promise { + const id = ++this.currentTokenizeId + const view = this.view + const text = view.state.doc.toString() + + const result = await tokenize(text, this.lang) + + // Stale check: if another tokenization was started, discard this one + if (id !== this.currentTokenizeId) return + if (!this.view) return + + const decorations = tokensToDecorations(result, text) + view.dispatch({ effects: setShikiDecorations.of(decorations) }) + } +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Create a CM6 extension that applies Shiki-based syntax highlighting. + * The `lang` parameter is the file extension (e.g. "ts", "py", "jl"). + */ +export function shikiHighlightExtension(lang: string): Extension[] { + return [ + shikiDecorationField, + ViewPlugin.define( + (view) => new ShikiHighlightPluginValue(view, lang), + ), + ] +} + +/** + * Notify the highlighter about a theme change (called from editor-core). + */ +export function updateWorkerTheme(): void { + void handleThemeUpdate() +} diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-worker.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-worker.ts new file mode 100644 index 00000000..8944cd7d --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-highlight-worker.ts @@ -0,0 +1,122 @@ +/// + +/** + * shiki-highlight-worker — Dedicated Web Worker for Shiki tokenization + * of CM6 editor content. Receives text + language + theme, returns tokens + * with colors for the CM6 decoration plugin. + * + * Uses the pure-JS regex engine (no oniguruma WASM) so the Worker + * initializes without fetching any binary — works in VS Code webviews, + * iframes, and any restricted context. + * + * @module + */ + +import { bundledLanguages, type BundledLanguage } from "shiki/langs" +import { createHighlighterCore, type HighlighterCore } from "shiki/core" +import { createJavaScriptRegexEngine } from "shiki/engine/javascript" +import type { ThemeRegistrationRaw } from "shiki/types" + +const jsEngine = createJavaScriptRegexEngine() + +let highlighter: Promise | undefined +let currentThemeName: string | undefined + +export interface ShikiTokenizeRequest { + type: "tokenize" + id: number + text: string + lang: string + theme: string +} + +export interface ShikiThemeUpdateRequest { + type: "theme-update" + theme: string | ThemeRegistrationRaw + name: string +} + +export interface ShikiInitRequest { + type: "init" + theme: ThemeRegistrationRaw + name: string +} + +export type ShikiHighlightWorkerRequest = + | ShikiTokenizeRequest + | ShikiThemeUpdateRequest + | ShikiInitRequest + +export interface ShikiTokenizeResult { + type: "tokenize-result" + id: number + lines: Array<{ + tokens: Array<{ offset: number; length: number; color: string; fontStyle?: number }> + }> +} + +function post(msg: ShikiTokenizeResult): void { + ;(self as unknown as Worker).postMessage(msg) +} + +self.onmessage = async (event: MessageEvent) => { + const data = event.data + + if (data.type === "init") { + currentThemeName = data.name + highlighter ??= createHighlighterCore({ + themes: [data.theme], + langs: [], + engine: jsEngine, + }) + return + } + + if (data.type === "theme-update") { + if (!highlighter) return + const instance = await highlighter + if (typeof data.theme === "object") { + // Custom theme from VS Code — register under its own name, then + // also ensure currentThemeName matches what the plugin requests. + const themed = { ...(data.theme as ThemeRegistrationRaw), name: data.name } + await instance.loadTheme(themed) + } + currentThemeName = data.name + return + } + + if (data.type === "tokenize") { + try { + const instance = await highlighter + if (!instance || !currentThemeName) { + post({ type: "tokenize-result", id: data.id, lines: [] }) + return + } + + const langId = data.lang + const language = langId in bundledLanguages ? langId as BundledLanguage : null + if (language && !instance.getLoadedLanguages().includes(language)) { + await instance.loadLanguage(bundledLanguages[language]) + } + + const result = instance.codeToTokens(data.text, { + lang: language ?? "text", + theme: currentThemeName, + }) + + const lines = result.tokens.map((lineTokens) => ({ + tokens: lineTokens.map((token) => ({ + offset: token.offset, + length: token.content.length, + color: token.color ?? "", + ...(token.fontStyle ? { fontStyle: token.fontStyle } : {}), + })), + })) + + post({ type: "tokenize-result", id: data.id, lines }) + } catch { + // On error, return empty — the editor falls back to lezer highlighting + post({ type: "tokenize-result", id: data.id, lines: [] }) + } + } +} diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.test.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.test.ts new file mode 100644 index 00000000..1720b59c --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test, beforeEach } from "bun:test" +import { + getActiveShikiTheme, + getActiveThemeObject, + handleSyntaxThemeMessage, + onThemeChange, + resetToFallback, +} from "./shiki-theme-state" + +describe("shiki-theme-state", () => { + beforeEach(() => { + resetToFallback() + }) + + test("defaults to OpenCode fallback theme", () => { + expect(getActiveShikiTheme()).toBe("OpenCode") + expect(getActiveThemeObject()).toBeNull() + }) + + test("accepts a Shiki built-in name string", () => { + handleSyntaxThemeMessage("dark-plus") + expect(getActiveShikiTheme()).toBe("dark-plus") + expect(getActiveThemeObject()).toBeNull() + }) + + test("accepts a full TextMate theme object", () => { + const themeObj = { + name: "My Theme", + tokenColors: [{ scope: "comment", settings: { foreground: "#888" } }], + } + handleSyntaxThemeMessage(themeObj) + expect(getActiveShikiTheme()).toBe("vscode-active") + expect(getActiveThemeObject()).toBe(themeObj) + }) + + test("notifies listeners on theme change", () => { + const received: string[] = [] + const unsub = onThemeChange((name) => received.push(name)) + + handleSyntaxThemeMessage("dracula") + handleSyntaxThemeMessage("nord") + unsub() + handleSyntaxThemeMessage("monokai") // should not be received + + expect(received).toEqual(["dracula", "nord"]) + }) + + test("resetToFallback restores OpenCode", () => { + handleSyntaxThemeMessage("dark-plus") + expect(getActiveShikiTheme()).toBe("dark-plus") + + resetToFallback() + expect(getActiveShikiTheme()).toBe("OpenCode") + expect(getActiveThemeObject()).toBeNull() + }) + + test("ignores invalid theme values", () => { + handleSyntaxThemeMessage("dark-plus") + handleSyntaxThemeMessage(null as any) + expect(getActiveShikiTheme()).toBe("dark-plus") // unchanged + + handleSyntaxThemeMessage(undefined as any) + expect(getActiveShikiTheme()).toBe("dark-plus") // unchanged + }) +}) diff --git a/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.ts b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.ts new file mode 100644 index 00000000..4f784d42 --- /dev/null +++ b/packages/app-bundle/overlay/packages/session-ui/src/v2/components/shiki-theme-state.ts @@ -0,0 +1,165 @@ +/** + * shiki-theme-state — Reactive Shiki theme state for the webview. + * + * Receives theme data from the extension's syntax-theme bridge and makes it + * available to all Shiki consumers (CM6 decoration plugin, markdown worker, + * @pierre/diffs pool). Falls back to "OpenCode" when no VS Code theme has + * been received. + * + * @module + */ + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/** The fallback theme name used when no VS Code theme is available. */ +const FALLBACK_THEME = "OpenCode" + +/** Current active theme name (a Shiki built-in name or "vscode-active" for custom themes). */ +let activeThemeName: string = FALLBACK_THEME + +/** Full TextMate theme object for custom themes (null when using a built-in name). */ +let activeThemeObject: object | null = null + +/** Listeners notified when the theme changes. */ +const listeners: Set<(name: string) => void> = new Set() + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Get the current active Shiki theme name. + * Returns either a Shiki built-in name (e.g. "dark-plus", "dracula") or + * "vscode-active" for custom themes extracted from the extension filesystem. + */ +export function getActiveShikiTheme(): string { + return activeThemeName +} + +/** + * Get the full TextMate theme object for custom themes. + * Returns null when the active theme is a Shiki built-in (resolved by name). + */ +export function getActiveThemeObject(): object | null { + return activeThemeObject +} + +/** + * Subscribe to theme changes. The callback receives the new theme name. + * Returns an unsubscribe function. + */ +export function onThemeChange(cb: (name: string) => void): () => void { + listeners.add(cb) + return () => { listeners.delete(cb) } +} + +/** + * Handle an incoming syntax-theme message from the extension host. + * Called from the app's message bridge (AmicodeThemeBridge in app.tsx). + * + * @param theme — either a string (Shiki built-in name) or a full TextMate theme object + */ +export function handleSyntaxThemeMessage(theme: string | object): void { + if (typeof theme === "string") { + // Shiki built-in name + activeThemeName = theme + activeThemeObject = null + } else if (theme && typeof theme === "object") { + // Full TextMate theme object — register under a fixed name + activeThemeName = "vscode-active" + activeThemeObject = theme + // Inject the theme's token colors as CSS custom properties so that + // lezer's HighlightStyle, markdown code blocks, and every other + // consumer of --syntax-* picks up the VS Code theme colors immediately. + applySyntaxCSSVariables(theme as TextMateThemeData) + } else { + return + } + + for (const cb of listeners) { + try { cb(activeThemeName) } catch { /* listener errors don't break the bridge */ } + } +} + +// --------------------------------------------------------------------------- +// CSS variable injection — map VS Code tokenColors to --syntax-* variables +// --------------------------------------------------------------------------- + +interface TokenColorRule { + scope?: string | string[] + settings?: { foreground?: string; fontStyle?: string } +} + +interface TextMateThemeData { + tokenColors?: TokenColorRule[] + colors?: Record +} + +/** + * Map from --syntax-* CSS variable names to the TextMate scopes they represent. + * Order within each array is precedence: first match wins. + */ +const SCOPE_TO_VAR: [string, string[]][] = [ + ["--syntax-comment", ["comment", "punctuation.definition.comment"]], + ["--syntax-keyword", ["keyword", "keyword.control", "keyword.operator", "storage", "storage.type", "storage.modifier"]], + ["--syntax-string", ["string", "string.quoted", "punctuation.definition.string"]], + ["--syntax-primitive",["constant.numeric", "constant.language", "constant.character"]], + ["--syntax-type", ["entity.name.type", "support.type", "entity.name.class", "entity.name.namespace"]], + ["--syntax-property", ["entity.name.function", "support.function", "entity.other.attribute-name", "variable.other.property"]], + ["--syntax-constant", ["constant", "constant.other", "variable.language", "entity.name.constant"]], +] + +/** + * Extract foreground colors from the theme's tokenColors and set them as + * CSS custom properties on . This makes every --syntax-* consumer + * (lezer HighlightStyle, markdown code blocks, inline code) reflect the + * VS Code theme instantly — no Worker or Shiki needed. + */ +function applySyntaxCSSVariables(theme: TextMateThemeData): void { + if (typeof document === "undefined") return + const rules = theme.tokenColors + if (!Array.isArray(rules) || rules.length === 0) return + + // Build a scope → foreground map (last rule wins, matching VS Code's semantics) + const scopeColors = new Map() + for (const rule of rules) { + const fg = rule.settings?.foreground + if (!fg) continue + const scopes = Array.isArray(rule.scope) ? rule.scope + : typeof rule.scope === "string" ? rule.scope.split(",").map(s => s.trim()) + : [] + for (const scope of scopes) { + scopeColors.set(scope, fg) + } + } + + // Also extract editor.foreground for the base text color + const editorFg = theme.colors?.["editor.foreground"] + + const root = document.documentElement + for (const [varName, scopes] of SCOPE_TO_VAR) { + // Find the first matching scope + for (const scope of scopes) { + const color = scopeColors.get(scope) + if (color) { + root.style.setProperty(varName, color) + break + } + } + } + + // Set editor foreground as the base text color if available + if (editorFg) { + root.style.setProperty("--syntax-base", editorFg) + } +} + +/** + * Reset to the fallback theme (for tests or when the bridge disconnects). + */ +export function resetToFallback(): void { + activeThemeName = FALLBACK_THEME + activeThemeObject = null +}