feat(ai-components): bring the AI agent work to V10

.prettierignore

@@ -41,6 +41,9 @@ package/src/theme/generated/
 perf/gauntlet/out/
 perf/gauntlet/runtime/generated/
 
+# A2UI's conformance cases, kept as upstream wrote them
+package/__fixtures__/a2ui-conformance/
+
 # Workflow / tooling configs not owned by this project's Prettier
 .github/
 .husky/

.yarnrc.yml

@@ -22,6 +22,8 @@ npmPreapprovedPackages:
   - '@stream-io/state-store'
   - '@stream-io/worker-timer'
   - '@stream-io/ws'
+  - '@stream-io/stream-agents'
+  - '@a2ui/web_core'
 
 npmPublishProvenance: true
 

AGENTS.md

@@ -219,6 +219,8 @@ Native code shared by both wrappers lives in `package/shared-native/{ios,android
 - the main entry points must never reach `src/ai-components` or the AI-only packages (`react-syntax-highlighter`, `@khanacademy/simple-markdown`, `zod`, `victory-native`, Skia). `@stream-io/state-store` and `lodash` are shared: the main entry imports them too (the stores in `state-store/`, the Poll contexts);
 - the AI entry points may only reach `src/ai-components` plus an explicit allowlist (`src/native.ts`, the native shimmer). The wrappers' `ai-components/index.js` registers the native pieces the AI code needs itself and must never import the wrappers' `src/index.js`.
 
+`AIConversation` and its list sit behind a third entry point, `ai-components/conversation` (`package/ai-components/conversation/`, `package/{native,expo}-package/ai-components/conversation/`), because `@legendapp/list/keyboard` requires `react-native-keyboard-controller` unguarded. The isolation check keeps the `ai-components` and `a2ui` entry points from reaching `src/ai-components/conversation` or `@legendapp/list/keyboard`, and lets them load keyboard-controller only through the guarded `src/ai-components/internal/keyboard.tsx`. The conversation's state layer (`AIConversationController`, its types, `useAIConversationState`) lives in `src/ai-components/conversation-state` and stays in the plain entry.
+
 Do not add AI slots to `DEFAULT_COMPONENTS`; core `require`s the defaults, which would break isolation. Skia and `victory-native` are optional peers (native libraries must be owned by the app); the JS-only AI dependencies are regular dependencies.
 
 Native pieces live in `package/shared-native/`: the Android-only `PerfText` Fabric view (Kotlin plus a C++ shadow node in `shared-native/android/jni`, built through each wrapper's `android/src/main/jni/CMakeLists.txt`, registered with `cmakeListsPath`/`componentDescriptors` in `react-native.config.js`) and the `AIDictation` TurboModule. **The iOS dictation sources are excluded from the pod unless the app opts in** (`$StreamChatEnableDictation = true` in the Podfile, or the `stream-chat-expo` config plugin in `package/expo-package/plugin`), because referencing the Speech framework makes App Store Connect demand a speech recognition usage description from every app. Never declare `RECORD_AUDIO` in a library manifest. The dictation codegen spec is duplicated into each wrapper's `src/native/` (codegen only reads a package's own `jsSrcsDir`); the isolation check keeps the copies identical.

examples/ExpoChatGPTSample/.env.example

@@ -0,0 +1,15 @@
+# Copy to .env.local (gitignored) and fill in. Public values only: Expo inlines every EXPO_PUBLIC_
+# variable into the app bundle. The Stream secret stays with the token backend.
+
+# The Stream app the Vision-Agents router runs in.
+EXPO_PUBLIC_STREAM_API_KEY=YOUR_STREAM_API_KEY
+
+# The router and its token backend (Vision-Agents' `go run ./backend`). On an Android emulator,
+# `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098` makes localhost reach them.
+EXPO_PUBLIC_AGENT_ROUTER_URL=http://localhost:8080
+EXPO_PUBLIC_AGENT_TOKEN_URL=http://localhost:8098
+
+# The customer and agent `go run ./configure` stored, and who this app is.
+EXPO_PUBLIC_AGENT_CUSTOMER_ID=examples
+EXPO_PUBLIC_AGENT_NAME=swift_assistant
+EXPO_PUBLIC_AGENT_USER_ID=rn-ai-sample-user

examples/ExpoChatGPTSample/README.md

@@ -9,17 +9,31 @@ Ported from the `ExpoChatGPTSample` of the deprecated `@stream-io/chat-react-nat
 
 ## How it works
 
-- The drawer lists the user's conversations; `+` starts a new one.
-- A new conversation is created on its first message. Its title is summarized from that message by the sample backend.
-- Before sending, the app asks the sample backend to add an AI agent to the channel (`http/requests.ts`). The agent
-  replies with streamed `ai_generated` messages, rendered by `StreamingMessageView`.
-
-The Stream app, user and backend in `chatConfig.ts` and `http/requests.ts` are Stream's public demo setup. To use your
-own, replace them and point `http/requests.ts` at your own AI agent server.
+The agent runs on a [Vision-Agents](https://github.com/GetStream/Vision-Agents) router, which the app talks to through
+[`@stream-io/stream-agents`](https://www.npmjs.com/package/@stream-io/stream-agents) (`agent/`).
+
+- Each conversation is a router session, kept in an `agent` channel. The drawer lists the user's `agent` channels. `+`
+  starts a new session, and one that is never asked anything is deleted when you leave it.
+- The conversation is `AIConversation`, over an `AIConversationController` that `contexts/AppContext.tsx` builds from
+  the SDK's `AgentChat` and `agentChatTransport` (`@stream-io/stream-agents/chat`). `AgentChat` watches the channel,
+  loads older pages, keeps each reply's full reasoning from its live updates, and follows what the agent is doing on
+  the session's socket. The controller offers no edit or regenerate: the router can't rewind a conversation.
+- Asking goes through the session, not the chat composer. The router writes the question and the streamed reply into
+  the channel, the reply's steps as `ai_reasoning` and `ai_tool_call` attachments, and titles the conversation itself.
+  Photos go as data URIs, and "Think harder" as a high reasoning effort. The phone's tools (`agent/deviceTools.ts`) are
+  declared on the session, which runs them.
+- The keyboard is handled by the AI components through `react-native-keyboard-controller`; the app only wraps itself
+  in `KeyboardProvider`. The starters and the voice button sit in the composer's `header`, so they ride the keyboard.
 
 ## Running
 
-From the repository root:
+1. Run a Vision-Agents router locally, with its `swift_assistant` agent and token backend, as in
+   [`examples/voice_agents/swift_assistant`](https://github.com/GetStream/Vision-Agents/tree/accelerate/examples/voice_agents/swift_assistant):
+   `docker compose up` (with `ROUTER_AUTH_MODE=proxy`), `go run ./configure`, then `go run ./backend`. The chat model
+   is Grok, so the router needs `XAI_API_KEY`.
+2. Copy `.env.example` to `.env.local` and set `EXPO_PUBLIC_STREAM_API_KEY` to the Stream app the router runs in.
+3. On an Android emulator, forward the ports: `adb reverse tcp:8080 tcp:8080 && adb reverse tcp:8098 tcp:8098`.
+4. From the repository root:
 
 ```bash
 yarn install
@@ -28,3 +42,6 @@ yarn workspace expochatgptsample ios      # or: android
 
 `ios` and `android` sync the SDK's shared native code, run `expo prebuild` and build the app. The native `ios` and
 `android` folders are generated and not committed.
+
+On iOS, if the app crashes at launch in `dyld` with a missing symbol, reinstall the pods without Expo's
+precompiled modules: `cd ios && EXPO_USE_PRECOMPILED_MODULES=0 pod install`, then build again.

examples/ExpoChatGPTSample/agent/UIComponents.json

@@ -0,0 +1,76 @@
+[
+  {
+    "name": "place_card",
+    "description": "One place of a plan, with its photo and a short summary, which the phone loads from Wikipedia, so it needs no image URL. Use one per sight, beach, monastery, village or day trip.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "title": {
+          "type": "string",
+          "description": "The place's exact English Wikipedia article title, such as \"Church of St. John at Kaneo\""
+        },
+        "note": {
+          "type": "string",
+          "description": "One short line about it for this plan, such as \"Best at sunset\" or \"2 hours\""
+        }
+      },
+      "required": ["title"],
+      "x-order": ["title", "note"]
+    },
+    "events": [
+      {
+        "name": "ask_about",
+        "description": "Tapping the card asks you about the place."
+      }
+    ]
+  },
+  {
+    "name": "trip_map",
+    "description": "A map of a plan with a pin for each place, drawn on the phone. Use one near the top of every trip plan.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "places": {
+          "type": "array",
+          "description": "Every place of the plan, with its coordinates",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": { "type": "string" },
+              "latitude": { "type": "number" },
+              "longitude": { "type": "number" }
+            },
+            "required": ["name", "latitude", "longitude"],
+            "x-order": ["name", "latitude", "longitude"]
+          }
+        }
+      },
+      "required": ["places"],
+      "x-order": ["places"]
+    },
+    "events": [
+      {
+        "name": "select_place",
+        "description": "Tapping a pin asks you about that place."
+      }
+    ]
+  },
+  {
+    "name": "calendar_check",
+    "description": "The person's own calendar for the days of a plan, read on the phone. You never see it: use it whenever a plan has dates, so they can see what the trip would clash with.",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "start": {
+          "type": "string",
+          "format": "date",
+          "description": "The first day, as YYYY-MM-DD"
+        },
+        "end": { "type": "string", "format": "date", "description": "The last day, as YYYY-MM-DD" }
+      },
+      "required": ["start", "end"],
+      "x-order": ["start", "end"]
+    },
+    "events": []
+  }
+]

examples/ExpoChatGPTSample/agent/agents.ts

@@ -0,0 +1,32 @@
+import { Client } from '@stream-io/stream-agents';
+// Lets `session.chat()` open the conversation's channel on the app's Stream Chat client.
+import '@stream-io/stream-agents/chat';
+
+import { AgentConfig } from './config';
+
+/** The router, as this app's user. */
+export const agents = new Client({
+  apiKey: AgentConfig.streamApiKey,
+  customerId: AgentConfig.customerId,
+  url: AgentConfig.routerUrl,
+});
+
+/** The agent's conversations: opening a new one and carrying an old one on. */
+export const agentSessions = () => agents.agent(AgentConfig.agentName).sessions;
+
+/** Asks the token backend for this user's Stream token, which both the chat and the router take. */
+export const streamToken = async (): Promise<string> => {
+  const response = await fetch(`${AgentConfig.tokenUrl}/stream-token`, {
+    body: JSON.stringify({ user_id: AgentConfig.user.id }),
+    headers: { 'Content-Type': 'application/json' },
+    method: 'POST',
+  });
+  if (!response.ok) {
+    throw new Error(`The token backend answered ${response.status}: is \`go run ./backend\` up?`);
+  }
+  const { token } = (await response.json()) as { token?: string };
+  if (!token) {
+    throw new Error('The token backend answered without a token.');
+  }
+  return token;
+};

examples/ExpoChatGPTSample/agent/config.ts

@@ -0,0 +1,32 @@
+/**
+ * Where the Vision-Agents router and its token backend are, and who this app is. Set in
+ * `.env.local`, which `.env.example` describes. Only public values belong there: Expo inlines every
+ * `EXPO_PUBLIC_` variable into the bundle, so the Stream secret stays with the token backend.
+ */
+
+const streamApiKey = process.env.EXPO_PUBLIC_STREAM_API_KEY;
+
+if (!streamApiKey) {
+  throw new Error(
+    'EXPO_PUBLIC_STREAM_API_KEY is not set: copy .env.example to .env.local and fill it in.',
+  );
+}
+
+export const AgentConfig = {
+  /** The Stream app the router runs in. Its channels hold the conversations. */
+  streamApiKey,
+  /** The router. On an Android emulator, `adb reverse tcp:8080 tcp:8080` makes localhost reach it. */
+  routerUrl: process.env.EXPO_PUBLIC_AGENT_ROUTER_URL || 'http://localhost:8080',
+  /** Mints this user's Stream token (Vision-Agents' `go run ./backend`). */
+  tokenUrl: process.env.EXPO_PUBLIC_AGENT_TOKEN_URL || 'http://localhost:8098',
+  /** The customer `go run ./configure` stored the agent under. */
+  customerId: process.env.EXPO_PUBLIC_AGENT_CUSTOMER_ID || 'examples',
+  /** The agent, by the name in its `agent.yaml`. */
+  agentName: process.env.EXPO_PUBLIC_AGENT_NAME || 'swift_assistant',
+  /** A model that streams its reasoning, so replies have reasoning steps. */
+  chatModel: 'xai/grok-4.7',
+  user: {
+    id: process.env.EXPO_PUBLIC_AGENT_USER_ID || 'rn-ai-sample-user',
+    name: 'Alex',
+  },
+} as const;

examples/ExpoChatGPTSample/agent/conversation.ts

@@ -0,0 +1,50 @@
+import type { StateStore } from '@stream-io/state-store';
+import type { AgentChat, AgentChatState } from '@stream-io/stream-agents/chat';
+import type { LocalMessage } from 'stream-chat';
+
+/** A written conversation with the agent, over the messages of the channel it is kept in. */
+export type Chat = AgentChat<LocalMessage>;
+
+export type ChatState = AgentChatState<LocalMessage>;
+
+/**
+ * The chat's state for `useStateStore`. The SDK's store has the shape of `@stream-io/state-store`'s
+ * `StateStore` (`getLatestValue` and `subscribeWithSelector`, which is all `useStateStore` calls)
+ * without depending on it, so the type is only told so here.
+ */
+export const chatStore = (chat: Chat) => chat.state as unknown as StateStore<ChatState>;
+
+let keys = 0;
+const chatKeys = new WeakMap<Chat, string>();
+
+/** Tells a conversation apart from the one shown before it, for a React key. */
+export const keyOf = (chat: Chat) => {
+  let key = chatKeys.get(chat);
+  if (!key) {
+    key = `chat-${++keys}`;
+    chatKeys.set(chat, key);
+  }
+  return key;
+};
+
+/** A reply the agent wrote. */
+export const isAgentReply = (message: LocalMessage) => message.custom?.ai_generated === true;
+
+/**
+ * Calls `used` when a new conversation is used, such as asked something: the router lists it from
+ * then on. A first question that never reached the router has its session deleted, and the next
+ * one creates another, which is listed in its turn. A conversation carried on is listed already,
+ * and isn't waited for. Returns a function that stops waiting.
+ */
+export const whenUsed = (chat: Chat, used: () => void) => {
+  if (chat.id !== undefined) return () => {};
+  let listed: string | undefined;
+  return chat.state.subscribeWithSelector(
+    ({ usedSessionId }) => ({ usedSessionId }),
+    ({ usedSessionId }) => {
+      if (usedSessionId === undefined || usedSessionId === listed) return;
+      listed = usedSessionId;
+      used();
+    },
+  );
+};